diff --git a/.github/workflows/load-test.yml b/.github/workflows/load-test.yml index e83725e8f..8a38604a5 100644 --- a/.github/workflows/load-test.yml +++ b/.github/workflows/load-test.yml @@ -82,7 +82,7 @@ jobs: with: path: ./tests/load/drain-test.js - # Sample Bull queue depth in Redis until empty → drain rate + peak depth. + # Sample BullMQ queue depth in Redis until empty → drain rate + peak depth. - name: Measure queue drain if: always() continue-on-error: true # measurement must never block the ramps that follow @@ -125,15 +125,15 @@ jobs: exit 0 fi llen() { n=$(rc LLEN "$1" | tr -dc '0-9'); echo "${n:-0}"; } - # Bull keys are namespaced per deployment (REDIS_KEY_PREFIX, defaulting to + # Queue keys are namespaced per deployment (REDIS_KEY_PREFIX, defaulting to # RELEASE_NAME) because every deployment in the namespace shares one Redis. PREFIX=$(oc get deploy "$DEP" -o jsonpath='{.spec.template.spec.containers[*].env[?(@.name=="RELEASE_NAME")].value}' 2>/dev/null) PREFIX=${PREFIX:-notify} - echo "bull key prefix: $PREFIX" + echo "queue key prefix: $PREFIX:mq" depth() { t=0 for q in notification-ingestion email-delivery; do - for s in wait active; do t=$((t + $(llen "$PREFIX:$q:$s"))); done + for s in wait active; do t=$((t + $(llen "$PREFIX:mq:$q:$s"))); done done echo "$t" } diff --git a/backend/package-lock.json b/backend/package-lock.json index 825e80a9f..8eb5a5d1c 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -18,7 +18,7 @@ "@nestjs/typeorm": "^12.0.0", "axios": "^1.20.0", "body-parser": "^2.3.0", - "bull": "^4.14.1", + "bullmq": "^6.3.19", "class-transformer": "^0.5.1", "class-validator": "^0.15.0", "express-prom-bundle": "^8.0.0", @@ -1722,9 +1722,9 @@ "license": "MIT" }, "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", - "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.4.tgz", + "integrity": "sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==", "cpu": [ "arm64" ], @@ -1735,9 +1735,9 @@ ] }, "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", - "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.4.tgz", + "integrity": "sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==", "cpu": [ "x64" ], @@ -1748,9 +1748,9 @@ ] }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", - "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.4.tgz", + "integrity": "sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==", "cpu": [ "arm" ], @@ -1761,9 +1761,9 @@ ] }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", - "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.4.tgz", + "integrity": "sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==", "cpu": [ "arm64" ], @@ -1774,9 +1774,9 @@ ] }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", - "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.4.tgz", + "integrity": "sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==", "cpu": [ "x64" ], @@ -1787,9 +1787,9 @@ ] }, "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", - "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.4.tgz", + "integrity": "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==", "cpu": [ "x64" ], @@ -4973,22 +4973,61 @@ "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", "license": "BSD-3-Clause" }, - "node_modules/bull": { - "version": "4.16.5", - "resolved": "https://registry.npmjs.org/bull/-/bull-4.16.5.tgz", - "integrity": "sha512-lDsx2BzkKe7gkCYiT5Acj02DpTwDznl/VNN7Psn7M3USPG7Vs/BaClZJJTAG+ufAR9++N1/NiUTdaFBWDIl5TQ==", + "node_modules/bullmq": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-6.4.0.tgz", + "integrity": "sha512-wEx8b4tDmW1upAOXS0SCy0No+DBTAkV82GFP/rb0EAM3qtAOAXAXKJ2OKiWq4g7xZ1Bv+WpkwvvJx1mxWY+taw==", "license": "MIT", "dependencies": { - "cron-parser": "^4.9.0", - "get-port": "^5.1.1", - "ioredis": "^5.3.2", - "lodash": "^4.17.21", - "msgpackr": "^1.11.2", - "semver": "^7.5.2", - "uuid": "^8.3.0" + "cron-parser": "5.10.2", + "msgpackr": "2.1.0", + "node-abort-controller": "3.1.1", + "semver": "7.8.5", + "tslib": "2.8.1" }, "engines": { - "node": ">=12" + "node": ">=14.17.0" + }, + "peerDependencies": { + "bullmq-otel": ">=2.0.0", + "ioredis": ">=5.0.0", + "pg": ">=8.0.0", + "redis": ">=5.0.0" + }, + "peerDependenciesMeta": { + "bullmq-otel": { + "optional": true + }, + "ioredis": { + "optional": true + }, + "pg": { + "optional": true + }, + "redis": { + "optional": true + } + } + }, + "node_modules/bullmq/node_modules/cron-parser": { + "version": "5.10.2", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-5.10.2.tgz", + "integrity": "sha512-JfAYMY0LZaWQTuict8PhtF/ksSnq8xefSDmQUnnoqvkPYkU/y3UP0DTWzJKFBz0P1IXsDG8q0vY3zHkoFZOhvw==", + "license": "MIT", + "dependencies": { + "luxon": "^3.7.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/bullmq/node_modules/msgpackr": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-2.1.0.tgz", + "integrity": "sha512-p/pBCVO63CsvvpkomUnNNag6+n38rULuDA6HHe70o2gtC8ODI52foF/4ko2qQcp6OiErJXTmrZeXmsGGHsIQNQ==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.4" } }, "node_modules/busboy": { @@ -5540,18 +5579,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/cron-parser": { - "version": "4.9.0", - "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", - "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", - "license": "MIT", - "dependencies": { - "luxon": "^3.2.1" - }, - "engines": { - "node": ">=12.0.0" - } - }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -7190,18 +7217,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-port": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", - "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/get-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", @@ -9509,19 +9524,10 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, - "node_modules/msgpackr": { - "version": "1.11.10", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.10.tgz", - "integrity": "sha512-iCZNq+HszvF+fC3anCm4nBmWEnbeIAfpDs6IStAEKhQ2YSgkjzVG2FF9XJqwwQh5bH3N9OUTUt4QwVN6MLMLtA==", - "license": "MIT", - "optionalDependencies": { - "msgpackr-extract": "^3.0.2" - } - }, "node_modules/msgpackr-extract": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", - "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.4.tgz", + "integrity": "sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==", "hasInstallScript": true, "license": "MIT", "optional": true, @@ -9532,12 +9538,12 @@ "download-msgpackr-prebuilds": "bin/download-prebuilds.js" }, "optionalDependencies": { - "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", - "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", - "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", - "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", - "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", - "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.4" } }, "node_modules/multer": { @@ -9676,6 +9682,12 @@ "winston": "^3.0.0" } }, + "node_modules/node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", + "license": "MIT" + }, "node_modules/node-emoji": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-2.2.0.tgz", @@ -11657,9 +11669,9 @@ } }, "node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "license": "ISC", "bin": { "semver": "bin/semver.js" diff --git a/backend/package.json b/backend/package.json index 7d2cc3e02..3ad551d28 100644 --- a/backend/package.json +++ b/backend/package.json @@ -36,7 +36,7 @@ "@nestjs/typeorm": "^12.0.0", "axios": "^1.20.0", "body-parser": "^2.3.0", - "bull": "^4.14.1", + "bullmq": "^6.3.19", "class-transformer": "^0.5.1", "class-validator": "^0.15.0", "express-prom-bundle": "^8.0.0", diff --git a/backend/src/api/admin/monitoring/monitoring-events.service.spec.ts b/backend/src/api/admin/monitoring/monitoring-events.service.spec.ts index c7a9f1f92..fbcd37394 100644 --- a/backend/src/api/admin/monitoring/monitoring-events.service.spec.ts +++ b/backend/src/api/admin/monitoring/monitoring-events.service.spec.ts @@ -1,32 +1,35 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' import { EventEmitter } from 'events' import type { ConfigService } from '@nestjs/config' -import { CHANGE_THROTTLE_MS, MonitoringEventsService } from './monitoring-events.service' -import { createRedisClient } from '../../../queue/redis-connection' +import { + CHANGE_THROTTLE_MS, + MonitoringEventsService, + QUEUE_CHANGE_EVENTS, +} from './monitoring-events.service' +import { createQueueEvents } from '../../../queue/redis-connection' import { QueueName } from '../../../enum/queue-name.enum' -import { bullKey } from '../../../queue/queue-metrics' -vi.mock('../../../queue/redis-connection', () => ({ createRedisClient: vi.fn() })) +vi.mock('../../../queue/redis-connection', () => ({ createQueueEvents: vi.fn() })) -function fakeRedis() { - return Object.assign(new EventEmitter(), { - psubscribe: vi.fn().mockResolvedValue(4), - quit: vi.fn().mockResolvedValue('OK'), - disconnect: vi.fn(), - }) +function fakeQueueEvents() { + return Object.assign(new EventEmitter(), { close: vi.fn().mockResolvedValue(undefined) }) } const config = (redis: unknown) => ({ get: vi.fn(() => redis) }) as unknown as ConfigService describe('MonitoringEventsService', () => { - let redis: ReturnType + let readers: Array> beforeEach(() => { vi.useFakeTimers() - redis = fakeRedis() - vi.mocked(createRedisClient) + readers = [] + vi.mocked(createQueueEvents) .mockReset() - .mockReturnValue(redis as never) + .mockImplementation(() => { + const reader = fakeQueueEvents() + readers.push(reader) + return reader as never + }) }) afterEach(() => { @@ -35,23 +38,22 @@ describe('MonitoringEventsService', () => { it('opens no Redis connection until someone watches', () => { new MonitoringEventsService(config({ host: 'localhost', port: 6379, db: 0 })) - expect(createRedisClient).not.toHaveBeenCalled() + expect(createQueueEvents).not.toHaveBeenCalled() }) - it('subscribes to every queue and shares one connection between viewers', () => { + it('reads every queue and shares the readers between viewers', () => { const service = new MonitoringEventsService(config({ host: 'localhost', port: 6379, db: 0 })) const a = service.changes$.subscribe() const b = service.changes$.subscribe() - expect(createRedisClient).toHaveBeenCalledTimes(1) - expect(redis.psubscribe).toHaveBeenCalledWith( - ...Object.values(QueueName).map((name) => bullKey(name, '*')), + expect(vi.mocked(createQueueEvents).mock.calls.map(([name]) => name)).toEqual( + Object.values(QueueName), ) a.unsubscribe() - expect(redis.quit).not.toHaveBeenCalled() + expect(readers[0].close).not.toHaveBeenCalled() b.unsubscribe() - expect(redis.quit).toHaveBeenCalledTimes(1) + for (const reader of readers) expect(reader.close).toHaveBeenCalledTimes(1) }) it('signals the first change at once and coalesces a burst into one trailing signal', () => { @@ -59,7 +61,9 @@ describe('MonitoringEventsService', () => { const received = vi.fn() const subscription = service.changes$.subscribe(received) - for (let i = 0; i < 500; i++) redis.emit('pmessage', 'p', 'c', 'm') + for (let i = 0; i < 500; i++) { + readers[i % readers.length].emit(QUEUE_CHANGE_EVENTS[i % QUEUE_CHANGE_EVENTS.length], {}) + } expect(received).toHaveBeenCalledTimes(1) vi.advanceTimersByTime(CHANGE_THROTTLE_MS) @@ -75,6 +79,6 @@ describe('MonitoringEventsService', () => { const complete = vi.fn() service.changes$.subscribe({ complete }) expect(complete).toHaveBeenCalled() - expect(createRedisClient).not.toHaveBeenCalled() + expect(createQueueEvents).not.toHaveBeenCalled() }) }) diff --git a/backend/src/api/admin/monitoring/monitoring-events.service.ts b/backend/src/api/admin/monitoring/monitoring-events.service.ts index 7c4d25527..58de33565 100644 --- a/backend/src/api/admin/monitoring/monitoring-events.service.ts +++ b/backend/src/api/admin/monitoring/monitoring-events.service.ts @@ -2,9 +2,21 @@ import { Injectable, Logger } from '@nestjs/common' import { ConfigService } from '@nestjs/config' import { EMPTY, Observable, asyncScheduler, share, throttleTime } from 'rxjs' import { QueueName } from '../../../enum/queue-name.enum' -import { createRedisClient } from '../../../queue/redis-connection' +import { createQueueEvents } from '../../../queue/redis-connection' import type { RedisConfig } from '../../../queue/redis-connection' -import { bullKey } from '../../../queue/queue-metrics' + +/** The job events that change what the monitoring page shows. */ +export const QUEUE_CHANGE_EVENTS = [ + 'added', + 'waiting', + 'delayed', + 'active', + 'progress', + 'completed', + 'failed', + 'stalled', + 'removed', +] as const /** Coalesces bursts: a 5,000-recipient batch reports progress per recipient. */ export const CHANGE_THROTTLE_MS = 2_000 @@ -12,10 +24,10 @@ export const CHANGE_THROTTLE_MS = 2_000 /** * Signals that queue state has changed so the admin monitoring page can refetch its snapshot. * - * Bull publishes every add, start, progress update, completion and failure on channels under - * `::`, from whichever pod did the work, so one pattern subscription per queue - * hears all of them. The subscriber connection exists only while at least one admin has the page - * open: share() opens it for the first stream and closes it when the last one ends. + * BullMQ writes every add, start, progress update, completion and failure to each queue's events + * stream, from whichever pod did the work, so one QueueEvents reader per queue hears all of them. + * The readers exist only while at least one admin has the page open: share() opens them for the + * first stream and closes them when the last one ends. */ @Injectable() export class MonitoringEventsService { @@ -29,15 +41,18 @@ export class MonitoringEventsService { return } - const patterns = Object.values(QueueName).map((name) => bullKey(name, '*')) this.changes$ = new Observable((subscriber) => { - const redis = createRedisClient(redisConfig, 'MonitoringEvents') - redis.on('pmessage', () => subscriber.next()) - redis.psubscribe(...patterns).catch((error: Error) => { - this.logger.error(`Failed to subscribe to queue events: ${error.message}`) + const readers = Object.values(QueueName).map((name) => { + const events = createQueueEvents(name, redisConfig) + for (const event of QUEUE_CHANGE_EVENTS) events.on(event, () => subscriber.next()) + return events }) return () => { - redis.quit().catch(() => redis.disconnect()) + for (const events of readers) { + events.close().catch((error: Error) => { + this.logger.warn(`Failed to close queue events reader: ${error.message}`) + }) + } } }).pipe( throttleTime(CHANGE_THROTTLE_MS, asyncScheduler, { leading: true, trailing: true }), diff --git a/backend/src/api/admin/monitoring/monitoring.controller.ts b/backend/src/api/admin/monitoring/monitoring.controller.ts index 7f136eb47..e231b6fd3 100644 --- a/backend/src/api/admin/monitoring/monitoring.controller.ts +++ b/backend/src/api/admin/monitoring/monitoring.controller.ts @@ -11,7 +11,7 @@ import type { QueueMonitoringResponseDto } from './schemas/queue-monitoring.dto' const KEEPALIVE_MS = 25_000 /** - * Admin monitoring of the Bull queues, their workers and Redis. + * Admin monitoring of the queues, their workers and Redis. * * Route: /api/v1/frontend/admin/monitoring * Guarded by NotifyAdminGuard + NOTIFY_ADMIN (SSO role); not tenant-scoped. diff --git a/backend/src/api/admin/monitoring/monitoring.service.spec.ts b/backend/src/api/admin/monitoring/monitoring.service.spec.ts index 9b2aafdd8..9a8e304c2 100644 --- a/backend/src/api/admin/monitoring/monitoring.service.spec.ts +++ b/backend/src/api/admin/monitoring/monitoring.service.spec.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi } from 'vitest' import { ServiceUnavailableException } from '@nestjs/common' -import type Bull from 'bull' +import type { Job, Queue } from 'bullmq' import type Redis from 'ioredis' import type { Repository } from 'typeorm' import { @@ -88,7 +88,7 @@ describe('buildRedisStats', () => { expect(buildRedisStats(healthyInfo({ used_memory: 660 * MB })).status).toBe('critical') }) - it('is critical when Redis could evict Bull keys', () => { + it('is critical when Redis could evict queue keys', () => { const stats = buildRedisStats(healthyInfo({ maxmemory_policy: 'allkeys-lru', evicted_keys: 3 })) expect(stats.status).toBe('critical') expect(stats.reasons).toHaveLength(2) @@ -235,7 +235,7 @@ describe('countUnheldActiveJobs', () => { }, ], }) - const job = (ageMs: number) => ({ processedOn: now - ageMs }) as unknown as Bull.Job + const job = (ageMs: number) => ({ processedOn: now - ageMs }) as unknown as Job it('is zero when live workers report holding every active job', () => { expect(countUnheldActiveJobs('email-delivery', [job(60_000)], [pod(1)], now)).toBe(0) @@ -367,15 +367,11 @@ describe('MonitoringService', () => { data: [], count: 0, }), - getJob: vi.fn().mockResolvedValue({ timestamp: now - 30_000, opts: {} }), + getJobs: vi.fn().mockResolvedValue([{ timestamp: now - 30_000, opts: {} }]), getFailed: vi.fn().mockResolvedValue([]), getActive: vi.fn().mockResolvedValue([]), - client: { - lindex: vi.fn().mockResolvedValue('job-1'), - mget: vi.fn((...keys: string[]) => Promise.resolve(keys.map(() => null))), - }, ...overrides, - } as unknown as Bull.Queue + } as unknown as Queue } /** Reconcile activity as the pipeline returns it: last pass, recent actions, then minute counts. */ @@ -414,7 +410,14 @@ describe('MonitoringService', () => { ), zremrangebyscore: vi.fn().mockResolvedValue(0), zrange: vi.fn().mockResolvedValue(workers.map((_, i) => `pod-${i}`)), - mget: vi.fn().mockResolvedValue(workers.map((worker) => JSON.stringify(worker))), + // Worker heartbeats; the per-minute added counters are read through the same call. + mget: vi.fn((...keys: string[]) => + Promise.resolve( + keys[0]?.includes(':metrics:added:') + ? keys.map(() => null) + : workers.map((worker) => JSON.stringify(worker)), + ), + ), } as unknown as Redis } @@ -682,9 +685,9 @@ describe('MonitoringService', () => { id: 'req-1-EMAIL-0', processedOn: now - 20_000, data: { notifyId: 'req-1', tenantId: 't-1', mailMergeData: { recipients: ['secret'] } }, - progress: () => ({ sent: 37, failed: 2, total: 100 }), + progress: { sent: 37, failed: 2, total: 100 }, } - const single = { id: '9', processedOn: now, data: {}, progress: () => 0 } + const single = { id: '9', processedOn: now, data: {}, progress: 0 } const queue = fakeQueue('email-delivery', { getActive: vi.fn().mockResolvedValue([batch, single]), }) diff --git a/backend/src/api/admin/monitoring/monitoring.service.ts b/backend/src/api/admin/monitoring/monitoring.service.ts index bb02e1749..6898ae63a 100644 --- a/backend/src/api/admin/monitoring/monitoring.service.ts +++ b/backend/src/api/admin/monitoring/monitoring.service.ts @@ -2,7 +2,7 @@ import { Inject, Injectable, Optional, ServiceUnavailableException } from '@nest import { InjectRepository } from '@nestjs/typeorm' import { ConfigService } from '@nestjs/config' import { In, Repository } from 'typeorm' -import type Bull from 'bull' +import type { Job, Queue } from 'bullmq' import type Redis from 'ioredis' import { QueueName } from '../../../enum/queue-name.enum' import { ProviderToken } from '../../../enum/provider-token.enum' @@ -15,11 +15,10 @@ import type { BatchProgress } from '../../../queue/batch-progress' import { MONITORING_WINDOW_MINUTES, addedCounterKey, - alignBullMetrics, - bullKey, + alignQueueMetrics, minuteOf, } from '../../../queue/queue-metrics' -import type { BullMetrics } from '../../../queue/queue-metrics' +import type { QueueMetrics } from '../../../queue/queue-metrics' import { HEARTBEAT_INTERVAL_MS, HEARTBEAT_TTL_SECONDS, @@ -105,7 +104,7 @@ function estimateClearMinutes(pending: number, finishedPerMinute: number): numbe const TRAILING_PUNCTUATION = new Set(['.', ',', ';', ':', '!', '?', ')', ']', '}', '>', "'", '"']) /** - * Bull's failure reasons are provider error messages, which can quote the recipient. The page + * Failure reasons are provider error messages, which can quote the recipient. The page * is for spotting patterns, so addresses and numbers are masked before they leave the backend. */ export function maskPersonalData(text: string): string { @@ -153,7 +152,7 @@ export function buildRedisStats(info: Record): RedisStatsDto { } if (maxMemoryPolicy !== 'noeviction') { statuses.push('critical') - reasons.push(`Eviction policy is ${maxMemoryPolicy}; Bull jobs can be silently dropped`) + reasons.push(`Eviction policy is ${maxMemoryPolicy}; queued jobs can be silently dropped`) } if (evictedKeys > 0) { statuses.push('critical') @@ -310,7 +309,7 @@ export function evaluateQueue(input: QueueEvaluationInput, messagesMoving = fals input.unheldActiveJobs > 0 ? { status: 'warning', - reason: `${input.unheldActiveJobs} active job(s) not held by any live worker (possibly stalled; Bull retries stalled jobs within about a minute)`, + reason: `${input.unheldActiveJobs} active job(s) not held by any live worker (possibly stalled; stalled jobs are retried within about a minute)`, } : null, backlog > 0 && liveWorkerPods === 0 @@ -337,12 +336,12 @@ export function evaluateQueue(input: QueueEvaluationInput, messagesMoving = fals } type JobIdentifiers = { notifyId?: unknown; notificationId?: unknown; tenantId?: unknown } -type QueuedJob = { queue: string; job: Bull.Job } +type QueuedJob = { queue: string; job: Job } const asString = (value: unknown): string | null => (typeof value === 'string' ? value : null) /** Only identifiers leave the job: its data also holds recipients and message content. */ -function jobIdentifiers(job: Bull.Job, tenantNames: Map) { +function jobIdentifiers(job: Job, tenantNames: Map) { const data = (job.data ?? {}) as JobIdentifiers const tenantId = asString(data.tenantId) return { @@ -354,12 +353,12 @@ function jobIdentifiers(job: Bull.Job, tenantNames: Map) { /** Active jobs that report merge-batch progress; other jobs finish too fast to need a bar. */ function activeBatches( - activeByQueue: Map, + activeByQueue: Map, ): Array { return [...activeByQueue.entries()] .flatMap(([queue, jobs]) => jobs.flatMap((job) => { - const progress = readBatchProgress(job.progress()) + const progress = readBatchProgress(job.progress) return progress ? [{ queue, job, progress }] : [] }), ) @@ -417,12 +416,12 @@ export function buildSendsInProgress( /** * Active jobs no live worker reports holding. A job stays "active" in Redis after the pod - * processing it dies, until Bull's stalled-job check moves it back to waiting, so these are + * processing it dies, until the stalled-job check moves it back to waiting, so these are * likely stalled. Only jobs older than UNHELD_GRACE_MS are counted, to allow for heartbeat lag. */ export function countUnheldActiveJobs( queueName: string, - activeJobs: Bull.Job[], + activeJobs: Job[], workers: WorkerPodDto[], now: number, ): number { @@ -539,27 +538,27 @@ export function buildReconcilerStats( } /** - * Read-only view of the Bull queues, their workers and the Redis instance behind them, for the + * Read-only view of the queues, their workers and the Redis instance behind them, for the * admin monitoring page. Everything is read from Redis on each request, so the answer is the * same whichever pod serves it. */ @Injectable() export class MonitoringService { - private readonly queues: Bull.Queue[] + private readonly queues: Queue[] constructor( @Optional() @Inject(ProviderToken.REDIS_CLIENT) private readonly redis: Redis | null, - @Optional() @Inject(QueueName.INGESTION) ingestionQueue: Bull.Queue | null, - @Optional() @Inject(QueueName.EMAIL_DELIVERY) emailQueue: Bull.Queue | null, - @Optional() @Inject(QueueName.SMS_DELIVERY) smsQueue: Bull.Queue | null, - @Optional() @Inject(QueueName.WEBHOOK_DELIVERY) webhookQueue: Bull.Queue | null, + @Optional() @Inject(QueueName.INGESTION) ingestionQueue: Queue | null, + @Optional() @Inject(QueueName.EMAIL_DELIVERY) emailQueue: Queue | null, + @Optional() @Inject(QueueName.SMS_DELIVERY) smsQueue: Queue | null, + @Optional() @Inject(QueueName.WEBHOOK_DELIVERY) webhookQueue: Queue | null, @InjectRepository(Tenant) private readonly tenantRepository: Repository, @InjectRepository(NotificationRequestDetail) private readonly detailRepository: Repository, @Optional() private readonly configService?: ConfigService, ) { this.queues = [ingestionQueue, emailQueue, smsQueue, webhookQueue].filter( - (queue): queue is Bull.Queue => !!queue, + (queue): queue is Queue => !!queue, ) } @@ -636,9 +635,9 @@ export class MonitoringService { } private async getQueueStats( - queue: Bull.Queue, + queue: Queue, workers: WorkerPodDto[], - activeJobs: Bull.Job[], + activeJobs: Job[], messages: MessageChannelStatsDto[], now: number, ): Promise { @@ -651,8 +650,8 @@ export class MonitoringService { queue.getMetrics('failed', 0, MONITORING_WINDOW_MINUTES - 1), ]) - const completedSeries = alignBullMetrics(completed as BullMetrics, now) - const failedSeries = alignBullMetrics(failed as BullMetrics, now) + const completedSeries = alignQueueMetrics(completed as QueueMetrics, now) + const failedSeries = alignQueueMetrics(failed as QueueMetrics, now) const window = MONITORING_THRESHOLDS.rateWindowMinutes const completedPerMinute = averageOfWholeMinutes(completedSeries, window) const failedPerMinute = averageOfWholeMinutes(failedSeries, window) @@ -664,12 +663,14 @@ export class MonitoringService { { name: queue.name, isPaused, + // A paused BullMQ queue keeps its jobs in waiting; they are reported as paused, as they + // were when a paused queue moved them to a list of their own. counts: { - waiting: counts.waiting, - active: counts.active, - delayed: counts.delayed, - failed: counts.failed, - paused: (counts as Bull.JobCounts & { paused?: number }).paused ?? 0, + waiting: isPaused ? 0 : (counts.waiting ?? 0), + active: counts.active ?? 0, + delayed: counts.delayed ?? 0, + failed: counts.failed ?? 0, + paused: isPaused ? (counts.waiting ?? 0) : 0, }, oldestWaitingAgeMs, throughput: { @@ -689,28 +690,23 @@ export class MonitoringService { } /** - * Workers take from the right of the wait list, so its last entry is the next job out. A - * delayed job (scheduled send or retry backoff) only became ready at timestamp + delay, which - * is what its wait is measured from. Pausing moves the wait list to `paused`. + * The oldest waiting job is the next one out. A delayed job (scheduled send or retry backoff) + * only became ready at timestamp + delay, which is what its wait is measured from. */ - private async getOldestWaitingAge(queue: Bull.Queue, now: number): Promise { - const jobId = - (await queue.client.lindex(bullKey(queue.name, 'wait'), -1)) ?? - (await queue.client.lindex(bullKey(queue.name, 'paused'), -1)) - if (!jobId) return null - const job = await queue.getJob(jobId) + private async getOldestWaitingAge(queue: Queue, now: number): Promise { + const [job] = await queue.getJobs(['waiting'], 0, 0, true) if (!job) return null const readyAt = job.timestamp + (job.opts?.delay ?? 0) return Math.max(0, now - readyAt) } - private async getAddedSeries(queue: Bull.Queue, now: number): Promise { + private async getAddedSeries(queue: Queue, now: number): Promise { const currentMinute = minuteOf(now) const keys: string[] = [] for (let offset = MONITORING_WINDOW_MINUTES - 1; offset >= 0; offset--) { keys.push(addedCounterKey(queue.name, currentMinute - offset)) } - const values = await queue.client.mget(...keys) + const values = await (this.redis as Redis).mget(...keys) return values.map((value) => Number(value ?? 0) || 0) } @@ -752,7 +748,7 @@ export class MonitoringService { .slice(0, RECENT_FAILURES_LIMIT) } - private async getActiveJobs(): Promise> { + private async getActiveJobs(): Promise> { const perQueue = await Promise.all( this.queues.map( async (queue) => diff --git a/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts b/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts index 91e4bad44..be5d662ff 100644 --- a/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts +++ b/backend/src/api/admin/monitoring/schemas/queue-monitoring.dto.ts @@ -85,7 +85,7 @@ export interface RecentFailureDto { notificationId: string | null tenantId: string | null tenantName: string | null - /** Bull's failure reason with email addresses and phone numbers masked. */ + /** The job's failure reason with email addresses and phone numbers masked. */ reason: string attemptsMade: number failedAt: string | null diff --git a/backend/src/api/gc-notify/gc-notify-internal-execution.service.spec.ts b/backend/src/api/gc-notify/gc-notify-internal-execution.service.spec.ts index ac3d1c5c8..a19ba46fb 100644 --- a/backend/src/api/gc-notify/gc-notify-internal-execution.service.spec.ts +++ b/backend/src/api/gc-notify/gc-notify-internal-execution.service.spec.ts @@ -55,7 +55,7 @@ describe('GcNotifyInternalExecutionService', () => { updateStatus: ReturnType } let mockConfigurationRepository: { findOne: ReturnType } - let mockIngestionQueue: { add: ReturnType } + let mockIngestionQueue: { name: string; add: ReturnType } let mockAttachmentValidationService: { validateAttachments: ReturnType } let mockAttachmentProcessingService: { processAttachments: ReturnType } let mockSafelistService: { findBlocked: ReturnType } @@ -88,7 +88,10 @@ describe('GcNotifyInternalExecutionService', () => { } mockNotificationRequestDetailService = { createPending: vi.fn(), updateStatus: vi.fn() } mockConfigurationRepository = { findOne: vi.fn().mockResolvedValue(null) } - mockIngestionQueue = { add: vi.fn().mockResolvedValue(undefined) } + mockIngestionQueue = { + name: 'notification-ingestion', + add: vi.fn().mockResolvedValue(undefined), + } mockAttachmentValidationService = { validateAttachments: vi.fn().mockResolvedValue(undefined) } mockAttachmentProcessingService = { // By default, echo back a single stored reference so callers can assert wiring. @@ -210,6 +213,7 @@ describe('GcNotifyInternalExecutionService', () => { await flushMicrotasks() expect(mockIngestionQueue.add).toHaveBeenCalledWith( + 'notification-ingestion', expect.objectContaining({ notifyId: 'notif-1', tenantId: TENANT_ID }), expect.objectContaining({ jobId: 'notif-1' }), ) @@ -239,7 +243,7 @@ describe('GcNotifyInternalExecutionService', () => { await service.sendEmail(body, TENANT_ID) await flushMicrotasks() - const [[jobPayload]] = mockIngestionQueue.add.mock.calls + const [[, jobPayload]] = mockIngestionQueue.add.mock.calls const content = jobPayload.request.email.content // GC Notify renders `#` as an h2, so this also pins the dialect, not just the conversion. @@ -336,7 +340,7 @@ describe('GcNotifyInternalExecutionService', () => { expect(mockAttachmentProcessingService.processAttachments).not.toHaveBeenCalled() await flushMicrotasks() - const [[jobPayload]] = mockIngestionQueue.add.mock.calls + const [[, jobPayload]] = mockIngestionQueue.add.mock.calls expect(jobPayload.request.email.attachments).toBeUndefined() }) @@ -374,7 +378,7 @@ describe('GcNotifyInternalExecutionService', () => { expect(mockAttachmentValidationService.validateAttachments).toHaveBeenCalledOnce() await flushMicrotasks() - const [[jobPayload]] = mockIngestionQueue.add.mock.calls + const [[, jobPayload]] = mockIngestionQueue.add.mock.calls expect(jobPayload.request.email.attachments).toEqual([{ attachmentId: 'att-1' }]) }) diff --git a/backend/src/api/gc-notify/gc-notify-internal-execution.service.ts b/backend/src/api/gc-notify/gc-notify-internal-execution.service.ts index 68e824c57..73c1831df 100644 --- a/backend/src/api/gc-notify/gc-notify-internal-execution.service.ts +++ b/backend/src/api/gc-notify/gc-notify-internal-execution.service.ts @@ -7,7 +7,7 @@ import { } from '@nestjs/common' import { InjectRepository } from '@nestjs/typeorm' import { Repository } from 'typeorm' -import Bull from 'bull' +import type { Queue } from 'bullmq' import { TemplatesRepository } from '../templates/templates.repository' import { TemplatesService } from '../templates/templates.service' import { Template } from '../templates/entities/template.entity' @@ -55,7 +55,7 @@ import type { NotifySimpleRequest } from '../notify/schemas/notify-simple-reques import type { NotifyEmailChannel } from '../notify/schemas/notify-email-channel' import type { NotifyAttachment } from '../notify/schemas/notify-attachment' import type { StoredNotifyAttachment } from '../notify/schemas/stored-notify-attachment' -import { COMPLETED_JOB_RETENTION, FAILED_JOB_RETENTION } from '../../queue/job-retention' +import { INGESTION_JOB_OPTIONS } from '../../queue/job-options' const TEMPLATE_LIST_QUERY_CONFIG: QueryableFieldsConfig = { sortableFields: { name: 'template.name', updatedAt: 'template.updatedAt' }, @@ -181,7 +181,7 @@ export class GcNotifyInternalExecutionService { private readonly tenantSettingsService: TenantSettingsService, @InjectRepository(NotifyConfiguration) private readonly configurationRepository: Repository, - @Inject(QueueName.INGESTION) private readonly ingestionQueue: Bull.Queue, + @Inject(QueueName.INGESTION) private readonly ingestionQueue: Queue, private readonly attachmentValidationService: AttachmentValidationService, private readonly attachmentProcessingService: AttachmentProcessingService, private readonly bulkValidationService: GcNotifyBulkValidationService, @@ -387,7 +387,7 @@ export class GcNotifyInternalExecutionService { const accepted = (await handleMerge( this.queueableContext(), - this.ingestionQueue as unknown as Bull.Queue, + this.ingestionQueue as unknown as Queue, QueueName.INGESTION, tenantId, notifyRequest, @@ -496,7 +496,7 @@ export class GcNotifyInternalExecutionService { apiKeyUsageService: this.apiKeyUsageService, smsSegmentService: this.smsSegmentService, notificationDedupService: this.notificationDedupService, - queueMap: new Map([[QueueName.INGESTION, this.ingestionQueue as unknown as Bull.Queue]]), + queueMap: new Map([[QueueName.INGESTION, this.ingestionQueue as unknown as Queue]]), } } @@ -794,12 +794,9 @@ export class GcNotifyInternalExecutionService { ...(scheduledFor && { scheduledFor }), } - await this.ingestionQueue.add(jobPayload, { + await this.ingestionQueue.add(this.ingestionQueue.name, jobPayload, { jobId: notificationRecord.id, - attempts: 3, - backoff: { type: 'exponential', delay: 2000 }, - removeOnComplete: COMPLETED_JOB_RETENTION, - removeOnFail: FAILED_JOB_RETENTION, + ...INGESTION_JOB_OPTIONS, ...(delayMs > 0 && { delay: delayMs }), }) diff --git a/backend/src/api/notification/notification-request-detail.service.spec.ts b/backend/src/api/notification/notification-request-detail.service.spec.ts index b0a1fec05..780ded47e 100644 --- a/backend/src/api/notification/notification-request-detail.service.spec.ts +++ b/backend/src/api/notification/notification-request-detail.service.spec.ts @@ -15,6 +15,7 @@ describe('NotificationRequestDetailService', () => { update: vi.fn().mockResolvedValue(undefined), increment: vi.fn().mockResolvedValue(undefined), find: vi.fn().mockResolvedValue([]), + createQueryBuilder: vi.fn(), } beforeEach(async () => { @@ -97,6 +98,37 @@ describe('NotificationRequestDetailService', () => { }) }) + it('markFailed leaves delivered and blocked rows alone', async () => { + await service.markFailed('request-1', 'gave up') + + const [where, values] = detailRepository.update.mock.calls[0] + expect(excludedStatuses(where)).toEqual(['sent', 'blocked']) + expect(values).toMatchObject({ status: 'failed', errorMessage: 'gave up' }) + }) + + describe('countByStatuses', () => { + it('counts every status in one grouped query', async () => { + const builder = { + select: vi.fn().mockReturnThis(), + addSelect: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + groupBy: vi.fn().mockReturnThis(), + getRawMany: vi.fn().mockResolvedValue([ + { status: 'sent', count: '7' }, + { status: 'failed', count: '2' }, + ]), + } + detailRepository.createQueryBuilder.mockReturnValue(builder) + + await expect(service.countByStatuses('request-1')).resolves.toEqual({ sent: 7, failed: 2 }) + expect(builder.where).toHaveBeenCalledWith(expect.any(String), { + notificationRequestId: 'request-1', + }) + expect(builder.groupBy).toHaveBeenCalledWith('detail.status') + expect(builder.getRawMany).toHaveBeenCalledTimes(1) + }) + }) + describe('createBlocked', () => { it('records one row per blocked recipient with the reason', async () => { await service.createBlocked( diff --git a/backend/src/api/notification/notification-request-detail.service.ts b/backend/src/api/notification/notification-request-detail.service.ts index 554dd15cd..ffa158f67 100644 --- a/backend/src/api/notification/notification-request-detail.service.ts +++ b/backend/src/api/notification/notification-request-detail.service.ts @@ -267,10 +267,18 @@ export class NotificationRequestDetailService { } /** - * Count detail records for a request in a given status (used to reconcile the parent request). + * A request's detail rows counted per status, in one query, for settling the parent request. + * A status with no rows is absent. */ - async countByStatus(notificationRequestId: string, status: string): Promise { - return this.detailRepository.count({ where: { notificationRequestId, status } }) + async countByStatuses(notificationRequestId: string): Promise> { + const rows = await this.detailRepository + .createQueryBuilder('detail') + .select('detail.status', 'status') + .addSelect('COUNT(*)', 'count') + .where('detail.notificationRequestId = :notificationRequestId', { notificationRequestId }) + .groupBy('detail.status') + .getRawMany<{ status: string; count: string }>() + return Object.fromEntries(rows.map(({ status, count }) => [status, Number(count)])) } /** @@ -292,11 +300,12 @@ export class NotificationRequestDetailService { // need more info on what the smtp gateway returns on individual failure /** - * Mark all request detail records for a request as failed. Only called on the final attempt. + * Mark a request's undelivered detail records failed. Only called once retrying has ended. */ async markFailed(notificationRequestId: string, errorMessage: string): Promise { + // A recipient already reached, or deliberately refused, keeps that record. await this.detailRepository.update( - { notificationRequestId }, + { notificationRequestId, status: Not(In(RETRY_PRESERVED_STATUSES)) }, { status: 'failed', errorMessage, lastAttemptAt: new Date(), updatedBy: 'system' }, ) } diff --git a/backend/src/api/notify/notify.controller.spec.ts b/backend/src/api/notify/notify.controller.spec.ts index 95090e441..50af49d53 100644 --- a/backend/src/api/notify/notify.controller.spec.ts +++ b/backend/src/api/notify/notify.controller.spec.ts @@ -82,6 +82,8 @@ const mockNotificationService = { { address: 'bob@example.com', params: {} }, ]), validateMailMergeRules: vi.fn().mockResolvedValue([]), + cancelOrRescheduleNotification: vi.fn(), + mapToDto: vi.fn((notification) => notification), } const mockAttachmentValidationService = { @@ -93,8 +95,9 @@ const mockAttachmentProcessingService = { } const mockIngestionQueue = { + name: 'notification-ingestion', add: vi.fn(), - process: vi.fn(), + getJob: vi.fn(), } const mockFeatureFlagService = { @@ -312,6 +315,88 @@ describe('Notify Controllers', () => { } }) + describe('cancel and reschedule', () => { + const cancelOrReschedule = (body: object, tenantId = 'tenant-1') => + (app.get(NotifySimpleController) as any).doCancelOrReschedule( + tenantId, + 'user-1', + 'notification-1', + body, + ) + + function queuedJob(state: string, tenantId = 'tenant-1') { + return { + id: 'notification-1', + data: { notifyId: 'notification-1', tenantId, scheduledFor: '2026-11-01T12:00:00.000Z' }, + getState: vi.fn().mockResolvedValue(state), + remove: vi.fn().mockResolvedValue(undefined), + } + } + + beforeEach(() => { + mockNotificationService.cancelOrRescheduleNotification.mockResolvedValue({ + id: 'notification-1', + }) + }) + + it('removes a cancelled send’s ingestion job, found by its id', async () => { + const job = queuedJob('delayed') + mockIngestionQueue.getJob.mockResolvedValue(job) + + await cancelOrReschedule({ action: 'cancel' }) + + expect(mockIngestionQueue.getJob).toHaveBeenCalledWith('notification-1') + expect(job.remove).toHaveBeenCalled() + }) + + it('re-queues a rescheduled send under the same id, due at the new time', async () => { + const job = queuedJob('delayed') + mockIngestionQueue.getJob.mockResolvedValue(job) + const sendAt = new Date(Date.now() + 2 * 60 * 60_000) + + await cancelOrReschedule({ scheduledTime: sendAt.toISOString() }) + + expect(job.remove).toHaveBeenCalled() + const [name, data, options] = mockIngestionQueue.add.mock.calls[0] + expect(name).toBe('notification-ingestion') + // Ingestion delays the delivery jobs until scheduledFor, so it moves with the job. + expect(data).toMatchObject({ + notifyId: 'notification-1', + scheduledFor: sendAt.toISOString(), + }) + expect(options).toMatchObject({ jobId: 'notification-1', attempts: 3 }) + expect(options.delay).toBeGreaterThan(119 * 60_000) + expect(options.delay).toBeLessThanOrEqual(120 * 60_000) + }) + + it.each(['active', 'completed'])('leaves a job that is already %s alone', async (state) => { + const job = queuedJob(state) + mockIngestionQueue.getJob.mockResolvedValue(job) + + await cancelOrReschedule({ action: 'cancel' }) + + expect(job.remove).not.toHaveBeenCalled() + }) + + it('never touches another tenant’s job', async () => { + const job = queuedJob('delayed', 'tenant-2') + mockIngestionQueue.getJob.mockResolvedValue(job) + + await cancelOrReschedule({ scheduledTime: new Date(Date.now() + 60_000).toISOString() }) + + expect(job.remove).not.toHaveBeenCalled() + expect(mockIngestionQueue.add).not.toHaveBeenCalled() + }) + + it('still returns the updated request when there is no job to change', async () => { + mockIngestionQueue.getJob.mockResolvedValue(undefined) + + await expect(cancelOrReschedule({ action: 'cancel' })).resolves.toEqual({ + id: 'notification-1', + }) + }) + }) + describe('accepted usage limit-alert wiring', () => { const dayPeriodStart = new Date('2026-07-29T00:00:00.000Z') const yearPeriodStart = new Date('2026-04-01T00:00:00.000Z') @@ -820,6 +905,7 @@ describe('Notify Controllers', () => { await vi.waitFor(() => { expect(mockIngestionQueue.add).toHaveBeenCalledWith( + 'notification-ingestion', expect.objectContaining({ request: expect.objectContaining({ sms: expect.objectContaining({ @@ -1545,6 +1631,7 @@ describe('Notify Controllers', () => { await vi.waitFor(() => { expect(mockIngestionQueue.add).toHaveBeenCalledWith( + 'notification-ingestion', expect.objectContaining({ request: processedPayload, }), @@ -1553,6 +1640,7 @@ describe('Notify Controllers', () => { }) expect(mockIngestionQueue.add).toHaveBeenCalledWith( + 'notification-ingestion', expect.objectContaining({ request: expect.not.objectContaining({ email: expect.objectContaining({ diff --git a/backend/src/api/notify/notify.controller.ts b/backend/src/api/notify/notify.controller.ts index 53f2fa648..44c5c3980 100644 --- a/backend/src/api/notify/notify.controller.ts +++ b/backend/src/api/notify/notify.controller.ts @@ -18,7 +18,7 @@ import { Request, Req, } from '@nestjs/common' -import Bull from 'bull' +import type { Queue } from 'bullmq' import { FeatureFlagGuard } from '../../common/guards/feature-flag.guard' import { SmsChannelFeatureFlagGuard } from '../../common/guards/sms-channel-feature-flag.guard' import { HtmlBodyTypeFeatureFlagGuard } from '../../common/guards/html-body-type-feature-flag.guard' @@ -43,6 +43,7 @@ import { } from './schemas/cancel-or-reschedule.dto' import { Queueable } from '../../common/decorators/queueable.decorator' import { QueueName } from '../../enum/queue-name.enum' +import { INGESTION_JOB_OPTIONS } from '../../queue/job-options' import { NotificationChannel } from '../../enum/notification-channel.enum' import { FeatureFlagCode } from '../../enum/feature-flag-code.enum' import { NotificationService } from '../notification/notification.service' @@ -90,7 +91,7 @@ import { @Controller('notifysimple') @UseGuards(NotifyServiceGuard, HtmlBodyTypeFeatureFlagGuard) export class NotifySimpleController { - private readonly queueMap: Map + private readonly queueMap: Map constructor( private readonly notifyService: NotifyService, @@ -103,7 +104,7 @@ export class NotifySimpleController { readonly safelistService: SafelistService, readonly smsSegmentService: SmsSegmentService, readonly notificationDedupService: NotificationDedupService, - @Inject(QueueName.INGESTION) private readonly ingestionQueue: Bull.Queue, + @Inject(QueueName.INGESTION) private readonly ingestionQueue: Queue, ) { this.queueMap = new Map([[QueueName.INGESTION, this.ingestionQueue]]) } @@ -518,6 +519,17 @@ export class NotifySimpleController { ) } + /** + * The request's ingestion job while it has not started - waiting, or delayed until its send + * time - and only if it belongs to this tenant. Null once it is running or done. + */ + private async pendingIngestionJob(notificationId: string, tenantId: string) { + const job = await this.ingestionQueue.getJob(notificationId) + if (!job || job.data?.tenantId !== tenantId) return null + const state = await job.getState() + return state === 'delayed' || state === 'waiting' ? job : null + } + private async doCancelOrReschedule( tenantId: string, updatedBy: string, @@ -554,8 +566,8 @@ export class NotifySimpleController { // If cancelling, attempt to remove from queue (non-fatal if it fails) if (action === 'cancel') { try { - const jobs = await this.ingestionQueue.getJobs(['delayed', 'waiting']) - const jobToRemove = jobs.find((job) => job.data?.notificationId === notificationId) + // The ingestion job is keyed by the notifyId. + const jobToRemove = await this.pendingIngestionJob(notificationId, tenantId) if (jobToRemove) { await jobToRemove.remove() @@ -573,19 +585,23 @@ export class NotifySimpleController { } else if (scheduledTime) { // If rescheduling, attempt to update the job delay (non-fatal if it fails) try { - const newDelay = new Date(scheduledTime).getTime() - Date.now() - - const jobs = await this.ingestionQueue.getJobs(['delayed', 'waiting']) - const jobToUpdate = jobs.find((job) => job.data?.notificationId === notificationId) + const sendAt = new Date(scheduledTime) + const newDelay = sendAt.getTime() - Date.now() + const jobToUpdate = await this.pendingIngestionJob(notificationId, tenantId) if (jobToUpdate) { await jobToUpdate.remove() - await this.ingestionQueue.add(jobToUpdate.data, { - delay: Math.max(0, newDelay), - jobId: `${tenantId}-${notificationId}`, - attempts: jobToUpdate.opts.attempts, - backoff: jobToUpdate.opts.backoff, - }) + // Same id, so dedup and the reconciler still find it. scheduledFor moves too: ingestion + // delays the delivery jobs until it, so a send moved earlier would otherwise wait. + await this.ingestionQueue.add( + this.ingestionQueue.name, + { ...jobToUpdate.data, scheduledFor: sendAt.toISOString() }, + { + jobId: notificationId, + ...INGESTION_JOB_OPTIONS, + delay: Math.max(0, newDelay), + }, + ) logger.log(`Updated queue job delay for rescheduled notification: ${notificationId}`, { tenantId, newDelay, @@ -631,7 +647,7 @@ export class NotifySimpleController { @Controller('frontend/notifysimple') @UseGuards(NotifyFrontendRoleGuard, HtmlBodyTypeFeatureFlagGuard) export class NotifySimpleFrontendController { - private readonly queueMap: Map + private readonly queueMap: Map constructor( private readonly notifyService: NotifyService, @@ -645,7 +661,7 @@ export class NotifySimpleFrontendController { readonly smsSegmentService: SmsSegmentService, readonly eventNotificationResolver: EventNotificationResolver, readonly notificationDedupService: NotificationDedupService, - @Inject(QueueName.INGESTION) private readonly ingestionQueue: Bull.Queue, + @Inject(QueueName.INGESTION) private readonly ingestionQueue: Queue, ) { this.queueMap = new Map([[QueueName.INGESTION, this.ingestionQueue]]) } diff --git a/backend/src/api/notify/services/limit-alert-notification.service.spec.ts b/backend/src/api/notify/services/limit-alert-notification.service.spec.ts index 6f18ea675..0bbe0c850 100644 --- a/backend/src/api/notify/services/limit-alert-notification.service.spec.ts +++ b/backend/src/api/notify/services/limit-alert-notification.service.spec.ts @@ -12,6 +12,7 @@ import { ProcessLimitAlertNotificationsInput, } from './limit-alert-notification.service' import { COMPLETED_JOB_RETENTION, FAILED_JOB_RETENTION } from '../../../queue/job-retention' +import { BACKOFF_STRATEGY } from '../../../queue/job-options' describe('LimitAlertNotificationService', () => { let service: LimitAlertNotificationService @@ -26,6 +27,7 @@ describe('LimitAlertNotificationService', () => { update: vi.fn(), } const ingestionQueue = { + name: 'notification-ingestion', add: vi.fn(), } const tenantsService = { @@ -233,6 +235,7 @@ describe('LimitAlertNotificationService', () => { const createdPayload = notificationService.create.mock.calls[0][0].payload expect(ingestionQueue.add).toHaveBeenCalledWith( + 'notification-ingestion', { notifyId: 'notification-1', tenantId: 'tenant-1', @@ -242,7 +245,7 @@ describe('LimitAlertNotificationService', () => { { jobId: 'notification-1', attempts: 3, - backoff: { type: 'exponential', delay: 2000 }, + backoff: { type: BACKOFF_STRATEGY, delay: 2000 }, removeOnComplete: COMPLETED_JOB_RETENTION, removeOnFail: FAILED_JOB_RETENTION, }, diff --git a/backend/src/api/notify/services/limit-alert-notification.service.ts b/backend/src/api/notify/services/limit-alert-notification.service.ts index 6dacf8769..bb58c9250 100644 --- a/backend/src/api/notify/services/limit-alert-notification.service.ts +++ b/backend/src/api/notify/services/limit-alert-notification.service.ts @@ -1,5 +1,5 @@ import { Inject, Injectable, Logger } from '@nestjs/common' -import Bull from 'bull' +import type { Queue } from 'bullmq' import { ClaimedLimitAlert, LimitAlertService, @@ -11,7 +11,7 @@ import { QueueName } from '../../../enum/queue-name.enum' import { IngestionJobPayload } from '../../../queue/queue.types' import { NotifySimpleRequest } from '../schemas/notify-simple-request' import { TenantsService } from '../../admin/tenants/tenants.service' -import { COMPLETED_JOB_RETENTION, FAILED_JOB_RETENTION } from '../../../queue/job-retention' +import { INGESTION_JOB_OPTIONS } from '../../../queue/job-options' export type ProcessLimitAlertNotificationsInput = ProcessLimitAlertUsageInput @@ -65,7 +65,7 @@ export class LimitAlertNotificationService { private readonly notificationService: NotificationService, private readonly tenantsService: TenantsService, @Inject(QueueName.INGESTION) - private readonly ingestionQueue: Bull.Queue, + private readonly ingestionQueue: Queue, ) {} async processAcceptedUsage(input: ProcessLimitAlertNotificationsInput): Promise { @@ -127,12 +127,9 @@ export class LimitAlertNotificationService { requestedAt: new Date().toISOString(), } - await this.ingestionQueue.add(jobPayload, { + await this.ingestionQueue.add(this.ingestionQueue.name, jobPayload, { jobId: notificationRecord.id, - attempts: 3, - backoff: { type: 'exponential', delay: 2000 }, - removeOnComplete: COMPLETED_JOB_RETENTION, - removeOnFail: FAILED_JOB_RETENTION, + ...INGESTION_JOB_OPTIONS, }) await this.notificationService.update(notificationRecord.id, claim.tenantId, { diff --git a/backend/src/common/decorators/queueable.decorator.ts b/backend/src/common/decorators/queueable.decorator.ts index 7b4b9fa13..1b3c74d2e 100644 --- a/backend/src/common/decorators/queueable.decorator.ts +++ b/backend/src/common/decorators/queueable.decorator.ts @@ -4,7 +4,7 @@ import { BadRequestException, UnprocessableEntityException, } from '@nestjs/common' -import Bull from 'bull' +import type { Queue } from 'bullmq' import { NotificationStatus } from '../../enum/notification-status.enum' import { NotificationService } from '../../api/notification/notification.service' import { NotificationPubSubService } from '../../api/notification/notification-pubsub.service' @@ -31,7 +31,7 @@ import type { NotificationDedupService, } from '../../api/notify/services/notification-dedup.service' import { mergeRequestChannel } from '../../queue/merge-batch-builder' -import { COMPLETED_JOB_RETENTION, FAILED_JOB_RETENTION } from '../../queue/job-retention' +import { INGESTION_JOB_OPTIONS } from '../../queue/job-options' interface AcceptedUsageResult extends RecordedUsageResult { channelCode: string @@ -53,7 +53,7 @@ export interface QueueableContext { smsSegmentService?: SmsSegmentService eventNotificationResolver?: EventNotificationResolver notificationDedupService?: NotificationDedupService - queueMap: Map + queueMap: Map } export type EventSendMode = 'full' | 'test' @@ -406,7 +406,7 @@ async function countMergeMessages( */ export async function handleMerge( ctx: QueueableContext, - queue: Bull.Queue, + queue: Queue, queueName: QueueName, tenantId: string, dto: NotifySimpleRequest, @@ -612,20 +612,12 @@ export async function handleMerge( ...(delayedSendTimestamp && { scheduledFor: delayedSendTimestamp }), } - const queueOptions: any = { + await queue.add(queue.name, jobPayload, { jobId: notificationRecord.id, - attempts: 3, - backoff: { type: 'exponential', delay: 2000 }, - removeOnComplete: COMPLETED_JOB_RETENTION, - removeOnFail: FAILED_JOB_RETENTION, - } - - // Add delay if this is a scheduled send - if (delayMs > 0) { - queueOptions.delay = delayMs - } - - await queue.add(jobPayload, queueOptions) + ...INGESTION_JOB_OPTIONS, + // Scheduled send + ...(delayMs > 0 && { delay: delayMs }), + }) logger.log( `Mail merge ingestion job enqueued: ${notificationRecord.id} (tenant=${tenantId}, queue=${queueName})`, @@ -667,7 +659,7 @@ export async function handleMerge( * Validation: Request payloads are validated by NestJS's global ValidationPipe before this decorator runs, * so the payload is guaranteed to match NotifySimpleRequest schema. The decorator focuses solely on queuing logic. * - * The decorator will attempt to queue the notification to the specified Bull queue. If queuing fails (e.g. Redis is down), + * The decorator will attempt to queue the notification to the specified queue. If queuing fails (e.g. Redis is down), * the notification remains in PENDING status and will be retried by a scheduled job. * * @param queueName - Queue to add job to (from QueueName enum) @@ -983,23 +975,12 @@ export function Queueable( ...(delayedSendTimestamp && { scheduledFor: delayedSendTimestamp }), } - const queueOptions: any = { + await queue.add(queue.name, jobPayload, { jobId: notificationRecord.id, - attempts: 3, - backoff: { - type: 'exponential', - delay: 2000, - }, - removeOnComplete: COMPLETED_JOB_RETENTION, - removeOnFail: FAILED_JOB_RETENTION, - } - - // Add delay if this is a scheduled send - if (delayMs > 0) { - queueOptions.delay = delayMs - } - - await queue.add(jobPayload, queueOptions) + ...INGESTION_JOB_OPTIONS, + // Scheduled send + ...(delayMs > 0 && { delay: delayMs }), + }) logger.log( `Job successfully enqueued: ${notificationRecord.id} (tenant=${tenantId}, queue=${queueName})`, diff --git a/backend/src/common/redis/redis-error.util.ts b/backend/src/common/redis/redis-error.util.ts index d3a3d9060..101c16e3e 100644 --- a/backend/src/common/redis/redis-error.util.ts +++ b/backend/src/common/redis/redis-error.util.ts @@ -11,11 +11,11 @@ import { Logger } from '@nestjs/common' * * If nothing listens for a client's `error` event, Node's default handler prints that whole * object to stderr, putting the credential into container logs and anything shipping them. - * Every Redis client and Bull queue must therefore have an error listener attached, and errors + * Every Redis client and BullMQ queue or worker must therefore have an error listener attached, and errors * must be formatted through formatRedisError rather than logged as objects. */ -/** An object that emits Node-style events; Redis clients and Bull queues both qualify. */ +/** An object that emits Node-style events; Redis clients and BullMQ queues and workers all qualify. */ interface ErrorEmitter { on(event: 'error', listener: (error: unknown) => void): unknown } diff --git a/backend/src/common/redis/redis-namespace.ts b/backend/src/common/redis/redis-namespace.ts index 005349fe9..c1fe2c38f 100644 --- a/backend/src/common/redis/redis-namespace.ts +++ b/backend/src/common/redis/redis-namespace.ts @@ -2,7 +2,7 @@ * Prefix for every Redis key this service owns. * * One Redis instance is shared by every deployment in a namespace, while each deployment has - * its own database. An unprefixed key is therefore read and written by all of them: for Bull + * its own database. An unprefixed key is therefore read and written by all of them: for a queue * that means one environment's worker consuming another's job, failing it with "notification * request not found", and stranding the originating row at QUEUED. * @@ -11,7 +11,14 @@ */ export const REDIS_KEY_PREFIX = process.env.REDIS_KEY_PREFIX || process.env.RELEASE_NAME || 'notify' -/** Namespace an application key. Bull namespaces its own keys through its `prefix` option. */ +/** + * Prefix for BullMQ's keys, passed as its `prefix` option. A level below REDIS_KEY_PREFIX because + * the queues ran on Bull before BullMQ, whose keys sit directly under REDIS_KEY_PREFIX with the + * same queue names: BullMQ cannot read that layout, so it must not share it. + */ +export const QUEUE_KEY_PREFIX = `${REDIS_KEY_PREFIX}:mq` + +/** Namespace an application key. BullMQ namespaces its own keys through QUEUE_KEY_PREFIX. */ export function redisKey(key: string): string { return `${REDIS_KEY_PREFIX}:${key}` } diff --git a/backend/src/queue/batch-progress.spec.ts b/backend/src/queue/batch-progress.spec.ts index 8e6dfff37..29fe92b93 100644 --- a/backend/src/queue/batch-progress.spec.ts +++ b/backend/src/queue/batch-progress.spec.ts @@ -1,35 +1,35 @@ import { describe, it, expect, vi } from 'vitest' -import type Bull from 'bull' +import type { Job } from 'bullmq' import { batchProgressReporter, readBatchProgress } from './batch-progress' const flush = () => new Promise((resolve) => setTimeout(resolve, 0)) describe('batchProgressReporter', () => { it('writes the progress to the job', async () => { - const job = { id: '1', progress: vi.fn().mockResolvedValue(undefined) } - batchProgressReporter(job as unknown as Bull.Job)({ sent: 3, failed: 1, total: 10 }) + const job = { id: '1', updateProgress: vi.fn().mockResolvedValue(undefined) } + batchProgressReporter(job as unknown as Job)({ sent: 3, failed: 1, total: 10 }) await flush() - expect(job.progress).toHaveBeenCalledWith({ sent: 3, failed: 1, total: 10 }) + expect(job.updateProgress).toHaveBeenCalledWith({ sent: 3, failed: 1, total: 10 }) }) it('never throws into the send loop, even when the write fails synchronously', async () => { const job = { id: '1', - progress: vi.fn(() => { + updateProgress: vi.fn(() => { throw new Error('not a function in this test double') }), } expect(() => - batchProgressReporter(job as unknown as Bull.Job)({ sent: 0, failed: 0, total: 1 }), + batchProgressReporter(job as unknown as Job)({ sent: 0, failed: 0, total: 1 }), ).not.toThrow() await flush() }) it('swallows a rejected write', async () => { - const job = { id: '1', progress: vi.fn().mockRejectedValue(new Error('redis down')) } - batchProgressReporter(job as unknown as Bull.Job)({ sent: 0, failed: 0, total: 1 }) + const job = { id: '1', updateProgress: vi.fn().mockRejectedValue(new Error('redis down')) } + batchProgressReporter(job as unknown as Job)({ sent: 0, failed: 0, total: 1 }) await flush() - expect(job.progress).toHaveBeenCalled() + expect(job.updateProgress).toHaveBeenCalled() }) }) @@ -42,7 +42,7 @@ describe('readBatchProgress', () => { }) }) - it('returns null for Bull default progress and malformed values', () => { + it('returns null for a default job progress and malformed values', () => { expect(readBatchProgress(0)).toBeNull() expect(readBatchProgress({ sent: '1', failed: 0, total: 5 })).toBeNull() expect(readBatchProgress(null)).toBeNull() diff --git a/backend/src/queue/batch-progress.ts b/backend/src/queue/batch-progress.ts index f2fd183ef..e0fdef8bf 100644 --- a/backend/src/queue/batch-progress.ts +++ b/backend/src/queue/batch-progress.ts @@ -1,7 +1,7 @@ -import type Bull from 'bull' +import type { Job } from 'bullmq' import { Logger } from '@nestjs/common' -/** How far a merge batch has got, stored as the Bull job's progress. */ +/** How far a merge batch has got, stored as the job's progress. */ export interface BatchProgress { sent: number failed: number @@ -13,15 +13,17 @@ export type BatchProgressReporter = (progress: BatchProgress) => void const logger = new Logger('BatchProgress') /** - * Report a merge batch's per-recipient progress on its Bull job, so the admin monitoring page + * Report a merge batch's per-recipient progress on its job, so the admin monitoring page * can show "37 of 100" for a batch that is one job but many emails. Best-effort: the write is * not awaited and a failure is only logged, so progress never slows or fails a send. */ -export function batchProgressReporter(job: Bull.Job): BatchProgressReporter { +export function batchProgressReporter( + job: Pick, +): BatchProgressReporter { return (progress) => { // Deferred through then() so even a synchronous throw lands in catch, not in the send loop. Promise.resolve() - .then(() => job.progress(progress)) + .then(() => job.updateProgress(progress)) .catch((error: Error) => { logger.warn(`Failed to record progress for job ${job.id}: ${error.message}`) }) diff --git a/backend/src/queue/job-options.spec.ts b/backend/src/queue/job-options.spec.ts new file mode 100644 index 000000000..3033dd9b5 --- /dev/null +++ b/backend/src/queue/job-options.spec.ts @@ -0,0 +1,60 @@ +import { describe, it, expect } from 'vitest' +import { + BACKOFF_STRATEGY, + DELIVERY_JOB_OPTIONS, + INGESTION_JOB_OPTIONS, + WEBHOOK_JOB_OPTIONS, + backoffStrategy, + maxAttempts, + retryDelay, +} from './job-options' +import { COMPLETED_JOB_RETENTION, FAILED_JOB_RETENTION } from './job-retention' + +const delayAfter = (attemptsMade: number, options: { backoff: { delay: number } }) => + backoffStrategy(attemptsMade, BACKOFF_STRATEGY, undefined, { opts: options }) + +describe('job options', () => { + // The retry spacing the queues had on Bull: (2^n - 1) x delay after the nth failed attempt. + it('retries ingestion and delivery jobs 2s, then 6s, after a failure', () => { + for (const options of [INGESTION_JOB_OPTIONS, DELIVERY_JOB_OPTIONS]) { + expect(options.attempts).toBe(3) + expect(options.backoff.type).toBe(BACKOFF_STRATEGY) + expect([1, 2].map((attempt) => delayAfter(attempt, options))).toEqual([2000, 6000]) + } + }) + + it('retries webhook jobs 5s, then 15s, after a failure', () => { + expect(WEBHOOK_JOB_OPTIONS.attempts).toBe(3) + expect([1, 2].map((attempt) => delayAfter(attempt, WEBHOOK_JOB_OPTIONS))).toEqual([5000, 15000]) + }) + + it('reads a numeric backoff as its delay', () => { + expect(backoffStrategy(3, BACKOFF_STRATEGY, undefined, { opts: { backoff: 1000 } })).toBe(7000) + }) + + it('refuses a strategy it does not implement rather than retrying immediately', () => { + expect(() => backoffStrategy(1, 'linear')).toThrow('Unknown backoff strategy linear') + }) + + it('keeps ingestion jobs for the dedup window, drops finished delivery jobs, keeps failures', () => { + expect(INGESTION_JOB_OPTIONS.removeOnComplete).toBe(COMPLETED_JOB_RETENTION) + expect(DELIVERY_JOB_OPTIONS.removeOnComplete).toBe(true) + expect(WEBHOOK_JOB_OPTIONS.removeOnComplete).toBe(COMPLETED_JOB_RETENTION) + for (const options of [INGESTION_JOB_OPTIONS, DELIVERY_JOB_OPTIONS, WEBHOOK_JOB_OPTIONS]) { + expect(options.removeOnFail).toBe(FAILED_JOB_RETENTION) + } + }) + + it('reports the retry delay the queue will use, for logging when a retry is due', () => { + expect([1, 2].map((attempt) => retryDelay({ opts: WEBHOOK_JOB_OPTIONS }, attempt))).toEqual([ + 5000, 15000, + ]) + expect(retryDelay({ opts: { backoff: 1000 } }, 2)).toBe(1000) + expect(retryDelay({ opts: {} }, 1)).toBe(0) + }) + + it('counts a job queued without attempts as attempted once, as BullMQ runs it', () => { + expect(maxAttempts({ opts: {} })).toBe(1) + expect(maxAttempts({ opts: DELIVERY_JOB_OPTIONS })).toBe(3) + }) +}) diff --git a/backend/src/queue/job-options.ts b/backend/src/queue/job-options.ts new file mode 100644 index 000000000..e94bdd096 --- /dev/null +++ b/backend/src/queue/job-options.ts @@ -0,0 +1,72 @@ +import type { BackoffOptions, JobsOptions } from 'bullmq' +import { COMPLETED_JOB_RETENTION, FAILED_JOB_RETENTION } from './job-retention' + +/** + * Backoff for every job here: retry n waits (2^n - 1) x delay, so 2s then 6s at a 2s base. The + * queues ran on Bull, whose exponential backoff this is; BullMQ's built-in `exponential` is + * 2^(n-1) x delay (2s, 4s), so jobs name this strategy instead and every Worker registers it. + */ +export const BACKOFF_STRATEGY = 'notify-exponential' + +/** BullMQ's `settings.backoffStrategy`. `attemptsMade` counts the attempt that just failed. */ +export function backoffStrategy( + attemptsMade: number, + type?: string, + _error?: Error, + job?: { opts: { backoff?: number | BackoffOptions } }, +): number { + if (type !== BACKOFF_STRATEGY) throw new Error(`Unknown backoff strategy ${type}`) + const backoff = job?.opts.backoff + const delay = (typeof backoff === 'object' ? backoff.delay : backoff) ?? 0 + return Math.round((2 ** attemptsMade - 1) * delay) +} + +/** Attempts a job gets in all. BullMQ runs a job queued without `attempts` once. */ +export function maxAttempts(job: { opts: { attempts?: number } }): number { + return job.opts.attempts ?? 1 +} + +/** + * How long the queue waits before retrying a job whose attempt number `attemptsMade` just failed: + * BACKOFF_STRATEGY for the jobs queued here, and a fixed delay for a plain number. + */ +export function retryDelay( + job: { opts: { backoff?: number | BackoffOptions } }, + attemptsMade: number, +): number { + const { backoff } = job.opts + if (backoff === undefined) return 0 + if (typeof backoff === 'number') return backoff + if (backoff.type === BACKOFF_STRATEGY) { + return backoffStrategy(attemptsMade, backoff.type, undefined, job) + } + return backoff.delay ?? 0 +} + +/** Three attempts, 2s then 6s apart. */ +export const RETRY_POLICY = { + attempts: 3, + backoff: { type: BACKOFF_STRATEGY, delay: 2000 }, +} satisfies JobsOptions + +/** An ingestion job: kept an hour once done, so a re-add of the same id is deduplicated. */ +export const INGESTION_JOB_OPTIONS = { + ...RETRY_POLICY, + removeOnComplete: COMPLETED_JOB_RETENTION, + removeOnFail: FAILED_JOB_RETENTION, +} satisfies JobsOptions + +/** A delivery job: removed once done. Postgres records what it sent. */ +export const DELIVERY_JOB_OPTIONS = { + ...RETRY_POLICY, + removeOnComplete: true, + removeOnFail: FAILED_JOB_RETENTION, +} satisfies JobsOptions + +/** A webhook job: three attempts, 5s then 15s apart. */ +export const WEBHOOK_JOB_OPTIONS = { + attempts: 3, + backoff: { type: BACKOFF_STRATEGY, delay: 5000 }, + removeOnComplete: COMPLETED_JOB_RETENTION, + removeOnFail: FAILED_JOB_RETENTION, +} satisfies JobsOptions diff --git a/backend/src/queue/job-retention.ts b/backend/src/queue/job-retention.ts index 4dc37f5c0..900ceb2c7 100644 --- a/backend/src/queue/job-retention.ts +++ b/backend/src/queue/job-retention.ts @@ -1,12 +1,12 @@ /** - * Retention for Bull jobs. + * Retention for queue jobs. * * Redis is capped at 768mb with `maxmemory-policy noeviction` (charts/redis/values.yml), so a * queue that keeps every job it has ever run eventually makes Redis reject writes - which * stops enqueue altogether, not just this queue. Both sets are bounded by age and count. * * The completed window is not an audit log; notification_request and its history table are. - * It exists so that re-adding the same jobId within the window is deduplicated by Bull - a + * It exists so that re-adding the same jobId within the window is deduplicated by BullMQ - a * request queued twice in quick succession (the original path, then a retry of the same add) runs * once. An hour is far longer than that race while still bounding what Redis holds. */ diff --git a/backend/src/queue/queue-metrics.spec.ts b/backend/src/queue/queue-metrics.spec.ts index 80e04343d..7383f06a1 100644 --- a/backend/src/queue/queue-metrics.spec.ts +++ b/backend/src/queue/queue-metrics.spec.ts @@ -1,26 +1,27 @@ import { describe, it, expect, vi } from 'vitest' -import type Bull from 'bull' +import type { Queue } from 'bullmq' +import type Redis from 'ioredis' import { MONITORING_WINDOW_MINUTES, addedCounterKey, - alignBullMetrics, + alignQueueMetrics, countAddedJobs, minuteOf, } from './queue-metrics' const MINUTE = 60_000 -describe('alignBullMetrics', () => { +describe('alignQueueMetrics', () => { const now = 1_000 * MINUTE + 30_000 it('returns a zeroed window when the queue has never finished a job', () => { - const series = alignBullMetrics({ meta: { count: 0, prevTS: 0, prevCount: 0 }, data: [] }, now) + const series = alignQueueMetrics({ meta: { count: 0, prevTS: 0, prevCount: 0 }, data: [] }, now) expect(series).toHaveLength(MONITORING_WINDOW_MINUTES) expect(series.every((value) => value === 0)).toBe(true) }) it('puts unflushed jobs in the current minute and flushed points behind it', () => { - const series = alignBullMetrics( + const series = alignQueueMetrics( { meta: { count: 12, prevTS: now - 5_000, prevCount: 9 }, data: ['4', '7'] }, now, ) @@ -30,8 +31,8 @@ describe('alignBullMetrics', () => { }) it('anchors points to the last flush, not to now, when the queue has gone quiet', () => { - // Last job finished three minutes ago; Bull has not written a point since. - const series = alignBullMetrics( + // Last job finished three minutes ago; BullMQ has not written a point since. + const series = alignQueueMetrics( { meta: { count: 10, prevTS: now - 3 * MINUTE, prevCount: 8 }, data: ['5'] }, now, ) @@ -51,16 +52,16 @@ describe('countAddedJobs', () => { const queue = { name: 'email-delivery', add: vi.fn().mockResolvedValue(job), - client: { multi: vi.fn(() => multi) }, } - return { queue, multi, job, exec } + const redis = { multi: vi.fn(() => multi) } + return { queue, redis, multi, job, exec } } it('increments the current minute counter after a job is added', async () => { - const { queue, multi, job } = fakeQueue() - countAddedJobs(queue as unknown as Bull.Queue) + const { queue, redis, multi, job } = fakeQueue() + countAddedJobs(queue as unknown as Queue, redis as unknown as Redis) - await expect(queue.add({ notifyId: 'n1' })).resolves.toBe(job) + await expect(queue.add('email-delivery', { notifyId: 'n1' })).resolves.toBe(job) const key = addedCounterKey('email-delivery', minuteOf(Date.now())) expect(multi.incr).toHaveBeenCalledWith(key) @@ -68,19 +69,19 @@ describe('countAddedJobs', () => { }) it('does not fail the add when counting fails', async () => { - const { queue, job, exec } = fakeQueue() + const { queue, redis, job, exec } = fakeQueue() exec.mockRejectedValue(new Error('redis down')) - countAddedJobs(queue as unknown as Bull.Queue) + countAddedJobs(queue as unknown as Queue, redis as unknown as Redis) - await expect(queue.add({})).resolves.toBe(job) + await expect(queue.add('email-delivery', {})).resolves.toBe(job) }) it('does not count a job whose add failed', async () => { - const { queue, multi } = fakeQueue() + const { queue, redis, multi } = fakeQueue() queue.add.mockRejectedValue(new Error('OOM command not allowed')) - countAddedJobs(queue as unknown as Bull.Queue) + countAddedJobs(queue as unknown as Queue, redis as unknown as Redis) - await expect(queue.add({})).rejects.toThrow('OOM') + await expect(queue.add('email-delivery', {})).rejects.toThrow('OOM') expect(multi.incr).not.toHaveBeenCalled() }) }) diff --git a/backend/src/queue/queue-metrics.ts b/backend/src/queue/queue-metrics.ts index ca389bc0a..b66a9734e 100644 --- a/backend/src/queue/queue-metrics.ts +++ b/backend/src/queue/queue-metrics.ts @@ -1,13 +1,14 @@ -import type Bull from 'bull' +import type { Queue } from 'bullmq' +import type Redis from 'ioredis' import { Logger } from '@nestjs/common' -import { REDIS_KEY_PREFIX } from '../common/redis/redis-namespace' +import { QUEUE_KEY_PREFIX } from '../common/redis/redis-namespace' const logger = new Logger('QueueMetrics') const MINUTE_MS = 60_000 /** - * Per-minute "added" counters live this long. Bull records completed/failed itself (see + * Per-minute "added" counters live this long. BullMQ records completed/failed itself (see * QUEUE_METRICS_MAX_DATA_POINTS) but has no equivalent for adds, so the fill rate is ours. */ const ADDED_COUNTER_TTL_SECONDS = 2 * 60 * 60 @@ -19,13 +20,9 @@ export function minuteOf(timestampMs: number): number { return Math.floor(timestampMs / MINUTE_MS) } -/** The key Bull itself uses for `suffix` on this queue, given our `prefix` option. */ -export function bullKey(queueName: string, suffix: string): string { - return `${REDIS_KEY_PREFIX}:${queueName}:${suffix}` -} - +/** Kept beside the queue's own keys, so it is namespaced and cleared with them. */ export function addedCounterKey(queueName: string, minute: number): string { - return bullKey(queueName, `metrics:added:${minute}`) + return `${QUEUE_KEY_PREFIX}:${queueName}:metrics:added:${minute}` } /** @@ -33,15 +30,15 @@ export function addedCounterKey(queueName: string, minute: number): string { * * Jobs are added from a dozen call sites, so this wraps `add` once where the queue is built * rather than at each of them. Counting is best-effort: a Redis error is logged and never - * fails the add, which has already succeeded by then. A re-add Bull deduplicates by jobId is + * fails the add, which has already succeeded by then. A re-add BullMQ deduplicates by jobId is * still counted, which slightly overstates the fill rate during a pending-sweep retry. */ -export function countAddedJobs(queue: Bull.Queue): void { - const add = queue.add.bind(queue) as (...args: unknown[]) => Promise - const counted = async (...args: unknown[]): Promise => { +export function countAddedJobs(queue: Queue, redis: Redis): void { + const add = queue.add.bind(queue) + queue.add = (async (...args: Parameters) => { const job = await add(...args) const key = addedCounterKey(queue.name, minuteOf(Date.now())) - queue.client + redis .multi() .incr(key) .expire(key, ADDED_COUNTER_TTL_SECONDS) @@ -50,26 +47,25 @@ export function countAddedJobs(queue: Bull.Queue): void { logger.warn(`Failed to count added job on ${queue.name}: ${error.message}`) }) return job - } - queue.add = counted as Bull.Queue['add'] + }) as Queue['add'] } -/** Bull's `getMetrics()` result, as returned at runtime (list entries are strings). */ -export interface BullMetrics { +/** BullMQ's `getMetrics()` result. */ +export interface QueueMetrics { meta: { count: number; prevTS: number; prevCount: number } data: Array } /** - * Re-align Bull's completed/failed data points onto wall-clock minutes, oldest first. + * Re-align BullMQ's completed/failed data points onto wall-clock minutes, oldest first. * - * Bull only writes a data point when a job finishes in a later minute than the previous one, + * BullMQ only writes a data point when a job finishes in a later minute than the previous one, * so its list is anchored to `prevTS`, not to now: `data[i]` is minute(prevTS) - 1 - i, and * jobs finished in minute(prevTS) are still held as `count - prevCount`. Reading the list as * if index 0 were the current minute shifts every point by however long the queue has been * quiet. */ -export function alignBullMetrics(metrics: BullMetrics, nowMs: number): number[] { +export function alignQueueMetrics(metrics: QueueMetrics, nowMs: number): number[] { const currentMinute = minuteOf(nowMs) const flushedMinute = minuteOf(metrics.meta.prevTS) const pending = Math.max(0, metrics.meta.count - metrics.meta.prevCount) diff --git a/backend/src/queue/queue.module.shutdown.spec.ts b/backend/src/queue/queue.module.shutdown.spec.ts index a68deb86a..7d9037209 100644 --- a/backend/src/queue/queue.module.shutdown.spec.ts +++ b/backend/src/queue/queue.module.shutdown.spec.ts @@ -1,23 +1,25 @@ import { describe, it, expect, vi, afterEach } from 'vitest' -import type Bull from 'bull' import { QueueModule } from './queue.module' +import { QueueName } from '../enum/queue-name.enum' import type { WorkerHeartbeat } from './worker-heartbeat' type Heartbeat = Pick function moduleWith(close: () => Promise) { - const queue = { - pause: vi.fn().mockResolvedValue(undefined), - close: vi.fn(close), - } as unknown as Bull.Queue + const worker = { pause: vi.fn().mockResolvedValue(undefined), close: vi.fn(close) } const heartbeat: Heartbeat = { markDraining: vi.fn(), remove: vi.fn().mockResolvedValue(undefined), } - const module = new QueueModule(queue, queue, queue, null as unknown as Bull.Queue) + const module = new QueueModule() // Set by onModuleInit when workers start; that path needs the whole worker graph. - ;(module as unknown as { heartbeat: Heartbeat }).heartbeat = heartbeat - return { module, queue, heartbeat } + const internals = module as unknown as { heartbeat: Heartbeat; workers: Map } + internals.heartbeat = heartbeat + // Three workers running (webhook absent). + for (const name of [QueueName.INGESTION, QueueName.EMAIL_DELIVERY, QueueName.SMS_DELIVERY]) { + internals.workers.set(name, worker) + } + return { module, worker, heartbeat } } describe('QueueModule shutdown', () => { @@ -33,14 +35,13 @@ describe('QueueModule shutdown', () => { }) it('waits for in-flight jobs, then drops the pod off the live list', async () => { - const { module, queue, heartbeat } = moduleWith(() => Promise.resolve()) + const { module, worker, heartbeat } = moduleWith(() => Promise.resolve()) await module.beforeApplicationShutdown() - // Three queues present (webhook absent): each stops taking work, then closes once its - // active jobs finish. - expect(queue.pause).toHaveBeenCalledTimes(3) - expect(queue.close).toHaveBeenCalledTimes(3) + // Each worker stops taking work, then closes once its active jobs finish. + expect(worker.pause).toHaveBeenCalledTimes(3) + expect(worker.close).toHaveBeenCalledTimes(3) expect(heartbeat.remove).toHaveBeenCalled() }) @@ -56,7 +57,7 @@ describe('QueueModule shutdown', () => { expect(heartbeat.remove).not.toHaveBeenCalled() }) - it('does not throw out of shutdown when closing a queue fails', async () => { + it('does not throw out of shutdown when closing a worker fails', async () => { const { module, heartbeat } = moduleWith(() => Promise.reject(new Error('connection lost'))) await expect(module.beforeApplicationShutdown()).resolves.toBeUndefined() diff --git a/backend/src/queue/queue.module.spec.ts b/backend/src/queue/queue.module.spec.ts index b73389558..82341a34d 100644 --- a/backend/src/queue/queue.module.spec.ts +++ b/backend/src/queue/queue.module.spec.ts @@ -3,10 +3,16 @@ import { ConfigModule, ConfigService } from '@nestjs/config' import { QueueModule } from './queue.module' import { ProviderToken } from '../enum/provider-token.enum' import { QueueName } from '../enum/queue-name.enum' -import Bull from 'bull' +import { Queue, type Job } from 'bullmq' import Redis from 'ioredis' import { Logger } from '@nestjs/common' -import { vi } from 'vitest' +import { vi, type Mock } from 'vitest' +import { createWorker } from './redis-connection' + +vi.mock('./redis-connection', async (importOriginal) => ({ + ...(await importOriginal()), + createWorker: vi.fn(), +})) describe('QueueModule', () => { describe.skip('Infrastructure Tests (requires Redis and Database)', () => { @@ -50,19 +56,19 @@ describe('QueueModule', () => { it('should provide ingestion queue', () => { const ingestionQueue = module.get(QueueName.INGESTION) expect(ingestionQueue).toBeDefined() - expect(ingestionQueue).toBeInstanceOf(Bull) + expect(ingestionQueue).toBeInstanceOf(Queue) }) it('should provide email delivery queue', () => { const emailQueue = module.get(QueueName.EMAIL_DELIVERY) expect(emailQueue).toBeDefined() - expect(emailQueue).toBeInstanceOf(Bull) + expect(emailQueue).toBeInstanceOf(Queue) }) it('should provide SMS delivery queue', () => { const smsQueue = module.get(QueueName.SMS_DELIVERY) expect(smsQueue).toBeDefined() - expect(smsQueue).toBeInstanceOf(Bull) + expect(smsQueue).toBeInstanceOf(Queue) }) it('should export all providers', async () => { @@ -89,7 +95,7 @@ describe('QueueModule', () => { describe('beforeApplicationShutdown', () => { const calls: string[] = [] - function mockQueue(name: string, close?: () => Promise): Bull.Queue { + function mockCloser(name: string, close?: () => Promise) { return { pause: vi.fn(async () => { calls.push(`pause:${name}`) @@ -100,7 +106,15 @@ describe('QueueModule', () => { calls.push(`close:${name}`) }), ), - } as unknown as Bull.Queue + } + } + + /** A module whose onModuleInit has started these workers. */ + function moduleWith(workers: Partial>, queues: unknown[] = []) { + const module = new QueueModule(...(queues as [Queue, Queue, Queue, Queue])) + const started = (module as unknown as { workers: Map }).workers + for (const [name, worker] of Object.entries(workers)) started.set(name as QueueName, worker) + return module } beforeEach(() => { @@ -114,14 +128,21 @@ describe('QueueModule', () => { vi.restoreAllMocks() }) - it('stops every queue taking work, then drains ingestion before delivery before webhooks', async () => { - const queues = ['ingestion', 'email', 'sms', 'webhook'].map((name) => mockQueue(name)) - const module = new QueueModule(queues[0], queues[1], queues[2], queues[3]) + it('stops every worker taking work, then drains ingestion before delivery before webhooks', async () => { + const workers = { + [QueueName.INGESTION]: mockCloser('ingestion'), + [QueueName.EMAIL_DELIVERY]: mockCloser('email'), + [QueueName.SMS_DELIVERY]: mockCloser('sms'), + [QueueName.WEBHOOK_DELIVERY]: mockCloser('webhook'), + } + const queue = mockCloser('queue') + const module = moduleWith(workers, [queue, queue, queue, queue]) await module.beforeApplicationShutdown() - for (const queue of queues) { - expect(queue.pause).toHaveBeenCalledWith(true, true) + for (const worker of Object.values(workers)) { + // Local pause, not waiting for active jobs: those are waited for by close(). + expect(worker.pause).toHaveBeenCalledWith(true) } expect(calls.slice(0, 4).every((call) => call.startsWith('pause:'))).toBe(true) expect(calls.slice(4)).toEqual([ @@ -129,13 +150,19 @@ describe('QueueModule', () => { 'close:email', 'close:sms', 'close:webhook', + 'close:queue', + 'close:queue', + 'close:queue', + 'close:queue', ]) }) it('gives up at the deadline rather than holding the pod past its grace period', async () => { process.env.QUEUE_SHUTDOWN_DRAIN_MS = '20' - const stuck = mockQueue('ingestion', () => new Promise(() => {})) - const module = new QueueModule(stuck, mockQueue('email'), mockQueue('sms'), undefined) + const module = moduleWith({ + [QueueName.INGESTION]: mockCloser('ingestion', () => new Promise(() => {})), + [QueueName.EMAIL_DELIVERY]: mockCloser('email'), + }) await module.beforeApplicationShutdown() @@ -151,4 +178,127 @@ describe('QueueModule', () => { await expect(module.beforeApplicationShutdown()).resolves.toBeUndefined() }) }) + + describe('onModuleInit', () => { + const redis = { host: 'localhost', port: 6379, db: 0 } + + function mockQueue(name: string) { + return { name } as unknown as Queue + } + + function mockProcessor(queueName: QueueName, configKey: string, defaultConcurrency: number) { + return { + queueName, + concurrency: { configKey, default: defaultConcurrency }, + process: vi.fn().mockResolvedValue('done'), + onCompleted: vi.fn(), + onFailed: vi.fn(), + } + } + + beforeEach(() => { + vi.mocked(createWorker).mockReset() + vi.mocked(createWorker).mockImplementation( + (name) => ({ name, on: vi.fn(), pause: vi.fn(), close: vi.fn() }) as never, + ) + vi.spyOn(Logger.prototype, 'log').mockImplementation(() => {}) + vi.spyOn(Logger.prototype, 'debug').mockImplementation(() => {}) + }) + + afterEach(() => vi.restoreAllMocks()) + + function config(values: Record) { + return { get: vi.fn((key: string) => values[key]) } as unknown as ConfigService + } + + it('runs each worker on its queue at the configured concurrency, or its default', async () => { + const queues = Object.values(QueueName).map(mockQueue) + const processors = [ + mockProcessor(QueueName.INGESTION, 'ingestionWorkerConcurrency', 1), + mockProcessor(QueueName.EMAIL_DELIVERY, 'emailDeliveryWorkerConcurrency', 2), + mockProcessor(QueueName.SMS_DELIVERY, 'smsDeliveryWorkerConcurrency', 2), + mockProcessor(QueueName.WEBHOOK_DELIVERY, 'webhookDeliveryWorkerConcurrency', 2), + ] + const module = new QueueModule( + queues[0], + queues[1], + queues[2], + queues[3], + config({ redis, 'queue.emailDeliveryWorkerConcurrency': 20 }), + ...(processors as never[]), + ) + + module.onModuleInit() + + const created = vi.mocked(createWorker).mock.calls + expect(created.map(([name, , , concurrency]) => [name, concurrency])).toEqual([ + [QueueName.INGESTION, 1], + [QueueName.EMAIL_DELIVERY, 20], + [QueueName.SMS_DELIVERY, 2], + [QueueName.WEBHOOK_DELIVERY, 2], + ]) + + // The worker runs the processor's own handler, and so are the lifecycle listeners. + const job = { id: 'job-1' } as Job + await expect(created[1][1](job)).resolves.toBe('done') + expect(processors[1].process).toHaveBeenCalledWith(job) + + const worker = vi.mocked(createWorker).mock.results[1].value as { on: Mock } + const listeners = Object.fromEntries(worker.on.mock.calls) as Record< + string, + (...args: unknown[]) => void + > + listeners.completed(job) + listeners.failed(job, new Error('boom')) + // A stalled job removed on failure arrives without its job; there is nothing to settle. + listeners.failed(undefined, new Error('stalled')) + expect(processors[1].onCompleted).toHaveBeenCalledWith(job) + await vi.waitFor(() => expect(processors[1].onFailed).toHaveBeenCalledTimes(1)) + expect(processors[1].onFailed).toHaveBeenCalledWith( + job, + expect.objectContaining({ message: 'boom' }), + ) + }) + + it('logs a failed-job handler that rejects instead of leaving the rejection unhandled', async () => { + const error = vi.spyOn(Logger.prototype, 'error').mockImplementation(() => {}) + const processor = mockProcessor(QueueName.EMAIL_DELIVERY, 'emailDeliveryWorkerConcurrency', 2) + processor.onFailed.mockRejectedValue(new Error('database down')) + const queues = Object.values(QueueName).map(mockQueue) + const module = new QueueModule( + queues[0], + queues[1], + queues[2], + queues[3], + config({ redis }), + undefined, + processor as never, + ) + + module.onModuleInit() + const worker = vi.mocked(createWorker).mock.results[0].value as { on: Mock } + const failed = worker.on.mock.calls.find(([event]) => event === 'failed')![1] + failed({ id: 'job-9' }, new Error('send failed')) + + await vi.waitFor(() => + expect(error).toHaveBeenCalledWith(expect.stringContaining('job job-9: database down')), + ) + }) + + it('starts no worker when Redis is not configured', () => { + const processor = mockProcessor(QueueName.INGESTION, 'ingestionWorkerConcurrency', 1) + const module = new QueueModule( + undefined, + undefined, + undefined, + undefined, + config({}), + processor as never, + ) + + module.onModuleInit() + + expect(createWorker).not.toHaveBeenCalled() + }) + }) }) diff --git a/backend/src/queue/queue.module.ts b/backend/src/queue/queue.module.ts index 0e14e2ebf..16f69bfd4 100644 --- a/backend/src/queue/queue.module.ts +++ b/backend/src/queue/queue.module.ts @@ -6,21 +6,24 @@ import { Inject, Logger, Optional, + Provider, forwardRef, } from '@nestjs/common' import { ConfigService } from '@nestjs/config' -import { InjectRepository, TypeOrmModule } from '@nestjs/typeorm' -import { Repository } from 'typeorm' -import type Bull from 'bull' +import { TypeOrmModule } from '@nestjs/typeorm' +import type { Job, Queue, Worker } from 'bullmq' import type Redis from 'ioredis' import { QueueName } from '../enum/queue-name.enum' -import { createQueue, createRedisClient } from './redis-connection' +import { createQueue, createRedisClient, createWorker, type RedisConfig } from './redis-connection' import { WorkerHeartbeat } from './worker-heartbeat' import { ProviderToken } from '../enum/provider-token.enum' import { IngestionWorker } from './workers/ingestion.worker' import { EmailDeliveryWorker } from './workers/email-delivery.worker' import { SmsDeliveryWorker } from './workers/sms-delivery.worker' import { WebhookDeliveryWorker } from './workers/webhook-delivery.worker' +import { MergeBatchRunner } from './workers/merge-batch-runner' +import { DeliveryFailureHandler } from './workers/delivery-failure-handler' +import type { QueueProcessor } from './workers/queue-processor' import { DeliveryReconcilerService } from './services/delivery-reconciler.service' import { WebhookTriggerService } from './services/webhook-trigger.service' import { NotificationRequest } from '../api/notification/entities/notification-request.entity' @@ -28,29 +31,29 @@ import { NotificationRequestDetail } from '../api/notification/entities/notifica import { NotificationService } from '../api/notification/notification.service' import { NotificationRequestDetailService } from '../api/notification/notification-request-detail.service' import { NotificationPubSubService } from '../api/notification/notification-pubsub.service' -import { TemplatesRepository } from '../api/templates/templates.repository' -import { TenantSettingsService } from '../api/tenant-settings/tenant-settings.service' -import { TemplatesService } from '../api/templates/templates.service' -import { InlineRenderingService } from '../services/rendering/inline-rendering.service' -import { EMAIL_ADAPTER, IEmailTransport, SMS_ADAPTER, ISmsTransport } from '../adapters' import { TenantsModule } from '../api/admin/tenants/tenants.module' import { TemplatesModule } from '../api/templates/templates.module' import { TenantSettingsModule } from '../api/tenant-settings/tenant-settings.module' import { FeatureFlagModule } from '../api/feature-flag/feature-flag.module' import { EventsModule } from '../api/events/events.module' -import { EventNotificationResolver } from '../api/events/event-notification.resolver' import { NotifyModule } from '../api/notify/notify.module' import { WebhookModule } from '../api/webhook/webhook.module' -import { WebhookService } from '../api/webhook/webhook.service' -import { WebhookDeliveryLogRepository } from '../api/webhook/webhook-delivery-log.repository' -import { AttachmentResolverService } from '../api/notify/services/attachment-resolver.service' -import { ClamavService } from '../services/clamav.service' import { ClamavModule } from '../services/clamav.module' import { AttachmentModule } from '../api/attachment/attachment.module' -import { AttachmentService } from '../api/attachment/attachment.service' -import { StructuredLoggerService } from '../common/logger' import { PhoneNumberService } from '../api/notify/services/phone-number.service' +/** A queue's provider; null when Redis is not configured (in tests), so nothing connects. */ +function queueProvider(name: QueueName): Provider { + return { + provide: name, + useFactory: (configService: ConfigService, redis: Redis | null) => { + const redisConfig = configService.get('redis') + return redisConfig ? createQueue(name, redisConfig, redis) : null + }, + inject: [ConfigService, ProviderToken.REDIS_CLIENT], + } +} + /** * Queue Module * @@ -84,6 +87,12 @@ import { PhoneNumberService } from '../api/notify/services/phone-number.service' NotificationRequestDetailService, NotificationPubSubService, PhoneNumberService, + MergeBatchRunner, + DeliveryFailureHandler, + IngestionWorker, + EmailDeliveryWorker, + SmsDeliveryWorker, + WebhookDeliveryWorker, // Provides a direct Redis connection for advanced use cases // Inject with: @Inject(ProviderToken.REDIS_CLIENT) redisClient: Redis { @@ -98,71 +107,10 @@ import { PhoneNumberService } from '../api/notify/services/phone-number.service' inject: [ConfigService], }, - // Each queue is a separate provider with its own Redis connection. - // BullMQ automatically handles job persistence, retries, and scheduling. - // Bull creates its own Redis connections to avoid conflicts with enableReadyCheck. - // - // Injection pattern: - // @Inject(QueueName.INGESTION) ingestionQueue: Bull.Queue - // @Inject(QueueName.EMAIL_DELIVERY) emailQueue: Bull.Queue - // @Inject(QueueName.SMS_DELIVERY) smsQueue: Bull.Queue - // @Inject(QueueName.WEBHOOK_DELIVERY) webhookQueue: Bull.Queue - { - provide: QueueName.INGESTION, - useFactory: (configService: ConfigService) => { - const redisConfig = configService.get('redis') - - // No redis config (e.g. in tests): skip queue initialization - if (!redisConfig) { - return null - } - - return createQueue(QueueName.INGESTION, redisConfig) - }, - inject: [ConfigService], - }, - { - provide: QueueName.EMAIL_DELIVERY, - useFactory: (configService: ConfigService) => { - const redisConfig = configService.get('redis') - - // No redis config (e.g. in tests): skip queue initialization - if (!redisConfig) { - return null - } - - return createQueue(QueueName.EMAIL_DELIVERY, redisConfig) - }, - inject: [ConfigService], - }, - { - provide: QueueName.SMS_DELIVERY, - useFactory: (configService: ConfigService) => { - const redisConfig = configService.get('redis') - - // No redis config (e.g. in tests): skip queue initialization - if (!redisConfig) { - return null - } - - return createQueue(QueueName.SMS_DELIVERY, redisConfig) - }, - inject: [ConfigService], - }, - { - provide: QueueName.WEBHOOK_DELIVERY, - useFactory: (configService: ConfigService) => { - const redisConfig = configService.get('redis') - - // No redis config (e.g. in tests): skip queue initialization - if (!redisConfig) { - return null - } - - return createQueue(QueueName.WEBHOOK_DELIVERY, redisConfig) - }, - inject: [ConfigService], - }, + // One BullMQ Queue - the producer side - per queue, each with its own Redis connection. + // Inject with: @Inject(QueueName.EMAIL_DELIVERY) emailQueue: Queue + // Workers - the consumer side - are created in onModuleInit. + ...Object.values(QueueName).map(queueProvider), WebhookTriggerService, ], exports: [ @@ -176,31 +124,19 @@ import { PhoneNumberService } from '../api/notify/services/phone-number.service' export class QueueModule implements OnModuleInit, OnModuleDestroy, BeforeApplicationShutdown { private readonly logger = new Logger(QueueModule.name) private heartbeat?: WorkerHeartbeat + /** This pod's workers, keyed by queue; created in onModuleInit. */ + private readonly workers = new Map() constructor( - @Inject(QueueName.INGESTION) private readonly ingestionQueue?: Bull.Queue, - @Inject(QueueName.EMAIL_DELIVERY) private readonly emailQueue?: Bull.Queue, - @Inject(QueueName.SMS_DELIVERY) private readonly smsQueue?: Bull.Queue, - @Inject(QueueName.WEBHOOK_DELIVERY) private readonly webhookQueue?: Bull.Queue, - @InjectRepository(NotificationRequest) - private readonly notificationRepository?: Repository, + @Inject(QueueName.INGESTION) private readonly ingestionQueue?: Queue | null, + @Inject(QueueName.EMAIL_DELIVERY) private readonly emailQueue?: Queue | null, + @Inject(QueueName.SMS_DELIVERY) private readonly smsQueue?: Queue | null, + @Inject(QueueName.WEBHOOK_DELIVERY) private readonly webhookQueue?: Queue | null, private readonly configService?: ConfigService, - private readonly notificationService?: NotificationService, - private readonly templatesRepository?: TemplatesRepository, - private readonly templatesService?: TemplatesService, - private readonly inlineRenderingService?: InlineRenderingService, - private readonly attachmentResolverService?: AttachmentResolverService, - private readonly attachmentService?: AttachmentService, - @Inject(EMAIL_ADAPTER) private readonly emailAdapter?: IEmailTransport, - @Inject(SMS_ADAPTER) private readonly smsAdapter?: ISmsTransport, - private readonly notificationRequestDetailService?: NotificationRequestDetailService, - private readonly tenantSettingsService?: TenantSettingsService, - private readonly clamavService?: ClamavService, - private readonly phoneNumberService?: PhoneNumberService, - private readonly webhookService?: WebhookService, - private readonly webhookDeliveryLogRepository?: WebhookDeliveryLogRepository, - @Optional() private readonly structuredLogger?: StructuredLoggerService, - @Optional() private readonly eventNotificationResolver?: EventNotificationResolver, + private readonly ingestionWorker?: IngestionWorker, + private readonly emailDeliveryWorker?: EmailDeliveryWorker, + private readonly smsDeliveryWorker?: SmsDeliveryWorker, + private readonly webhookDeliveryWorker?: WebhookDeliveryWorker, @Optional() @Inject(ProviderToken.REDIS_CLIENT) private readonly redisClient?: Redis | null, @@ -214,33 +150,41 @@ export class QueueModule implements OnModuleInit, OnModuleDestroy, BeforeApplica /** * Let in-flight jobs finish before the pod exits. Without this a deploy or an HPA scale-down - * kills a merge batch part-way through and Bull re-runs it on another pod once its lock lapses. + * kills a merge batch part-way through and BullMQ re-runs it on another pod once its lock + * lapses. * - * Every queue stops taking new work on this pod first; then ingestion drains ahead of the - * delivery queues it feeds, and delivery ahead of the webhooks it triggers. Runs before - * onApplicationShutdown, where TypeORM closes its connection, so a finishing job can still - * record what it sent. The wait is capped below the pod's termination grace period - * (QUEUE_SHUTDOWN_DRAIN_MS, set by the chart); a batch still running then is re-run elsewhere, - * and the delivery workers skip recipients already sent. + * Every worker stops taking new work on this pod first; then ingestion drains ahead of the + * delivery queues it feeds, and delivery ahead of the webhooks it triggers. The queues' own + * connections close last. Runs before onApplicationShutdown, where TypeORM closes its + * connection, so a finishing job can still record what it sent. The wait is capped below the + * pod's termination grace period (QUEUE_SHUTDOWN_DRAIN_MS, set by the chart); a batch still + * running then is re-run elsewhere, and the delivery workers skip recipients already sent. */ async beforeApplicationShutdown(): Promise { const stages = [ - [this.ingestionQueue], - [this.emailQueue, this.smsQueue], - [this.webhookQueue], - ].map((stage) => stage.filter((queue): queue is Bull.Queue => !!queue)) - const queues = stages.flat() - if (queues.length === 0) return + [QueueName.INGESTION], + [QueueName.EMAIL_DELIVERY, QueueName.SMS_DELIVERY], + [QueueName.WEBHOOK_DELIVERY], + ].map((stage) => + stage.map((name) => this.workers.get(name)).filter((worker): worker is Worker => !!worker), + ) + const workers = stages.flat() + const queues = [this.ingestionQueue, this.emailQueue, this.smsQueue, this.webhookQueue].filter( + (queue): queue is Queue => !!queue, + ) + if (workers.length === 0 && queues.length === 0) return const drainMs = Number.parseInt(process.env.QUEUE_SHUTDOWN_DRAIN_MS || '100000', 10) this.logger.log(`Draining queue workers (up to ${drainMs / 1000}s)`) const drain = async (): Promise => { // Local pause without waiting: stops fetching new jobs on this pod only. - await Promise.all(queues.map((queue) => queue.pause(true, true))) + await Promise.all(workers.map((worker) => worker.pause(true))) + // A stage closes once its active jobs finish. for (const stage of stages) { - await Promise.all(stage.map((queue) => queue.close())) + await Promise.all(stage.map((worker) => worker.close())) } + await Promise.all(queues.map((queue) => queue.close())) } let timer: NodeJS.Timeout | undefined @@ -267,103 +211,27 @@ export class QueueModule implements OnModuleInit, OnModuleDestroy, BeforeApplica } onModuleInit(): void { + const redisConfig = this.configService?.get('redis') // Skip queue initialization if queues are not available (e.g., in tests without Redis) - if (!this.ingestionQueue || !this.emailQueue || !this.smsQueue) { + if (!redisConfig || !this.ingestionQueue || !this.emailQueue || !this.smsQueue) { this.logger.debug('Queue configuration not available - skipping worker initialization') return } - this.logger.debug('Initializing queue workers...') - this.logger.debug( - `Dependency check - notificationService available: ${!!this.notificationService}`, - ) - - // Read concurrency configuration - const concurrency = this.configService?.get('queue.ingestionWorkerConcurrency') || 1 - this.logger.debug(`Ingestion worker concurrency: ${concurrency}`) - // Published per pod so the admin monitoring page can list every pod's workers. this.heartbeat = this.redisClient ? new WorkerHeartbeat(this.redisClient) : undefined - // Initialize workers in background - don't block app startup - // Workers will be ready when first job is queued + const workers: Array<[Queue | null | undefined, QueueProcessor | undefined]> = [ + [this.ingestionQueue, this.ingestionWorker], + [this.emailQueue, this.emailDeliveryWorker], + [this.smsQueue, this.smsDeliveryWorker], + [this.webhookQueue, this.webhookDeliveryWorker], + ] try { - this.logger.debug('About to initialize ingestion worker...') - // Initialize ingestion worker - orchestrates fan-out to delivery queues - IngestionWorker.initialize({ - ingestionQueue: this.ingestionQueue, - emailQueue: this.emailQueue, - smsQueue: this.smsQueue, - notificationService: this.notificationService, - requestDetailService: this.notificationRequestDetailService, - configService: this.configService, - clamavService: this.clamavService, - concurrency, - attachmentService: this.attachmentService, - phoneNumberService: this.phoneNumberService, - }) - this.heartbeat?.track(this.ingestionQueue, concurrency) - this.logger.debug('Ingestion worker initialization started') - - this.logger.debug('About to initialize email delivery worker...') - // Initialize email delivery worker - handles email sending - // IMPORTANT: Do NOT await these - initialize() does not await process() - // and returns immediately after setting up listeners - const emailConcurrency = - this.configService?.get('queue.emailDeliveryWorkerConcurrency') || 2 - EmailDeliveryWorker.initialize({ - emailQueue: this.emailQueue, - notificationService: this.notificationService, - templatesRepository: this.templatesRepository, - templatesService: this.templatesService, - inlineRenderingService: this.inlineRenderingService, - attachmentResolverService: this.attachmentResolverService, - emailAdapter: this.emailAdapter, - requestDetailService: this.notificationRequestDetailService, - concurrency: emailConcurrency, - structuredLogger: this.structuredLogger, - tenantSettingsService: this.tenantSettingsService, - eventNotificationResolver: this.eventNotificationResolver, - }) - this.heartbeat?.track(this.emailQueue, emailConcurrency) - this.logger.log('Email delivery worker initialization started') - - this.logger.log('About to initialize SMS delivery worker...') - // Initialize SMS delivery worker - handles SMS sending - // IMPORTANT: Do NOT await these - initialize() does not await process() - // and returns immediately after setting up listeners - const smsConcurrency = - this.configService?.get('queue.smsDeliveryWorkerConcurrency') || 2 - SmsDeliveryWorker.initialize({ - smsQueue: this.smsQueue, - notificationService: this.notificationService, - templatesRepository: this.templatesRepository, - templatesService: this.templatesService, - inlineRenderingService: this.inlineRenderingService, - smsAdapter: this.smsAdapter, - requestDetailService: this.notificationRequestDetailService, - concurrency: smsConcurrency, - structuredLogger: this.structuredLogger, - }) - this.heartbeat?.track(this.smsQueue, smsConcurrency) - this.logger.log('SMS delivery worker initialization started') - - // Initialize webhook delivery worker — handles HTTP POST callbacks - if (this.webhookQueue && this.webhookService && this.webhookDeliveryLogRepository) { - const webhookConcurrency = - this.configService?.get('queue.webhookDeliveryWorkerConcurrency') || 2 - WebhookDeliveryWorker.initialize( - this.webhookQueue, - this.webhookService, - this.webhookDeliveryLogRepository, - webhookConcurrency, - ) - this.heartbeat?.track(this.webhookQueue, webhookConcurrency) - this.logger.log('Webhook delivery worker initialization started') + for (const [queue, worker] of workers) { + if (queue && worker) this.startWorker(worker, redisConfig) } - this.heartbeat?.start() - this.logger.log('Queue workers initialized successfully') } catch (error) { this.logger.error( @@ -372,4 +240,39 @@ export class QueueModule implements OnModuleInit, OnModuleDestroy, BeforeApplica this.logger.debug(error) } } + + /** Run a worker against its queue at its configured concurrency. */ + private startWorker(processor: QueueProcessor, redisConfig: RedisConfig): void { + const concurrency = + this.configService?.get(`queue.${processor.concurrency.configKey}`) || + processor.concurrency.default + this.logger.log(`Registering ${processor.queueName} worker (concurrency=${concurrency})`) + + const worker = createWorker( + processor.queueName, + (job) => processor.process(job), + redisConfig, + concurrency, + ) + const { onCompleted, onFailed } = processor + if (onCompleted) { + worker.on('completed', (job: Job) => onCompleted.call(processor, job)) + } + if (onFailed) { + // BullMQ types the job as optional here; a job that stalls out never reaches this event. + worker.on('failed', (job: Job | undefined, error: Error) => { + if (!job) return + // An event listener's rejection reaches nothing but Node's unhandled-rejection handler. + Promise.resolve() + .then(() => onFailed.call(processor, job, error)) + .catch((handlerError: Error) => { + this.logger.error( + `[${processor.queueName}] Failed-job handler threw for job ${job.id}: ${handlerError.message}`, + ) + }) + }) + } + this.heartbeat?.track(worker as Worker, concurrency) + this.workers.set(processor.queueName, worker as Worker) + } } diff --git a/backend/src/queue/redis-connection.spec.ts b/backend/src/queue/redis-connection.spec.ts index 27a685ad5..1046a265b 100644 --- a/backend/src/queue/redis-connection.spec.ts +++ b/backend/src/queue/redis-connection.spec.ts @@ -1,18 +1,26 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { QueueName } from '../enum/queue-name.enum' -const bullConstructor = vi.fn() +const queueConstructor = vi.fn() +const workerConstructor = vi.fn() -vi.mock('bull', () => ({ - default: vi.fn().mockImplementation(function (name: string, opts: unknown) { - bullConstructor(name, opts) +vi.mock('bullmq', () => ({ + Queue: vi.fn().mockImplementation(function (name: string, opts: unknown) { + queueConstructor(name, opts) return { name, on: vi.fn(), add: vi.fn() } }), + Worker: vi.fn().mockImplementation(function (name: string, processor: unknown, opts: unknown) { + workerConstructor(name, processor, opts) + return { name, on: vi.fn() } + }), })) +import { QUEUE_KEY_PREFIX } from '../common/redis/redis-namespace' +import { backoffStrategy } from './job-options' import { buildRedisOptions, createQueue, + createWorker, QUEUE_MAX_STALLED_COUNT, QUEUE_METRICS_MAX_DATA_POINTS, type RedisConfig, @@ -106,15 +114,39 @@ describe('buildRedisOptions', () => { describe('createQueue', () => { beforeEach(() => { - bullConstructor.mockClear() + queueConstructor.mockClear() }) - it('enables Bull metrics so completed/failed throughput can be read back per minute', () => { + it('keeps BullMQ keys under their own prefix, apart from the Bull keys before them', () => { createQueue(QueueName.EMAIL_DELIVERY, { host: 'localhost', port: 6379, db: 0 }) - expect(bullConstructor).toHaveBeenCalledWith( + expect(QUEUE_KEY_PREFIX).toMatch(/:mq$/) + expect(queueConstructor).toHaveBeenCalledWith( + QueueName.EMAIL_DELIVERY, + expect.objectContaining({ + prefix: QUEUE_KEY_PREFIX, + connection: expect.objectContaining({ maxRetriesPerRequest: null }), + }), + ) + }) +}) + +describe('createWorker', () => { + const processor = vi.fn() + + beforeEach(() => { + workerConstructor.mockClear() + }) + + it('enables metrics so completed/failed throughput can be read back per minute', () => { + createWorker(QueueName.EMAIL_DELIVERY, processor, { host: 'localhost', port: 6379, db: 0 }, 4) + + expect(workerConstructor).toHaveBeenCalledWith( QueueName.EMAIL_DELIVERY, + processor, expect.objectContaining({ + prefix: QUEUE_KEY_PREFIX, + concurrency: 4, metrics: { maxDataPoints: QUEUE_METRICS_MAX_DATA_POINTS }, }), ) @@ -124,15 +156,26 @@ describe('createQueue', () => { expect(QUEUE_METRICS_MAX_DATA_POINTS).toBe(1440) }) - it('lets a job stall more than once before Bull fails it', () => { - // A batch can be interrupted twice in one send (a pod deleted, then a deploy). Bull's default + it('lets a job stall more than once before it is failed', () => { + // A batch can be interrupted twice in one send (a pod deleted, then a deploy). The default // of 1 fails it on the second; retries are safe because merge batches skip sent recipients. - createQueue(QueueName.EMAIL_DELIVERY, { host: 'localhost', port: 6379, db: 0 }) + createWorker(QueueName.EMAIL_DELIVERY, processor, { host: 'localhost', port: 6379, db: 0 }, 1) expect(QUEUE_MAX_STALLED_COUNT).toBe(3) - expect(bullConstructor).toHaveBeenCalledWith( + expect(workerConstructor).toHaveBeenCalledWith( QueueName.EMAIL_DELIVERY, - expect.objectContaining({ settings: { maxStalledCount: 3 } }), + processor, + expect.objectContaining({ maxStalledCount: 3 }), + ) + }) + + it('registers the backoff strategy every job names', () => { + createWorker(QueueName.SMS_DELIVERY, processor, { host: 'localhost', port: 6379, db: 0 }, 1) + + expect(workerConstructor).toHaveBeenCalledWith( + QueueName.SMS_DELIVERY, + processor, + expect.objectContaining({ settings: { backoffStrategy } }), ) }) }) diff --git a/backend/src/queue/redis-connection.ts b/backend/src/queue/redis-connection.ts index 7db367931..d0fef55a9 100644 --- a/backend/src/queue/redis-connection.ts +++ b/backend/src/queue/redis-connection.ts @@ -1,9 +1,10 @@ -import Bull from 'bull' +import { Job, Queue, QueueEvents, Worker } from 'bullmq' import Redis from 'ioredis' import { attachRedisErrorLogging } from '../common/redis/redis-error.util' -import { REDIS_KEY_PREFIX } from '../common/redis/redis-namespace' +import { QUEUE_KEY_PREFIX } from '../common/redis/redis-namespace' import { QueueName } from '../enum/queue-name.enum' import { countAddedJobs } from './queue-metrics' +import { backoffStrategy } from './job-options' /** Shape of the `redis` block in configuration.ts. */ export interface RedisSentinelNode { @@ -47,7 +48,7 @@ export function buildRedisOptions( } /** - * Minutes of completed/failed counts Bull keeps per queue, read back with `queue.getMetrics()`. + * Minutes of completed/failed counts BullMQ keeps per queue, read back with `queue.getMetrics()`. * Each point is one integer in a capped Redis list, so a day of history is a few KB per queue. */ export const QUEUE_METRICS_MAX_DATA_POINTS = Number.parseInt( @@ -56,8 +57,8 @@ export const QUEUE_METRICS_MAX_DATA_POINTS = Number.parseInt( ) /** - * Times a job may stall (its pod dies or loses its lock mid-job) before Bull fails it instead - * of retrying. Bull's default of 1 fails a batch interrupted twice, e.g. a pod deleted and then + * Times a job may stall (its pod dies or loses its lock mid-job) before BullMQ fails it instead + * of retrying. The default of 1 fails a batch interrupted twice, e.g. a pod deleted and then * restarted during the same send. A retry is safe: merge batches skip recipients already sent. */ export const QUEUE_MAX_STALLED_COUNT = Number.parseInt( @@ -65,30 +66,73 @@ export const QUEUE_MAX_STALLED_COUNT = Number.parseInt( 10, ) +/** Approximate length the events stream is trimmed to, per queue. */ +export const QUEUE_EVENTS_MAX_LEN = 1000 + +/** + * BullMQ opens its own connections from these options, one per Queue and two per Worker. Its + * workers block on Redis, so a command must never be failed for taking too long. + */ +function queueConnection(redisConfig: RedisConfig): Record { + return buildRedisOptions(redisConfig, { enableReadyCheck: false, maxRetriesPerRequest: null }) +} + /** - * Create a Bull queue with an error listener attached. + * Create a BullMQ queue - the producer side - with an error listener attached. * * The listener is not optional: without one, a Redis failure prints the raw ioredis error - * which carries the AUTH command's arguments - straight to stderr. See redis-error.util. + * `redis` counts each add for the monitoring page's fill rate; omit it to skip counting. */ -export function createQueue(name: QueueName, redisConfig: RedisConfig): Bull.Queue { - const queue = new Bull(name, { +export function createQueue( + name: QueueName, + redisConfig: RedisConfig, + redis?: Redis | null, +): Queue { + const queue = new Queue(name, { // Scope the queue to this deployment. Without it every deployment sharing the Redis // instance consumes from the same queue, and a job is processed by whichever pod wins the // race - which then cannot find the notification in its own database. - prefix: REDIS_KEY_PREFIX, - redis: buildRedisOptions(redisConfig, { - enableReadyCheck: false, - maxRetriesPerRequest: null, - }), - metrics: { maxDataPoints: QUEUE_METRICS_MAX_DATA_POINTS }, - settings: { maxStalledCount: QUEUE_MAX_STALLED_COUNT }, + prefix: QUEUE_KEY_PREFIX, + connection: queueConnection(redisConfig), + // The events stream is only read live, by the monitoring page; the default keeps 10,000 + // events per queue per deployment in a Redis that refuses writes once it is full. + streams: { events: { maxLen: QUEUE_EVENTS_MAX_LEN } }, }) attachRedisErrorLogging(queue, `Queue[${name}]`) - countAddedJobs(queue) + if (redis) countAddedJobs(queue as Queue, redis) return queue } +/** Create a BullMQ worker for a queue, with the same error-logging guarantee. */ +export function createWorker( + name: QueueName, + processor: (job: Job) => Promise, + redisConfig: RedisConfig, + concurrency: number, +): Worker { + const worker = new Worker(name, processor, { + prefix: QUEUE_KEY_PREFIX, + connection: queueConnection(redisConfig), + concurrency, + maxStalledCount: QUEUE_MAX_STALLED_COUNT, + metrics: { maxDataPoints: QUEUE_METRICS_MAX_DATA_POINTS }, + settings: { backoffStrategy }, + }) + attachRedisErrorLogging(worker, `Worker[${name}]`) + return worker +} + +/** Listen to a queue's events stream, from every pod, with the same error-logging guarantee. */ +export function createQueueEvents(name: QueueName, redisConfig: RedisConfig): QueueEvents { + const events = new QueueEvents(name, { + prefix: QUEUE_KEY_PREFIX, + connection: queueConnection(redisConfig), + }) + attachRedisErrorLogging(events, `QueueEvents[${name}]`) + return events +} + /** Create a plain Redis client with the same error-logging guarantee. */ export function createRedisClient( redisConfig: RedisConfig, diff --git a/backend/src/queue/services/delivery-reconciler.service.spec.ts b/backend/src/queue/services/delivery-reconciler.service.spec.ts index b0f7ec16d..d3b081f38 100644 --- a/backend/src/queue/services/delivery-reconciler.service.spec.ts +++ b/backend/src/queue/services/delivery-reconciler.service.spec.ts @@ -1,7 +1,8 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import type Bull from 'bull' +import type { Job, Queue } from 'bullmq' import type Redis from 'ioredis' import type { Repository } from 'typeorm' +import { QueueName } from '../../enum/queue-name.enum' import { DeliveryReconcilerService } from './delivery-reconciler.service' import { NotificationStatus } from '../../enum/notification-status.enum' import type { NotificationRequestDetail } from '../../api/notification/entities/notification-request-detail.entity' @@ -46,7 +47,7 @@ function setup({ scheduled = [], ingestionJob = null, }: { - job?: Partial | null + job?: Partial | null locked?: string | null attempt?: number /** Stale merge batches, then stale plain-send deliveries: the two detail queries, in order. */ @@ -58,7 +59,7 @@ function setup({ pending?: object[] /** Scheduled sends past their send time; queried after the QUEUED ones. */ scheduled?: object[] - ingestionJob?: Partial | null + ingestionJob?: Partial | null }) { const results = [stale, deliveries] const builder: Record = {} @@ -99,6 +100,7 @@ function setup({ createQueryBuilder: vi.fn(() => requestBuilder), } const ingestionQueue = { + name: QueueName.INGESTION, getJob: vi.fn().mockResolvedValue(ingestionJob), add: vi.fn().mockResolvedValue({}), } @@ -120,6 +122,7 @@ function setup({ expire: vi.fn().mockResolvedValue(1), } const queue = { + name: QueueName.EMAIL_DELIVERY, getJob: vi.fn().mockResolvedValue(job ?? null), add: vi.fn().mockResolvedValue({}), } @@ -129,9 +132,9 @@ function setup({ requestRepository as unknown as Repository, notificationService as unknown as NotificationService, redis as unknown as Redis, - queue as unknown as Bull.Queue, + queue as unknown as Queue, null, - ingestionQueue as unknown as Bull.Queue, + ingestionQueue as unknown as Queue, structuredLogger as unknown as StructuredLoggerService, ) return { @@ -147,28 +150,31 @@ function setup({ } } -const jobIn = (state: string, extra: Partial = {}) => +const jobIn = (state: string, extra: Partial = {}) => ({ getState: vi.fn().mockResolvedValue(state), retry: vi.fn().mockResolvedValue(undefined), remove: vi.fn().mockResolvedValue(undefined), failedReason: 'job stalled more than allowable limit', ...extra, - }) as unknown as Bull.Job + }) as unknown as Job describe('DeliveryReconcilerService', () => { beforeEach(() => vi.clearAllMocks()) - it.each(['waiting', 'active', 'delayed'])('leaves a %s batch alone', async (state) => { - const job = jobIn(state) - const { service, queue, redis } = setup({ job }) + it.each(['waiting', 'active', 'delayed', 'prioritized'])( + 'leaves a %s batch alone', + async (state) => { + const job = jobIn(state) + const { service, queue, redis } = setup({ job }) - expect(await service.reconcile()).toEqual({ 'in-progress': 1 }) - expect(job.retry).not.toHaveBeenCalled() - expect(queue.add).not.toHaveBeenCalled() - // A long batch still running must not use up its recovery attempts. - expect(redis.incr).not.toHaveBeenCalled() - }) + expect(await service.reconcile()).toEqual({ 'in-progress': 1 }) + expect(job.retry).not.toHaveBeenCalled() + expect(queue.add).not.toHaveBeenCalled() + // A long batch still running must not use up its recovery attempts. + expect(redis.incr).not.toHaveBeenCalled() + }, + ) it('retries a batch whose job failed, keeping its original payload', async () => { const job = jobIn('failed') @@ -183,7 +189,7 @@ describe('DeliveryReconcilerService', () => { const { service, queue } = setup({ job: null }) expect(await service.reconcile()).toEqual({ requeued: 1 }) - const [payload, options] = queue.add.mock.calls[0] + const [, payload, options] = queue.add.mock.calls[0] expect(payload).toMatchObject({ notifyId: 'req-1', tenantId: 'tenant-1', @@ -335,6 +341,7 @@ describe('DeliveryReconcilerService', () => { expect(await service.reconcile()).toEqual({ requeued: 1 }) expect(queue.add).not.toHaveBeenCalled() expect(ingestionQueue.add).toHaveBeenCalledWith( + QueueName.INGESTION, expect.objectContaining({ notifyId: 'req-1', tenantId: 'tenant-1' }), expect.objectContaining({ jobId: 'req-1', attempts: 3 }), ) @@ -383,6 +390,7 @@ describe('DeliveryReconcilerService', () => { expect(await service.reconcile()).toEqual({ requeued: 1 }) expect(ingestionQueue.add).toHaveBeenCalledWith( + QueueName.INGESTION, expect.objectContaining({ notifyId: 'req-1', mailMerge: true, mailMergeChannel: 'EMAIL' }), expect.objectContaining({ jobId: 'req-1' }), ) @@ -404,15 +412,15 @@ describe('DeliveryReconcilerService', () => { describe('requests never queued (PENDING)', () => { const PENDING = [{ notifyId: 'req-1', tenantId: 'tenant-1' }] - it('queues ingestion from the stored request, unnamed, and moves the request to QUEUED', async () => { + it('queues ingestion from the stored request and moves the request to QUEUED', async () => { const { service, ingestionQueue, notificationService } = setup({ stale: [], pending: PENDING, }) expect(await service.reconcile()).toEqual({ requeued: 1 }) - const [payload, options] = ingestionQueue.add.mock.calls[0] - // Unnamed: the ingestion worker registers Bull's default handler; a named job finds none. + const [name, payload, options] = ingestionQueue.add.mock.calls[0] + expect(name).toBe(QueueName.INGESTION) expect(typeof payload).toBe('object') expect(payload).toMatchObject({ notifyId: 'req-1', @@ -446,7 +454,7 @@ describe('DeliveryReconcilerService', () => { await service.reconcile() - const [payload, options] = ingestionQueue.add.mock.calls[0] + const [, payload, options] = ingestionQueue.add.mock.calls[0] expect(payload.scheduledFor).toBe(sendAt) expect(options.delay).toBeGreaterThan(59 * 60_000) expect(notificationService.update).toHaveBeenCalledWith('req-1', 'tenant-1', { @@ -488,7 +496,7 @@ describe('DeliveryReconcilerService', () => { requestRepository.findOne.mockResolvedValue(overdueRequest) expect(await service.reconcile()).toEqual({ requeued: 1 }) - const [payload, options] = ingestionQueue.add.mock.calls[0] + const [, payload, options] = ingestionQueue.add.mock.calls[0] expect(payload).toMatchObject({ notifyId: 'req-1', scheduledFor: '2026-10-04T13:00:00Z' }) expect(options).toMatchObject({ jobId: 'req-1' }) expect(options).not.toHaveProperty('delay') diff --git a/backend/src/queue/services/delivery-reconciler.service.ts b/backend/src/queue/services/delivery-reconciler.service.ts index 26b5751a2..2c480d678 100644 --- a/backend/src/queue/services/delivery-reconciler.service.ts +++ b/backend/src/queue/services/delivery-reconciler.service.ts @@ -9,7 +9,7 @@ import { import { InjectRepository } from '@nestjs/typeorm' import { FindOptionsWhere, In, IsNull, Repository } from 'typeorm' import { randomUUID } from 'node:crypto' -import type Bull from 'bull' +import type { Queue } from 'bullmq' import type Redis from 'ioredis' import { NotificationRequestDetail } from '../../api/notification/entities/notification-request-detail.entity' import { NotificationRequest } from '../../api/notification/entities/notification-request.entity' @@ -20,8 +20,8 @@ import { NotificationStatus } from '../../enum/notification-status.enum' import { ProviderToken } from '../../enum/provider-token.enum' import { QueueName } from '../../enum/queue-name.enum' import { redisKey } from '../../common/redis/redis-namespace' +import { DELIVERY_JOB_OPTIONS, INGESTION_JOB_OPTIONS } from '../job-options' import { StructuredLoggerService } from '../../common/logger' -import { COMPLETED_JOB_RETENTION, FAILED_JOB_RETENTION } from '../job-retention' import { delayUntilScheduled, ingestionJobFromRequest, @@ -65,19 +65,14 @@ end return 0 ` -/** Bull states in which a job will still run without help. */ -const LIVE_JOB_STATES = new Set(['waiting', 'active', 'delayed', 'paused']) +/** States in which a job will still run without help. */ +const LIVE_JOB_STATES = new Set(['waiting', 'active', 'delayed', 'prioritized', 'waiting-children']) const DELIVERY_CHANNELS = [NotificationChannel.EMAIL, NotificationChannel.SMS] -const JOB_OPTIONS = { - attempts: 3, - backoff: { type: 'exponential', delay: 2000 }, -} as const - export type ReconcileOutcome = 'in-progress' | 'retried' | 'requeued' | 'gave-up' /** - * Work Postgres says is owed, keyed by the Bull job that should be doing it: + * Work Postgres says is owed, keyed by the job that should be doing it: * - `pending`: a request stored but never queued (Redis unreachable at acceptance), job * `` on ingestion; * - `ingestion`: a request queued but never fanned out, job `` on ingestion; @@ -99,7 +94,7 @@ interface StuckWork { * Finds work Postgres says is still owed but no job is left to do, and puts it back on a queue - * from a request that never reached Redis at all to one batch of a merge that lost its job. * - * The queue's own recovery covers a worker dying mid-job: Bull notices the lapsed lock and + * The queue's own recovery covers a worker dying mid-job: BullMQ notices the lapsed lock and * retries. It does not cover a job that ends up failed (out of retries, or stalled more often * than maxStalledCount) or one that never reached Redis; what it owed would stay pending forever. * Postgres is the record of what is owed, so this compares it with what the queue holds: @@ -127,9 +122,9 @@ export class DeliveryReconcilerService implements OnApplicationBootstrap, OnModu private readonly requestRepository: Repository, private readonly notificationService: NotificationService, @Optional() @Inject(ProviderToken.REDIS_CLIENT) private readonly redis?: Redis | null, - @Optional() @Inject(QueueName.EMAIL_DELIVERY) private readonly emailQueue?: Bull.Queue | null, - @Optional() @Inject(QueueName.SMS_DELIVERY) private readonly smsQueue?: Bull.Queue | null, - @Optional() @Inject(QueueName.INGESTION) private readonly ingestionQueue?: Bull.Queue | null, + @Optional() @Inject(QueueName.EMAIL_DELIVERY) private readonly emailQueue?: Queue | null, + @Optional() @Inject(QueueName.SMS_DELIVERY) private readonly smsQueue?: Queue | null, + @Optional() @Inject(QueueName.INGESTION) private readonly ingestionQueue?: Queue | null, @Optional() private readonly structuredLogger?: StructuredLoggerService, ) {} @@ -318,7 +313,7 @@ export class DeliveryReconcilerService implements OnApplicationBootstrap, OnModu .andWhere('request.status <> :scheduled', { scheduled: NotificationStatus.SCHEDULED }) } - private queueFor(item: StuckWork): Bull.Queue | null | undefined { + private queueFor(item: StuckWork): Queue | null | undefined { if (item.kind === 'ingestion' || item.kind === 'pending' || item.kind === 'scheduled') return this.ingestionQueue return item.channel === NotificationChannel.SMS ? this.smsQueue : this.emailQueue @@ -355,11 +350,9 @@ export class DeliveryReconcilerService implements OnApplicationBootstrap, OnModu // Finished or gone while work is still owed: replace it. if (job) await job.remove().catch(() => undefined) if (item.kind === 'batch') { - await queue.add(await this.rebuildBatchJob(item), { + await queue.add(queue.name, await this.rebuildBatchJob(item), { jobId: item.jobId, - ...JOB_OPTIONS, - removeOnComplete: true, - removeOnFail: FAILED_JOB_RETENTION, + ...DELIVERY_JOB_OPTIONS, }) } else { await this.reingest(item) @@ -377,7 +370,7 @@ export class DeliveryReconcilerService implements OnApplicationBootstrap, OnModu /** * Run ingestion for the request again, from the stored request. A finished ingestion job of the - * same id is removed first, or Bull would treat the add as a duplicate and do nothing. A + * same id is removed first, or BullMQ would treat the add as a duplicate and do nothing. A * scheduled send keeps its delay. A request recovered from PENDING then moves on as acceptance * would have moved it - to SCHEDULED or QUEUED - so it leaves this finder. */ @@ -393,12 +386,10 @@ export class DeliveryReconcilerService implements OnApplicationBootstrap, OnModu const delay = delayUntilScheduled(job) const previous = await this.ingestionQueue.getJob(item.notifyId) if (previous) await previous.remove().catch(() => undefined) - await this.ingestionQueue.add(job, { + await this.ingestionQueue.add(this.ingestionQueue.name, job, { jobId: item.notifyId, - ...JOB_OPTIONS, + ...INGESTION_JOB_OPTIONS, ...(delay > 0 && { delay }), - removeOnComplete: COMPLETED_JOB_RETENTION, - removeOnFail: FAILED_JOB_RETENTION, }) if (item.kind === 'pending') { diff --git a/backend/src/queue/services/webhook-trigger.service.ts b/backend/src/queue/services/webhook-trigger.service.ts index 96fc82bed..839352d95 100644 --- a/backend/src/queue/services/webhook-trigger.service.ts +++ b/backend/src/queue/services/webhook-trigger.service.ts @@ -1,12 +1,12 @@ import { Injectable, Logger, OnModuleInit, OnModuleDestroy, Inject, Optional } from '@nestjs/common' import { ConfigService } from '@nestjs/config' import Redis from 'ioredis' -import Bull from 'bull' +import type { Queue } from 'bullmq' import { WebhookService } from '../../api/webhook/webhook.service' import { WebhookJobPayload } from '../queue.types' import { QueueName } from '../../enum/queue-name.enum' import { NotificationRequestDto } from '../../api/notification/schemas/notification-request' -import { COMPLETED_JOB_RETENTION, FAILED_JOB_RETENTION } from '../job-retention' +import { WEBHOOK_JOB_OPTIONS } from '../job-options' import { buildRedisOptions } from '../redis-connection' /** @@ -36,7 +36,7 @@ export class WebhookTriggerService implements OnModuleInit, OnModuleDestroy { private readonly webhookService: WebhookService, @Optional() @Inject(QueueName.WEBHOOK_DELIVERY) - private readonly webhookQueue?: Bull.Queue, + private readonly webhookQueue?: Queue, ) {} onModuleInit(): void { @@ -94,67 +94,68 @@ export class WebhookTriggerService implements OnModuleInit, OnModuleDestroy { const notificationStatus = notification.status?.code?.toLowerCase() const notificationChannels = (notification.channelCodes ?? []).map((c) => c.toLowerCase()) - for (const webhook of activeWebhooks) { - const triggerTypes = webhook.triggerOn ?? [] - - // Map spec trigger values to notification statuses: - // 'success' → notification status 'completed' - // 'failure' → notification status 'failed' - // Empty triggerOn = fire for any status - const statusMatches = - triggerTypes.length === 0 || - triggerTypes.some((t) => { - if (t === 'success') return notificationStatus === 'completed' - if (t === 'failure') return notificationStatus === 'failed' - return false - }) - - if (!statusMatches) continue - - // Empty channelType = fire for any channel; otherwise fire if the request targeted any of - // the webhook's channels - const channelTypes = webhook.channelType ?? [] - const channelMatches = - channelTypes.length === 0 || - channelTypes.some((ct) => notificationChannels.includes(ct.toLowerCase())) - - if (!channelMatches) continue - - const jobPayload: WebhookJobPayload = { - notificationId: notification.id, - tenantId, - webhookId: webhook.id, - event: 'notification.status.changed', - payload: { - notifyId: notification.id, - status: notification.status?.code, // Emit the status code rather than the status object - statusDisplayName: notification.status?.displayName, - channel: notification.channelCodes?.join(', '), - createdAt: notification.createdAt, - updatedAt: notification.updatedAt, - }, - } - - try { - // Job ID includes status to allow one delivery per webhook per status transition - const jobId = `webhook_${webhook.id}_${notification.id}_${notificationStatus}` - await this.webhookQueue!.add(jobPayload, { - jobId, - attempts: 3, - backoff: { type: 'exponential', delay: 5000 }, - // Every pod receives each status change and adds this job. Keeping the completed job is - // what lets Bull drop a slower pod's add as a duplicate instead of delivering it again. - removeOnComplete: COMPLETED_JOB_RETENTION, - removeOnFail: FAILED_JOB_RETENTION, - }) - this.logger.log( - `[${notification.id}] Queued webhook delivery for webhook=${webhook.id} (status=${notificationStatus})`, - ) - } catch (err) { - this.logger.error( - `[${notification.id}] Failed to queue webhook delivery for webhook=${webhook.id}: ${(err as Error)?.message}`, - ) - } - } + // A tenant's webhooks are independent; each logs its own failure. + await Promise.all( + activeWebhooks.map(async (webhook) => { + const triggerTypes = webhook.triggerOn ?? [] + + // Map spec trigger values to notification statuses: + // 'success' → notification status 'completed' + // 'failure' → notification status 'failed' + // Empty triggerOn = fire for any status + const statusMatches = + triggerTypes.length === 0 || + triggerTypes.some((t) => { + if (t === 'success') return notificationStatus === 'completed' + if (t === 'failure') return notificationStatus === 'failed' + return false + }) + + if (!statusMatches) return + + // Empty channelType = fire for any channel; otherwise fire if the request targeted any of + // the webhook's channels + const channelTypes = webhook.channelType ?? [] + const channelMatches = + channelTypes.length === 0 || + channelTypes.some((ct) => notificationChannels.includes(ct.toLowerCase())) + + if (!channelMatches) return + + const jobPayload: WebhookJobPayload = { + notificationId: notification.id, + tenantId, + webhookId: webhook.id, + event: 'notification.status.changed', + payload: { + notifyId: notification.id, + status: notification.status?.code, // Emit the status code rather than the status object + statusDisplayName: notification.status?.displayName, + channel: notification.channelCodes?.join(', '), + createdAt: notification.createdAt, + updatedAt: notification.updatedAt, + }, + } + + try { + // Job ID includes status to allow one delivery per webhook per status transition + const jobId = `webhook_${webhook.id}_${notification.id}_${notificationStatus}` + // Every pod receives each status change and adds this job. Keeping the completed job + // (WEBHOOK_JOB_OPTIONS) is what lets BullMQ drop a slower pod's add as a duplicate + // instead of delivering it again. + await this.webhookQueue!.add(this.webhookQueue!.name, jobPayload, { + jobId, + ...WEBHOOK_JOB_OPTIONS, + }) + this.logger.log( + `[${notification.id}] Queued webhook delivery for webhook=${webhook.id} (status=${notificationStatus})`, + ) + } catch (err) { + this.logger.error( + `[${notification.id}] Failed to queue webhook delivery for webhook=${webhook.id}: ${(err as Error)?.message}`, + ) + } + }), + ) } } diff --git a/backend/src/queue/worker-heartbeat.spec.ts b/backend/src/queue/worker-heartbeat.spec.ts index 716616cd3..f573b3b05 100644 --- a/backend/src/queue/worker-heartbeat.spec.ts +++ b/backend/src/queue/worker-heartbeat.spec.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi } from 'vitest' import { EventEmitter } from 'events' -import type Bull from 'bull' +import type { Job, Worker } from 'bullmq' import type Redis from 'ioredis' import { HEARTBEAT_TTL_SECONDS, @@ -25,17 +25,17 @@ function fakeRedis() { return { redis: { multi: vi.fn(() => multi) } as unknown as Redis, multi } } -const job = (id: string | number) => ({ id }) as unknown as Bull.Job +const job = (id: string | number) => ({ id }) as unknown as Job -function fakeQueue(name: string) { - return Object.assign(new EventEmitter(), { name }) as unknown as Bull.Queue +function fakeWorker(name: string) { + return Object.assign(new EventEmitter(), { name }) as unknown as Worker } describe('WorkerHeartbeat', () => { it('publishes per-queue activity with an expiring key and indexes the pod', async () => { const { redis, multi } = fakeRedis() const heartbeat = new WorkerHeartbeat(redis) - const queue = fakeQueue('email-delivery') + const queue = fakeWorker('email-delivery') heartbeat.track(queue, 2) queue.emit('active', job(1)) @@ -66,7 +66,7 @@ describe('WorkerHeartbeat', () => { it('ignores a job this pod never ran finishing', async () => { const { redis, multi } = fakeRedis() const heartbeat = new WorkerHeartbeat(redis) - const queue = fakeQueue('sms-delivery') + const queue = fakeWorker('sms-delivery') heartbeat.track(queue, 1) queue.emit('completed', job(9)) @@ -76,12 +76,10 @@ describe('WorkerHeartbeat', () => { expect(payload.queues[0]).toMatchObject({ active: 0, completed: 0 }) }) - it("keeps a running job busy when the stalled-job check fails another pod's job", async () => { - // Bull's stalled-job check runs on every worker and emits a local `failed` for a job it - // gives up on, even one a dead pod was running. That must not end this pod's own job. + it('ignores a finish for a job this worker never started', async () => { const { redis, multi } = fakeRedis() const heartbeat = new WorkerHeartbeat(redis) - const queue = fakeQueue('email-delivery') + const queue = fakeWorker('email-delivery') heartbeat.track(queue, 2) queue.emit('active', job('mine')) @@ -101,7 +99,7 @@ describe('WorkerHeartbeat', () => { it('reports draining while the pod shuts down, then removes itself', async () => { const { redis, multi } = fakeRedis() const heartbeat = new WorkerHeartbeat(redis) - heartbeat.track(fakeQueue('email-delivery'), 2) + heartbeat.track(fakeWorker('email-delivery'), 2) await heartbeat.beat() expect(JSON.parse(multi.set.mock.calls[0][1]).draining).toBe(false) diff --git a/backend/src/queue/worker-heartbeat.ts b/backend/src/queue/worker-heartbeat.ts index 0bc519aed..8caca066e 100644 --- a/backend/src/queue/worker-heartbeat.ts +++ b/backend/src/queue/worker-heartbeat.ts @@ -1,4 +1,4 @@ -import type Bull from 'bull' +import type { Job, Worker } from 'bullmq' import type Redis from 'ioredis' import { hostname } from 'node:os' import { Logger } from '@nestjs/common' @@ -38,14 +38,14 @@ export interface WorkerHeartbeatPayload { const logger = new Logger('WorkerHeartbeat') /** - * Publishes what this pod's Bull workers are doing so the admin monitoring page can list every + * Publishes what this pod's queue workers are doing so the admin monitoring page can list every * pod's workers, not just those of the pod that served the request. * * The counts are this pod's own and are meant to disappear with it: the key expires * HEARTBEAT_TTL_SECONDS after the last write, so a crashed or scaled-down pod falls off the * list on its own. `completed` and `failed` are since this pod started, not lifetime totals. - * Bull's `getWorkers()` is not used because it depends on CLIENT LIST, and returns undefined - * rather than failing when a managed Redis disallows it. + * BullMQ's `getWorkers()` is not used because it depends on CLIENT LIST, which a managed Redis + * may disallow. */ export class WorkerHeartbeat { private readonly podId = process.env.HOSTNAME || hostname() @@ -56,34 +56,33 @@ export class WorkerHeartbeat { constructor(private readonly redis: Redis) {} - /** Track a queue this pod processes. Call once per queue, after its worker is registered. */ - track(queue: Bull.Queue, concurrency: number): void { + /** Track a worker this pod runs. Call once per queue, as its worker is created. */ + track(worker: Pick, concurrency: number): void { const state: WorkerQueueState = { - queue: queue.name, + queue: worker.name, concurrency, active: 0, completed: 0, failed: 0, lastFinishedAt: null, } - this.states.set(queue.name, state) + this.states.set(worker.name, state) - // Keyed by job id, not counted: Bull's stalled-job check emits a local `failed` for jobs - // another pod abandoned, which a plain counter would take as one of this pod's jobs ending. - const running = new Set() - const finish = (job: Bull.Job | undefined, outcome: 'completed' | 'failed') => { + // Keyed by job id, not counted, so a finish only counts for a job this worker started. + const running = new Set() + const finish = (job: Job | undefined, outcome: 'completed' | 'failed') => { if (!job || !running.delete(job.id)) return state.active = running.size state[outcome]++ state.lastFinishedAt = Date.now() } - queue.on('active', (job: Bull.Job) => { + worker.on('active', (job: Job) => { running.add(job.id) state.active = running.size }) - queue.on('completed', (job: Bull.Job) => finish(job, 'completed')) - queue.on('failed', (job: Bull.Job) => finish(job, 'failed')) + worker.on('completed', (job: Job) => finish(job, 'completed')) + worker.on('failed', (job: Job | undefined) => finish(job, 'failed')) } start(): void { diff --git a/backend/src/queue/workers/delivery-channel.ts b/backend/src/queue/workers/delivery-channel.ts new file mode 100644 index 000000000..062e22d99 --- /dev/null +++ b/backend/src/queue/workers/delivery-channel.ts @@ -0,0 +1,37 @@ +/** What the email and SMS delivery workers name differently in logs, errors and audit columns. */ +export interface DeliveryChannel { + /** Structured-log channel. */ + channel: 'email' | 'sms' + /** The channel as written in log lines. */ + displayName: string + /** templates.channel_code a template must have to be sent on this channel. */ + templateCode: 'EMAIL' | 'SMS' + /** A merge send on this channel, as written in log lines and errors. */ + mergeLabel: string + /** updated_by on the parent request when a merge settles. */ + updatedBy: string + /** Logger and structured-log context. */ + context: string +} + +export const EMAIL_CHANNEL: DeliveryChannel = { + channel: 'email', + displayName: 'email', + templateCode: 'EMAIL', + mergeLabel: 'mail merge', + updatedBy: 'email-delivery-worker', + context: 'EmailDeliveryWorker', +} + +export const SMS_CHANNEL: DeliveryChannel = { + channel: 'sms', + displayName: 'SMS', + templateCode: 'SMS', + mergeLabel: 'SMS merge', + updatedBy: 'sms-delivery-worker', + context: 'SmsDeliveryWorker', +} + +export function capitalize(text: string): string { + return text.charAt(0).toUpperCase() + text.slice(1) +} diff --git a/backend/src/queue/workers/delivery-failure-handler.spec.ts b/backend/src/queue/workers/delivery-failure-handler.spec.ts new file mode 100644 index 000000000..45f2a2a9c --- /dev/null +++ b/backend/src/queue/workers/delivery-failure-handler.spec.ts @@ -0,0 +1,59 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { Logger } from '@nestjs/common' +import type { Job } from 'bullmq' +import { DeliveryFailureHandler } from './delivery-failure-handler' +import { EMAIL_CHANNEL } from './delivery-channel' +import type { MergeBatchRunner } from './merge-batch-runner' +import type { DeliveryJobPayload } from '../queue.types' +import type { NotificationService } from '../../api/notification/notification.service' +import type { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' + +describe('DeliveryFailureHandler', () => { + const notificationService = { update: vi.fn().mockResolvedValue(undefined) } + const requestDetailService = { markFailed: vi.fn().mockResolvedValue(undefined) } + const handler = new DeliveryFailureHandler( + notificationService as unknown as NotificationService, + requestDetailService as unknown as NotificationRequestDetailService, + {} as MergeBatchRunner, + ) + const job = (attemptsMade: number, attempts?: number) => + ({ + data: { notifyId: 'n1', tenantId: 't1' }, + opts: attempts === undefined ? {} : { attempts }, + attemptsMade, + }) as unknown as Job + + let error: ReturnType + beforeEach(() => { + vi.clearAllMocks() + error = vi.spyOn(Logger.prototype, 'error').mockImplementation(() => {}) + }) + + it('reports attempts from the job, not a fixed count', async () => { + await expect( + handler.fail(job(3, 5), new Error('boom'), EMAIL_CHANNEL, Date.now()), + ).rejects.toThrow() + + expect(error).toHaveBeenCalledWith(expect.stringContaining('(attempt 4/5)'), expect.anything()) + // Not the last of five, so nothing is marked yet. + expect(requestDetailService.markFailed).not.toHaveBeenCalled() + }) + + it('marks the send failed on the job’s own last attempt', async () => { + await expect( + handler.fail(job(4, 5), new Error('boom'), EMAIL_CHANNEL, Date.now()), + ).rejects.toThrow() + + expect(requestDetailService.markFailed).toHaveBeenCalledWith('n1', 'boom') + expect(error).toHaveBeenCalledWith(expect.stringContaining('after 5 attempt(s)')) + }) + + it('treats a job queued without attempts as on its only attempt, as BullMQ does', async () => { + await expect( + handler.fail(job(0), new Error('boom'), EMAIL_CHANNEL, Date.now()), + ).rejects.toThrow() + + expect(error).toHaveBeenCalledWith(expect.stringContaining('(attempt 1/1)'), expect.anything()) + expect(requestDetailService.markFailed).toHaveBeenCalled() + }) +}) diff --git a/backend/src/queue/workers/delivery-failure-handler.ts b/backend/src/queue/workers/delivery-failure-handler.ts new file mode 100644 index 000000000..82caae057 --- /dev/null +++ b/backend/src/queue/workers/delivery-failure-handler.ts @@ -0,0 +1,145 @@ +import { HttpException, Injectable, Logger, Optional } from '@nestjs/common' +import { describeError } from '../../common/utils/describe-error' +import { Job, UnrecoverableError } from 'bullmq' +import { DeliveryJobPayload } from '../queue.types' +import { DeliveryChannel } from './delivery-channel' +import { maxAttempts } from '../job-options' +import { MergeBatchRunner } from './merge-batch-runner' +import { isTransientDeliveryError } from '../../adapters/delivery-errors' +import { NotificationService } from '../../api/notification/notification.service' +import { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' +import { NotificationStatus } from '../../enum/notification-status.enum' +import { StructuredLoggerService } from '../../common/logger' + +/** A 400 will fail the same way on every attempt, so it is not retried. */ +export function isPermanentValidationError(error: unknown): error is HttpException { + return error instanceof HttpException && error.getStatus() === 400 +} + +/** The same failure, marked so BullMQ fails the job now instead of spending its retries. */ +export function notRetried(error: unknown): UnrecoverableError { + const unrecoverable = new UnrecoverableError(describeError(error)) + unrecoverable.cause = error + return unrecoverable +} + +/** + * Decides what a failed email or SMS delivery attempt leaves behind, then rethrows so the queue + * applies its retry policy - or fails the job outright, when retrying cannot help: + * - a transient provider error marks nothing, even on the last attempt - the recipients stay + * owed, and the delivery reconciler re-queues the job once the queue's retries are spent; + * - a merge batch fails only its own unsent rows, on its last attempt or a permanent error; + * - a plain send is marked FAILED on a permanent error (and not retried) or its last attempt. + */ +@Injectable() +export class DeliveryFailureHandler { + constructor( + private readonly notificationService: NotificationService, + private readonly requestDetailService: NotificationRequestDetailService, + private readonly mergeBatches: MergeBatchRunner, + @Optional() private readonly structuredLogger?: StructuredLoggerService, + ) {} + + async fail( + job: Job, + error: unknown, + channel: DeliveryChannel, + startedAt: number, + ): Promise { + const logger = new Logger(channel.context) + const { notifyId, tenantId } = job.data + const errorMessage = describeError(error) + const attempt = (job.attemptsMade ?? 0) + 1 + const attempts = maxAttempts(job) + const finalAttempt = attempt >= attempts + logger.error( + `[${notifyId}] Failed to send ${channel.displayName} delivery job (attempt ${attempt}/${attempts}): ${errorMessage}`, + error instanceof Error ? error.stack : '', + ) + + if (isTransientDeliveryError(error)) throw error + + if (job.data.mailMerge && job.data.batchId) { + if (finalAttempt || isPermanentValidationError(error)) { + await this.mergeBatches.failUnsent(job, errorMessage, channel) + throw notRetried(error) + } + throw error + } + + if (isPermanentValidationError(error)) { + await this.markFailed(notifyId, tenantId, errorMessage) + logger.error( + `[${notifyId}] Notification marked as FAILED after permanent validation error. Error: ${errorMessage}`, + ) + this.structuredLogger?.logNotificationFailure( + notifyId, + tenantId, + channel.channel, + error, + Date.now() - startedAt, + channel.context, + ) + throw notRetried(error) + } + + if (finalAttempt) { + await this.markFailed(notifyId, tenantId, errorMessage) + logger.error( + `[${notifyId}] Notification marked as FAILED after ${attempts} attempt(s). Error: ${errorMessage}`, + ) + this.structuredLogger?.logNotificationFailure( + notifyId, + tenantId, + channel.channel, + error instanceof Error ? error : errorMessage, + Date.now() - startedAt, + channel.context, + ) + } + throw error + } + + /** + * Once the queue has spent every retry, mark a plain send FAILED even if the processor's + * final-attempt handling did not run (a job queued without a retry count, or a different + * worker processing the last attempt). Not for a transient error - the recipients stay owed + * for the reconciler - nor for a merge batch, which the processor settled itself, scoped to + * that batch. A job that stalls out never reaches the worker's `failed` event, so this does not + * see it; the delivery reconciler finds it failed and retries it. + */ + async failExhausted( + job: Job, + error: Error, + channel: DeliveryChannel, + ): Promise { + if (isTransientDeliveryError(error) || (job.data.mailMerge && job.data.batchId)) return + const attemptsMade = job.attemptsMade ?? 0 + if (attemptsMade < maxAttempts(job)) return + + const logger = new Logger(channel.context) + const { notifyId, tenantId } = job.data + try { + await this.markFailed(notifyId, tenantId, error.message) + logger.error( + `[${notifyId}] Notification marked as FAILED (delivery job exhausted after ${attemptsMade} attempt(s))`, + ) + } catch (markError) { + logger.error( + `[${notifyId}] Failed to mark notification FAILED after job exhaustion: ${describeError( + markError, + )}`, + ) + } + } + + /** The request FAILED, and every recipient it had not reached. */ + async markFailed(notifyId: string, tenantId: string, errorMessage: string): Promise { + await this.notificationService.update(notifyId, tenantId, { + status: NotificationStatus.FAILED, + updatedBy: 'system', + errorReason: errorMessage, + }) + await this.requestDetailService.markFailed(notifyId, errorMessage) + } +} diff --git a/backend/src/queue/workers/delivery-job.ts b/backend/src/queue/workers/delivery-job.ts new file mode 100644 index 000000000..8c367c86b --- /dev/null +++ b/backend/src/queue/workers/delivery-job.ts @@ -0,0 +1,51 @@ +import { Logger } from '@nestjs/common' +import type { Job } from 'bullmq' +import { DeliveryJobPayload } from '../queue.types' +import { capitalize, DeliveryChannel } from './delivery-channel' +import { StructuredLoggerService } from '../../common/logger' + +/** A delivery job that cannot even say which request it is for is malformed, not retryable. */ +export function assertDeliveryJob({ notifyId, tenantId }: DeliveryJobPayload): void { + if (!notifyId || typeof notifyId !== 'string') { + throw new Error('Invalid delivery job: notifyId is missing or invalid') + } + if (!tenantId || typeof tenantId !== 'string') { + throw new Error('Invalid delivery job: tenantId is missing or invalid') + } +} + +export function logJobCompleted( + job: Job, + queueName: string, + channel: DeliveryChannel, + structuredLogger?: StructuredLoggerService, +): void { + const { notifyId } = job.data + new Logger(channel.context).debug( + `[${notifyId}] ${capitalize(channel.displayName)} delivery job completed`, + ) + structuredLogger?.logQueueOperation('complete', queueName, job.id?.toString(), notifyId, { + channel: channel.channel, + context: channel.context, + }) +} + +export function logJobFailed( + job: Job, + error: Error, + queueName: string, + channel: DeliveryChannel, + structuredLogger?: StructuredLoggerService, +): void { + const { notifyId } = job.data + new Logger(channel.context).error( + `[${notifyId}] ${capitalize(channel.displayName)} delivery job failed (attempt ${job.attemptsMade}/${job.opts.attempts}): error=${error.message}`, + ) + structuredLogger?.logQueueOperation('failed', queueName, job.id?.toString(), notifyId, { + channel: channel.channel, + context: channel.context, + error: error.message, + attemptsMade: job.attemptsMade, + maxAttempts: job.opts.attempts, + }) +} diff --git a/backend/src/queue/workers/delivery-worker.ts b/backend/src/queue/workers/delivery-worker.ts new file mode 100644 index 000000000..5278b52c5 --- /dev/null +++ b/backend/src/queue/workers/delivery-worker.ts @@ -0,0 +1,68 @@ +import { Logger } from '@nestjs/common' +import type { Job } from 'bullmq' +import { DeliveryJobPayload } from '../queue.types' +import { QueueProcessor } from './queue-processor' +import { DeliveryChannel } from './delivery-channel' +import { DeliveryFailureHandler } from './delivery-failure-handler' +import { isMergeBatch, MergeBatchRunner, MergeSender } from './merge-batch-runner' +import { assertDeliveryJob, logJobCompleted, logJobFailed } from './delivery-job' +import type { QueueName } from '../../enum/queue-name.enum' +import { StructuredLoggerService } from '../../common/logger' + +/** + * A delivery job's lifecycle, the same on every channel: validate the job, send a merge batch + * through MergeBatchRunner or a plain send through the channel's own `deliver`, hand any failure + * to DeliveryFailureHandler, and settle a plain send whose retries ran out. A channel supplies only + * how it renders and sends. + */ +export abstract class DeliveryWorker implements QueueProcessor { + abstract readonly queueName: QueueName + abstract readonly concurrency: { configKey: string; default: number } + protected abstract readonly channel: DeliveryChannel + protected abstract readonly logger: Logger + + constructor( + protected readonly mergeBatches: MergeBatchRunner, + protected readonly failures: DeliveryFailureHandler, + protected readonly structuredLogger?: StructuredLoggerService, + ) {} + + async process(job: Job): Promise { + const { notifyId, tenantId } = job.data + const startedAt = Date.now() + this.logger.debug( + `[${notifyId}] Processing ${this.channel.displayName} delivery job for tenant=${tenantId}`, + ) + + try { + assertDeliveryJob(job.data) + if (isMergeBatch(job.data)) { + return await this.mergeBatches.run(job, await this.mergeSender(job)) + } + this.structuredLogger?.logNotificationStart( + notifyId, + tenantId, + this.channel.channel, + this.channel.context, + ) + return await this.deliver(job, startedAt) + } catch (error) { + return this.failures.fail(job, error, this.channel, startedAt) + } + } + + onCompleted(job: Job): void { + logJobCompleted(job, this.queueName, this.channel, this.structuredLogger) + } + + async onFailed(job: Job, error: Error): Promise { + logJobFailed(job, error, this.queueName, this.channel, this.structuredLogger) + await this.failures.failExhausted(job, error, this.channel) + } + + /** How this channel renders and sends each recipient of a merge batch. */ + protected abstract mergeSender(job: Job): MergeSender | Promise + + /** Send a plain (non-merge) delivery job. */ + protected abstract deliver(job: Job, startedAt: number): Promise +} diff --git a/backend/src/queue/workers/email-delivery.worker.spec.ts b/backend/src/queue/workers/email-delivery.worker.spec.ts index fb39808d2..63c353fc4 100644 --- a/backend/src/queue/workers/email-delivery.worker.spec.ts +++ b/backend/src/queue/workers/email-delivery.worker.spec.ts @@ -1,17 +1,26 @@ import { BadRequestException, Logger } from '@nestjs/common' import { TransientDeliveryError } from '../../adapters/delivery-errors' -import Bull from 'bull' +import type { Job } from 'bullmq' import { vi, describe, it, expect, beforeEach, afterEach } from 'vitest' +import { UnrecoverableError } from 'bullmq' import { EmailDeliveryWorker } from './email-delivery.worker' import { DeliveryJobPayload } from '../queue.types' import { NotificationChannel } from '../../enum/notification-channel.enum' import { NotificationStatus } from '../../enum/notification-status.enum' import { IEmailTransport } from '../../adapters' import { AttachmentResolverService } from '../../api/notify/services/attachment-resolver.service' -import { StructuredLoggerService } from '../../common/logger' +import { MergeBatchRunner } from './merge-batch-runner' +import { DeliveryFailureHandler } from './delivery-failure-handler' +import type { NotificationService } from '../../api/notification/notification.service' +import type { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' +import type { TemplatesRepository } from '../../api/templates/templates.repository' +import type { TemplatesService } from '../../api/templates/templates.service' +import type { TenantSettingsService } from '../../api/tenant-settings/tenant-settings.service' +import type { InlineRenderingService } from '../../services/rendering/inline-rendering.service' +import type { EventNotificationResolver } from '../../api/events/event-notification.resolver' +import type { StructuredLoggerService } from '../../common/logger' describe('EmailDeliveryWorker', () => { - let mockEmailQueue: Partial> let mockNotificationService: any let mockTemplatesRepository: any let mockTemplatesService: any @@ -26,9 +35,53 @@ describe('EmailDeliveryWorker', () => { | 'logQueueOperation', ReturnType > - let processHandler: (job: Bull.Job) => Promise - let completedCallback: (job: Bull.Job) => void - let failedCallback: (job: Bull.Job, err: Error) => void + let processHandler: (job: Job) => Promise + let completedCallback: (job: Job) => void + let failedCallback: (job: Job, err: Error) => void + + /** The worker as QueueModule builds it, with its handlers captured for the tests to call. */ + function initialize(deps: { + notificationService: NotificationService + templatesRepository: TemplatesRepository + templatesService: TemplatesService + inlineRenderingService: InlineRenderingService + attachmentResolverService: AttachmentResolverService + emailAdapter: IEmailTransport + requestDetailService: NotificationRequestDetailService + concurrency?: number + structuredLogger?: StructuredLoggerService + tenantSettingsService?: TenantSettingsService + eventNotificationResolver?: EventNotificationResolver + }) { + const mergeBatches = new MergeBatchRunner( + deps.notificationService, + deps.requestDetailService, + deps.templatesRepository, + deps.structuredLogger, + ) + const worker = new EmailDeliveryWorker( + deps.notificationService, + deps.requestDetailService, + deps.templatesRepository, + deps.templatesService, + deps.inlineRenderingService, + deps.attachmentResolverService, + deps.emailAdapter, + mergeBatches, + new DeliveryFailureHandler( + deps.notificationService, + deps.requestDetailService, + mergeBatches, + deps.structuredLogger, + ), + deps.structuredLogger, + deps.tenantSettingsService, + deps.eventNotificationResolver, + ) + processHandler = (job) => worker.process(job) + completedCallback = (job) => worker.onCompleted(job) + failedCallback = (job, err) => worker.onFailed(job, err) + } beforeEach(() => { mockEmailAdapter = { @@ -48,7 +101,7 @@ describe('EmailDeliveryWorker', () => { markRecipientSent: vi.fn().mockResolvedValue(undefined), markRecipientFailed: vi.fn().mockResolvedValue(undefined), markBatchUnsentFailed: vi.fn().mockResolvedValue(undefined), - countByStatus: vi.fn().mockResolvedValue(0), + countByStatuses: vi.fn().mockResolvedValue({}), findSentAddresses: vi.fn().mockResolvedValue(new Set()), findBatchRecipients: vi.fn().mockResolvedValue([]), } @@ -85,21 +138,6 @@ describe('EmailDeliveryWorker', () => { resolveEmailAttachments: vi.fn().mockResolvedValue(undefined), } - mockEmailQueue = { - process: vi.fn().mockImplementation((...args) => { - const handler = typeof args[0] === 'function' ? args[0] : args[1] - processHandler = handler - return Promise.resolve() - }), - on: vi.fn().mockImplementation((event, callback) => { - if (event === 'completed') { - completedCallback = callback - } else if (event === 'failed') { - failedCallback = callback - } - }), - } - vi.spyOn(Logger.prototype, 'debug').mockImplementation(() => {}) vi.spyOn(Logger.prototype, 'log').mockImplementation(() => {}) vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => {}) @@ -111,40 +149,8 @@ describe('EmailDeliveryWorker', () => { }) describe('initialize', () => { - it('should register a process handler on the email queue', () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, - notificationService: mockNotificationService, - templatesRepository: mockTemplatesRepository, - templatesService: mockTemplatesService, - inlineRenderingService: mockInlineRenderingService, - attachmentResolverService: mockAttachmentResolverService as AttachmentResolverService, - emailAdapter: mockEmailAdapter, - requestDetailService: mockRequestDetailService, - }) - - expect(mockEmailQueue.process).toHaveBeenCalled() - }) - - it('should register event listeners on the email queue', () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, - notificationService: mockNotificationService, - templatesRepository: mockTemplatesRepository, - templatesService: mockTemplatesService, - inlineRenderingService: mockInlineRenderingService, - attachmentResolverService: mockAttachmentResolverService as AttachmentResolverService, - emailAdapter: mockEmailAdapter, - requestDetailService: mockRequestDetailService, - }) - - expect(mockEmailQueue.on).toHaveBeenCalledWith('completed', expect.any(Function)) - expect(mockEmailQueue.on).toHaveBeenCalledWith('failed', expect.any(Function)) - }) - it('should process email delivery job successfully', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -154,7 +160,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', tenantId: 'tenant-123', @@ -176,7 +182,7 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result).toEqual({ success: true, @@ -207,8 +213,7 @@ describe('EmailDeliveryWorker', () => { getSenderAddress: vi.fn().mockResolvedValue('permits@gov.bc.ca'), } - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -233,7 +238,7 @@ describe('EmailDeliveryWorker', () => { attempt: 0, }, attemptsMade: 0, - } as Bull.Job) + } as Job) expect(mockTenantSettingsService.getSenderAddress).toHaveBeenCalledWith('tenant-123') expect(mockEmailAdapter.send).toHaveBeenCalledWith( @@ -246,8 +251,7 @@ describe('EmailDeliveryWorker', () => { getSenderAddress: vi.fn().mockResolvedValue(null), } - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -272,7 +276,7 @@ describe('EmailDeliveryWorker', () => { attempt: 0, }, attemptsMade: 0, - } as Bull.Job) + } as Job) // Absent, not undefined: the adapter falls back to `ches.from` only when the key is missing. const [sent] = mockEmailAdapter.send.mock.calls.at(-1) @@ -280,8 +284,7 @@ describe('EmailDeliveryWorker', () => { }) it('brands inline emails before handing them to the adapter', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -308,7 +311,7 @@ describe('EmailDeliveryWorker', () => { attempt: 0, }, attemptsMade: 0, - } as Bull.Job) + } as Job) expect(mockTemplatesService.applyEmailLayout).toHaveBeenCalledWith( { tenantId: 'tenant-123', channelCode: 'EMAIL' }, { subject: 'Inline', body: 'Hello', bodyType: 'text' }, @@ -336,8 +339,7 @@ describe('EmailDeliveryWorker', () => { headerTitle: 'Permits', }), } - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -362,7 +364,7 @@ describe('EmailDeliveryWorker', () => { attempt: 0, }, attemptsMade: 0, - } as Bull.Job) + } as Job) expect(mockTemplatesService.applyEmailLayout).toHaveBeenCalledWith( { tenantId: 'tenant-123', channelCode: 'EMAIL' }, @@ -372,8 +374,7 @@ describe('EmailDeliveryWorker', () => { }) it('should handle multiple recipients', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -383,7 +384,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-456', tenantId: 'tenant-456', @@ -409,15 +410,14 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result.success).toBe(true) expect(mockNotificationService.update).toHaveBeenCalledTimes(2) }) it('should throw error when email payload is missing', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -427,7 +427,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', tenantId: 'tenant-123', @@ -445,7 +445,7 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid delivery job: email payload is missing or invalid', ) @@ -454,8 +454,7 @@ describe('EmailDeliveryWorker', () => { }) it('should throw error when recipient email is missing', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -465,7 +464,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', @@ -487,14 +486,13 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid email payload: recipient email address is missing or invalid', ) }) it('should throw error when email subject is missing', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -504,7 +502,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', @@ -527,14 +525,13 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid email payload: subject is missing or invalid', ) }) it('should throw error when email body is missing', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -544,7 +541,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', @@ -567,14 +564,13 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid email payload: body is missing or invalid', ) }) it('should resolve a template-only email whose content has no inline subject/body', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -602,7 +598,7 @@ describe('EmailDeliveryWorker', () => { bodyType: 'html', }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-tmpl-only', tenantId: 'tenant-123', @@ -623,7 +619,7 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result.success).toBe(true) expect(mockTemplatesRepository.findById).toHaveBeenCalledWith('tenant-123', 'template-uuid') @@ -651,8 +647,7 @@ describe('EmailDeliveryWorker', () => { }) it('should merge channel-level params over global params when rendering a template email', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -673,7 +668,7 @@ describe('EmailDeliveryWorker', () => { bodyType: 'html', }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-tmpl-params', tenantId: 'tenant-123', @@ -695,7 +690,7 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result.success).toBe(true) // Channel-level param wins per-key; non-overlapping global param is retained. @@ -707,8 +702,7 @@ describe('EmailDeliveryWorker', () => { }) it('should merge channel-level params over global params when rendering inline content', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -723,7 +717,7 @@ describe('EmailDeliveryWorker', () => { body: 'Rendered body', }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-inline-params', tenantId: 'tenant-123', @@ -749,7 +743,7 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result.success).toBe(true) expect(mockTemplatesService.renderTemplateContent).not.toHaveBeenCalled() @@ -770,8 +764,7 @@ describe('EmailDeliveryWorker', () => { new BadRequestException('Missing personalisation for template ID template-uuid: firstName'), ) - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -782,7 +775,7 @@ describe('EmailDeliveryWorker', () => { structuredLogger: mockStructuredLogger as unknown as StructuredLoggerService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-template-missing', tenantId: 'tenant-123', @@ -798,15 +791,16 @@ describe('EmailDeliveryWorker', () => { } as DeliveryJobPayload, opts: { attempts: 3 } as any, attemptsMade: 0, - discard: vi.fn(), } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + const failure = processHandler(job as Job) + await expect(failure).rejects.toThrow( 'Missing personalisation for template ID template-uuid: firstName', ) expect(mockEmailAdapter.send).not.toHaveBeenCalled() - expect(job.discard).toHaveBeenCalledTimes(1) + // Failed outright rather than retried. + await expect(failure).rejects.toBeInstanceOf(UnrecoverableError) expect(mockNotificationService.update).toHaveBeenCalledWith( 'notify-template-missing', 'tenant-123', @@ -830,8 +824,7 @@ describe('EmailDeliveryWorker', () => { ) }) it('should throw error when notifyId is missing', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -841,7 +834,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: undefined as any, @@ -864,14 +857,13 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid delivery job: notifyId is missing or invalid', ) }) it('should throw error when tenantId is missing', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -881,7 +873,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', @@ -904,14 +896,13 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid delivery job: tenantId is missing or invalid', ) }) it('should throw error when attempt is invalid', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -921,7 +912,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', @@ -945,14 +936,13 @@ describe('EmailDeliveryWorker', () => { } // Note: attempt field in payload is not validated/used by worker - // Worker uses job.attemptsMade from Bull queue metadata instead - const result = await processHandler(job as Bull.Job) + // Worker uses job.attemptsMade from the job's queue metadata instead + const result = await processHandler(job as Job) expect(result.success).toBe(true) }) it('should mark notification as FAILED on final attempt (attempt 2)', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -965,7 +955,7 @@ describe('EmailDeliveryWorker', () => { // Simulate final attempt mockNotificationService.update.mockRejectedValueOnce(new Error('DB Error')) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-fail', @@ -988,7 +978,7 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow('DB Error') + await expect(processHandler(job as Job)).rejects.toThrow('DB Error') // Verify we attempted the first update to SENDING expect(mockNotificationService.update).toHaveBeenCalledWith( @@ -1001,8 +991,7 @@ describe('EmailDeliveryWorker', () => { }) it('leaves a plain send owed after a CHES outage outlasts every attempt', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -1013,7 +1002,7 @@ describe('EmailDeliveryWorker', () => { }) const outage = new TransientDeliveryError('CHES email: upstream 503', 502) vi.mocked(mockEmailAdapter.send).mockRejectedValue(outage) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-outage', tenantId: 'tenant-1', @@ -1029,8 +1018,8 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 2, } - await expect(processHandler(job as Bull.Job)).rejects.toBe(outage) - await failedCallback({ ...job, attemptsMade: 3 } as Bull.Job, outage) + await expect(processHandler(job as Job)).rejects.toBe(outage) + await failedCallback({ ...job, attemptsMade: 3 } as Job, outage) expect(mockRequestDetailService.markFailed).not.toHaveBeenCalled() expect(mockNotificationService.update).not.toHaveBeenCalledWith( @@ -1041,8 +1030,7 @@ describe('EmailDeliveryWorker', () => { }) it('should NOT mark notification as FAILED on non-final attempts', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -1052,7 +1040,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-retry', @@ -1084,7 +1072,7 @@ describe('EmailDeliveryWorker', () => { throw new Error('Network error - should retry') }) - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Network error - should retry', ) @@ -1107,8 +1095,7 @@ describe('EmailDeliveryWorker', () => { }) it('should call completed event listener on job success', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -1118,7 +1105,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', @@ -1141,18 +1128,17 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - await processHandler(job as Bull.Job) + await processHandler(job as Job) // Manually trigger the completed event - completedCallback(job as Bull.Job) + completedCallback(job as Job) // Just verify the event listener was registered and can be called expect(completedCallback).toBeDefined() }) it('should call failed event listener on job failure', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -1162,7 +1148,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-fail', @@ -1183,18 +1169,17 @@ describe('EmailDeliveryWorker', () => { const error = new Error('Test error') - await expect(processHandler(job as Bull.Job)).rejects.toThrow() + await expect(processHandler(job as Job)).rejects.toThrow() // Manually trigger the failed event - failedCallback(job as Bull.Job, error) + failedCallback(job as Job, error) // Just verify the event listener was registered and can be called expect(failedCallback).toBeDefined() }) it('should update status to COMPLETED with correct updatedBy', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -1204,7 +1189,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', @@ -1227,7 +1212,7 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - await processHandler(job as Bull.Job) + await processHandler(job as Job) const completedCall = mockNotificationService.update.mock.calls.find( (call: any[]) => call[2]?.status === NotificationStatus.COMPLETED, @@ -1239,8 +1224,7 @@ describe('EmailDeliveryWorker', () => { it('should log with notifyId for end-to-end tracing', async () => { const debugSpy = vi.spyOn(Logger.prototype, 'debug') - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -1250,7 +1234,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', @@ -1273,7 +1257,7 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - await processHandler(job as Bull.Job) + await processHandler(job as Job) // Verify logs include notifyId prefix const logCalls = debugSpy.mock.calls @@ -1286,8 +1270,7 @@ describe('EmailDeliveryWorker', () => { }) it('should handle email with CC and BCC', async () => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -1297,7 +1280,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', @@ -1324,35 +1307,12 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result.success).toBe(true) expect(mockNotificationService.update).toHaveBeenCalledTimes(2) }) - it('should use correct concurrency level', () => { - const initSpy = vi.spyOn(EmailDeliveryWorker, 'initialize' as any) - - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, - notificationService: mockNotificationService, - templatesRepository: mockTemplatesRepository, - templatesService: mockTemplatesService, - inlineRenderingService: mockInlineRenderingService, - attachmentResolverService: mockAttachmentResolverService as AttachmentResolverService, - emailAdapter: mockEmailAdapter, - requestDetailService: mockRequestDetailService, - concurrency: 5, - }) - - expect(mockEmailQueue.process).toHaveBeenCalled() - // Verify that concurrency was passed (it's in the args) - const processArgs = (mockEmailQueue.process as any).mock.calls[0] - expect(processArgs[0]).toBe(5) - - initSpy.mockRestore() - }) - it('should resolve stored attachments and pass adapter-ready attachments to the email adapter', async () => { const content = Buffer.from('hello world') mockAttachmentResolverService.resolveEmailAttachments.mockResolvedValue([ @@ -1364,8 +1324,7 @@ describe('EmailDeliveryWorker', () => { }, ]) - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -1375,7 +1334,7 @@ describe('EmailDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-attachments', tenantId: 'tenant-123', @@ -1396,7 +1355,7 @@ describe('EmailDeliveryWorker', () => { attemptsMade: 0, } - await processHandler(job as Bull.Job) + await processHandler(job as Job) expect(mockAttachmentResolverService.resolveEmailAttachments).toHaveBeenCalledTimes(1) expect(mockEmailAdapter.send).toHaveBeenCalledWith( @@ -1424,8 +1383,7 @@ describe('EmailDeliveryWorker', () => { }, ]) - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -1454,7 +1412,7 @@ describe('EmailDeliveryWorker', () => { delay: 2000, }, } as any, - } as Bull.Job) + } as Job) expect(mockAttachmentResolverService.resolveEmailAttachments).toHaveBeenCalledWith( 'tenant-123', @@ -1479,8 +1437,7 @@ describe('EmailDeliveryWorker', () => { new Error('Failed to download attachment'), ) - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -1506,7 +1463,7 @@ describe('EmailDeliveryWorker', () => { } as any, opts: { attempts: 3 } as any, attemptsMade: 2, - } as Bull.Job), + } as Job), ).rejects.toThrow('Failed to download attachment') expect(mockEmailAdapter.send).not.toHaveBeenCalled() @@ -1535,8 +1492,7 @@ describe('EmailDeliveryWorker', () => { }) vi.mocked(mockEmailAdapter.send).mockResolvedValue({ messageId: 'ext-123' }) - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -1551,7 +1507,7 @@ describe('EmailDeliveryWorker', () => { function makeBulkJob( addresses: string[], overrides: Partial = {}, - ): Partial> { + ): Partial> { return { data: { notifyId: 'notify-bulk', @@ -1577,7 +1533,7 @@ describe('EmailDeliveryWorker', () => { it('should send to each address and mark recipients as sent', async () => { const job = makeBulkJob(['alice@example.com', 'bob@example.com']) - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result).toEqual({ success: true, @@ -1605,9 +1561,12 @@ describe('EmailDeliveryWorker', () => { .mockResolvedValueOnce({ messageId: 'ext-1' }) .mockRejectedValueOnce(new Error('rejected')) const progress = vi.fn().mockResolvedValue(undefined) - const job = { ...makeBulkJob(['alice@example.com', 'bob@example.com']), progress } + const job = { + ...makeBulkJob(['alice@example.com', 'bob@example.com']), + updateProgress: progress, + } - await processHandler(job as unknown as Bull.Job) + await processHandler(job as unknown as Job) await new Promise((resolve) => setTimeout(resolve, 0)) expect(progress.mock.calls.map(([value]) => value)).toEqual([ @@ -1621,9 +1580,12 @@ describe('EmailDeliveryWorker', () => { // A pod replaced mid-batch leaves the job to be retried from the top. mockRequestDetailService.findSentAddresses.mockResolvedValue(new Set(['alice@example.com'])) const progress = vi.fn().mockResolvedValue(undefined) - const job = { ...makeBulkJob(['alice@example.com', 'bob@example.com']), progress } + const job = { + ...makeBulkJob(['alice@example.com', 'bob@example.com']), + updateProgress: progress, + } - const result = await processHandler(job as unknown as Bull.Job) + const result = await processHandler(job as unknown as Job) await new Promise((resolve) => setTimeout(resolve, 0)) expect(mockRequestDetailService.findSentAddresses).toHaveBeenCalledWith( @@ -1653,7 +1615,7 @@ describe('EmailDeliveryWorker', () => { const job = makeBulkJob([]) delete (job.data!.mailMergeData as { recipients?: unknown }).recipients - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(mockRequestDetailService.findBatchRecipients).toHaveBeenCalledWith( 'notify-bulk', @@ -1682,7 +1644,7 @@ describe('EmailDeliveryWorker', () => { }, } as Partial) - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result).toMatchObject({ success: true, sent: 1, failed: 0 }) // Template path not used; inline renderer is invoked with merged params (handlebars default) @@ -1708,8 +1670,7 @@ describe('EmailDeliveryWorker', () => { }), } - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -1727,7 +1688,7 @@ describe('EmailDeliveryWorker', () => { const job = makeBulkJob(['alice@example.com', 'bob@example.com']) - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result).toMatchObject({ success: true, sent: 2, failed: 0 }) // An event test send is addressed by merge rows, so the event's own logo and title have @@ -1754,7 +1715,7 @@ describe('EmailDeliveryWorker', () => { }, } as Partial) - await processHandler(job as Bull.Job) + await processHandler(job as Job) expect(mockTemplatesService.applyEmailLayout).toHaveBeenCalledWith( { tenantId: 'tenant-bulk', channelCode: 'EMAIL' }, @@ -1765,14 +1726,15 @@ describe('EmailDeliveryWorker', () => { it('should mark parent COMPLETED when all recipients sent and no pending remain', async () => { // pending=0, failed=0, sent=2 - mockRequestDetailService.countByStatus - .mockResolvedValueOnce(0) - .mockResolvedValueOnce(0) - .mockResolvedValueOnce(2) + mockRequestDetailService.countByStatuses.mockResolvedValueOnce({ + pending: 0, + failed: 0, + sent: 2, + }) const job = makeBulkJob(['alice@example.com']) - await processHandler(job as Bull.Job) + await processHandler(job as Job) expect(mockNotificationService.update).toHaveBeenCalledWith('notify-bulk', 'tenant-bulk', { status: NotificationStatus.COMPLETED, @@ -1787,17 +1749,18 @@ describe('EmailDeliveryWorker', () => { .mockRejectedValueOnce(new Error('SMTP timeout')) // pending=0, failed=1, sent=1 - mockRequestDetailService.countByStatus - .mockResolvedValueOnce(0) - .mockResolvedValueOnce(1) - .mockResolvedValueOnce(1) + mockRequestDetailService.countByStatuses.mockResolvedValueOnce({ + pending: 0, + failed: 1, + sent: 1, + }) const job = makeBulkJob(['alice@example.com', 'bob@example.com']) - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result).toEqual({ - success: true, + success: false, batchId: 'notify-bulk-EMAIL-0', sent: 1, failed: 1, @@ -1826,17 +1789,18 @@ describe('EmailDeliveryWorker', () => { vi.mocked(mockEmailAdapter.send).mockRejectedValue(new Error('SMTP down')) // pending=0, failed=2, sent=0 - mockRequestDetailService.countByStatus - .mockResolvedValueOnce(0) - .mockResolvedValueOnce(2) - .mockResolvedValueOnce(0) + mockRequestDetailService.countByStatuses.mockResolvedValueOnce({ + pending: 0, + failed: 2, + sent: 0, + }) const job = makeBulkJob(['alice@example.com', 'bob@example.com']) - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result).toEqual({ - success: true, + success: false, batchId: 'notify-bulk-EMAIL-0', sent: 0, failed: 2, @@ -1859,11 +1823,11 @@ describe('EmailDeliveryWorker', () => { it('should not update parent status when other batches are still pending', async () => { // pending=1 → other batches still in progress, no final status set - mockRequestDetailService.countByStatus.mockResolvedValue(1) + mockRequestDetailService.countByStatuses.mockResolvedValue({ pending: 1 }) const job = makeBulkJob(['alice@example.com']) - await processHandler(job as Bull.Job) + await processHandler(job as Job) expect(mockNotificationService.update).not.toHaveBeenCalled() expect(mockStructuredLogger.logNotificationFailure).not.toHaveBeenCalled() @@ -1874,7 +1838,7 @@ describe('EmailDeliveryWorker', () => { const job = makeBulkJob(['alice@example.com']) - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( "Template 'template-uuid' not found for tenant 'tenant-bulk'", ) expect(mockEmailAdapter.send).not.toHaveBeenCalled() @@ -1885,7 +1849,7 @@ describe('EmailDeliveryWorker', () => { const job = makeBulkJob(['alice@example.com']) - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( "Template 'template-uuid' is not an EMAIL template", ) expect(mockEmailAdapter.send).not.toHaveBeenCalled() @@ -1898,7 +1862,7 @@ describe('EmailDeliveryWorker', () => { const job = makeBulkJob(['alice@example.com', 'bob@example.com', 'carol@example.com']) - await expect(processHandler(job as Bull.Job)).rejects.toBeInstanceOf( + await expect(processHandler(job as Job)).rejects.toBeInstanceOf( TransientDeliveryError, ) expect(mockRequestDetailService.markRecipientSent).toHaveBeenCalledTimes(1) @@ -1913,9 +1877,9 @@ describe('EmailDeliveryWorker', () => { vi.mocked(mockEmailAdapter.send).mockRejectedValue( new TransientDeliveryError('CHES email: upstream 503', 502), ) - const job = { ...makeBulkJob(['alice@example.com']), attemptsMade: 2, discard: vi.fn() } + const job = { ...makeBulkJob(['alice@example.com']), attemptsMade: 2 } - await expect(processHandler(job as Bull.Job)).rejects.toThrow('503') + await expect(processHandler(job as Job)).rejects.toThrow('503') expect(mockRequestDetailService.markBatchUnsentFailed).not.toHaveBeenCalled() expect(mockRequestDetailService.markFailed).not.toHaveBeenCalled() }) @@ -1923,15 +1887,14 @@ describe('EmailDeliveryWorker', () => { it('fails only this batch, never the whole send, once a systemic error exhausts retries', async () => { mockTemplatesRepository.findById.mockResolvedValue(null) // pending=0, failed=1, sent=24 once this batch is settled - mockRequestDetailService.countByStatus - .mockResolvedValueOnce(0) - .mockResolvedValueOnce(1) - .mockResolvedValueOnce(24) - const job = { ...makeBulkJob(['alice@example.com']), attemptsMade: 2, discard: vi.fn() } - - await expect(processHandler(job as Bull.Job)).rejects.toThrow( - 'not found', - ) + mockRequestDetailService.countByStatuses.mockResolvedValueOnce({ + pending: 0, + failed: 1, + sent: 24, + }) + const job = { ...makeBulkJob(['alice@example.com']), attemptsMade: 2 } + + await expect(processHandler(job as Job)).rejects.toThrow('not found') expect(mockRequestDetailService.markBatchUnsentFailed).toHaveBeenCalledWith( 'notify-bulk', 'notify-bulk-EMAIL-0', @@ -1956,7 +1919,7 @@ describe('EmailDeliveryWorker', () => { mockTemplatesRepository.findById.mockResolvedValue(null) const job = makeBulkJob(['alice@example.com']) - await expect(processHandler(job as Bull.Job)).rejects.toThrow() + await expect(processHandler(job as Job)).rejects.toThrow() expect(mockRequestDetailService.markBatchUnsentFailed).not.toHaveBeenCalled() }) @@ -1964,7 +1927,7 @@ describe('EmailDeliveryWorker', () => { const job = { ...makeBulkJob(['alice@example.com']), attemptsMade: 3 } await failedCallback( - job as Bull.Job, + job as Job, new Error('job stalled more than allowable limit'), ) @@ -1975,7 +1938,7 @@ describe('EmailDeliveryWorker', () => { it('should resolve template once and render content per recipient', async () => { const job = makeBulkJob(['a@example.com', 'b@example.com', 'c@example.com']) - await processHandler(job as Bull.Job) + await processHandler(job as Job) expect(mockTemplatesRepository.findById).toHaveBeenCalledTimes(1) expect(mockTemplatesService.renderTemplateContent).toHaveBeenCalledTimes(3) @@ -1983,10 +1946,10 @@ describe('EmailDeliveryWorker', () => { }) it('skips addresses an earlier run of the batch already delivered', async () => { mockRequestDetailService.findSentAddresses.mockResolvedValue(new Set(['alice@example.com'])) - // attemptsMade 0: Bull re-runs a stalled job without counting it as a retry. + // attemptsMade 0: BullMQ re-runs a stalled job without counting it as a retry. const job = makeBulkJob(['alice@example.com', 'bob@example.com']) - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(mockRequestDetailService.findSentAddresses).toHaveBeenCalledWith( 'notify-bulk', @@ -2002,8 +1965,7 @@ describe('EmailDeliveryWorker', () => { }) describe('redelivery', () => { beforeEach(() => { - EmailDeliveryWorker.initialize({ - emailQueue: mockEmailQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -2014,9 +1976,9 @@ describe('EmailDeliveryWorker', () => { }) }) - // attemptsMade stays 0 on these jobs: Bull re-runs a stalled job (its pod stopped mid-send) + // attemptsMade stays 0 on these jobs: BullMQ re-runs a stalled job (its pod stopped mid-send) // without counting it as a retry, so the check cannot depend on the attempt count. - function makeJob(attemptsMade = 0): Partial> { + function makeJob(attemptsMade = 0): Partial> { return { data: { notifyId: 'notify-redelivered', @@ -2039,7 +2001,7 @@ describe('EmailDeliveryWorker', () => { new Set(['a@example.com', 'b@example.com']), ) - const result = await processHandler(makeJob() as Bull.Job) + const result = await processHandler(makeJob() as Job) expect(mockEmailAdapter.send).not.toHaveBeenCalled() expect(result).toEqual({ success: true, notifyId: 'notify-redelivered' }) @@ -2052,7 +2014,7 @@ describe('EmailDeliveryWorker', () => { }) it('sends when nothing was delivered yet, without rewriting delivered rows', async () => { - const result = await processHandler(makeJob(1) as Bull.Job) + const result = await processHandler(makeJob(1) as Job) expect(mockRequestDetailService.resetForRetry).toHaveBeenCalledWith('notify-redelivered') expect(mockEmailAdapter.send).toHaveBeenCalledTimes(1) diff --git a/backend/src/queue/workers/email-delivery.worker.ts b/backend/src/queue/workers/email-delivery.worker.ts index 0b6ec0926..b3a6965e0 100644 --- a/backend/src/queue/workers/email-delivery.worker.ts +++ b/backend/src/queue/workers/email-delivery.worker.ts @@ -1,9 +1,11 @@ -import { HttpException, Logger, NotFoundException } from '@nestjs/common' -import Bull from 'bull' -import { DeliveryJobPayload, MailMergeJobData } from '../queue.types' -import { batchProgressReporter, type BatchProgressReporter } from '../batch-progress' -import { loadBatchRecipients } from './merge-batch-recipients' -import { isTransientDeliveryError } from '../../adapters/delivery-errors' +import { Inject, Injectable, Logger, NotFoundException, Optional } from '@nestjs/common' +import type { Job } from 'bullmq' +import { DeliveryJobPayload } from '../queue.types' +import { EMAIL_CHANNEL } from './delivery-channel' +import { DeliveryFailureHandler } from './delivery-failure-handler' +import { MergeBatchRunner, MergeSender } from './merge-batch-runner' +import { DeliveryWorker } from './delivery-worker' +import { QueueName } from '../../enum/queue-name.enum' import { NotificationService } from '../../api/notification/notification.service' import { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' import { TemplatesRepository } from '../../api/templates/templates.repository' @@ -14,12 +16,12 @@ import { NotificationStatus } from '../../enum/notification-status.enum' import { NotifyEmailChannel } from '../../api/notify/schemas/notify-email-channel' import { ProcessedNotifyEmailChannel } from '../../api/notify/schemas/stored-notify-attachment' import { AttachmentResolverService } from '../../api/notify/services/attachment-resolver.service' -import type { +import { EventEmailSendSettings, EventNotificationResolver, } from '../../api/events/event-notification.resolver' import type { EmailHeaderOverride } from '../../api/templates/email-template-layout.service' -import { IEmailTransport } from '../../adapters' +import { EMAIL_ADAPTER, IEmailTransport } from '../../adapters' import { SendEmailOptions } from '../../adapters/interfaces/delivery/email.interface' import { StructuredLoggerService } from '../../common/logger' @@ -27,93 +29,113 @@ type ResolvedEmailDeliveryPayload = Omit & { attachments?: SendEmailOptions['attachments'] } -export interface EmailDeliveryWorkerDeps { - /** The Bull queue instance for email delivery jobs */ - emailQueue: Bull.Queue - /** Service for database updates */ - notificationService: NotificationService - /** Repository for template resolution */ - templatesRepository: TemplatesRepository - /** Service for template rendering */ - templatesService: TemplatesService - /** Service for inline template rendering */ - inlineRenderingService: InlineRenderingService - attachmentResolverService: AttachmentResolverService - /** Email transport adapter for sending emails */ - emailAdapter: IEmailTransport - requestDetailService: NotificationRequestDetailService - /** Number of jobs to process in parallel (default: 2) */ - concurrency?: number - structuredLogger?: StructuredLoggerService - tenantSettingsService?: TenantSettingsService - /** Resolves the sender and header of the event a request came from */ - eventNotificationResolver?: EventNotificationResolver +type EmailPayload = NotifyEmailChannel | ProcessedNotifyEmailChannel | ResolvedEmailDeliveryPayload + +/** Sender and header for every recipient of one job. */ +interface SendSettings { + /** Null leaves the adapter on its own default sender. */ + fromAddress: string | null + headerOverride?: EmailHeaderOverride +} + +function normalizeTemplateBodyType( + bodyType: 'text' | 'markdown' | 'html' | undefined, +): 'markdown' | undefined { + return bodyType === 'markdown' || bodyType === 'text' ? 'markdown' : undefined +} + +function hasAttachmentReferences( + attachments: unknown, +): attachments is NonNullable { + return ( + Array.isArray(attachments) && + attachments.every( + (attachment) => + attachment && + typeof attachment === 'object' && + typeof (attachment as { attachmentId?: unknown }).attachmentId === 'string', + ) + ) +} + +function hasRecipients(payload: EmailPayload): boolean { + const to = payload.recipients?.to + return Array.isArray(to) && to.length > 0 +} + +function assertContent(payload: EmailPayload): void { + if (!payload.content?.subject || typeof payload.content.subject !== 'string') { + throw new Error('Invalid email payload: subject is missing or invalid') + } + if (!payload.content?.body || typeof payload.content.body !== 'string') { + throw new Error('Invalid email payload: body is missing or invalid') + } } /** * Email Delivery Worker * - * Processes email delivery jobs: - * 1. Receives email delivery jobs from the EMAIL_DELIVERY queue - * 2. Updates notification status to SENDING in database - * 3. Gets the appropriate adapter (currently CHES, future: SNS, SendGrid, etc) - * 4. Sends email via adapter - * 5. Updates notification status to COMPLETED on success, FAILED on error - * 6. Implements retry logic with exponential backoff + * A plain send is one email to all of its recipients: render (a server template or inline + * content), brand it with the tenant's layout, resolve stored attachments, send it through the + * email adapter, and mark the request COMPLETED. A merge batch sends each recipient their own + * rendering (MergeBatchRunner). A failure is settled by DeliveryFailureHandler and retried. * - * Idempotency: Job key is notifyId_email, preventing duplicate delivery - * Tracing: All operations logged with notifyId for end-to-end visibility + * Idempotency: the job key is notifyId_email, or the batchId for a merge batch. */ -export class EmailDeliveryWorker { - private readonly logger = new Logger(EmailDeliveryWorker.name) - - private static normalizeTemplateBodyType( - bodyType: 'text' | 'markdown' | 'html' | undefined, - ): 'markdown' | undefined { - if (bodyType === 'markdown' || bodyType === 'text') { - return 'markdown' - } - - return undefined - } - - private static hasAttachmentReferences( - attachments: unknown, - ): attachments is NonNullable { - return ( - Array.isArray(attachments) && - attachments.every( - (attachment) => - attachment && - typeof attachment === 'object' && - typeof (attachment as { attachmentId?: unknown }).attachmentId === 'string', - ) - ) - } - - private static isPermanentValidationError(error: unknown): error is HttpException { - return error instanceof HttpException && error.getStatus() === 400 +@Injectable() +export class EmailDeliveryWorker extends DeliveryWorker { + readonly queueName = QueueName.EMAIL_DELIVERY + readonly concurrency = { configKey: 'emailDeliveryWorkerConcurrency', default: 2 } + protected readonly channel = EMAIL_CHANNEL + protected readonly logger = new Logger(EmailDeliveryWorker.name) + + constructor( + private readonly notificationService: NotificationService, + private readonly requestDetailService: NotificationRequestDetailService, + private readonly templatesRepository: TemplatesRepository, + private readonly templatesService: TemplatesService, + private readonly inlineRenderingService: InlineRenderingService, + private readonly attachmentResolverService: AttachmentResolverService, + @Inject(EMAIL_ADAPTER) private readonly emailAdapter: IEmailTransport, + mergeBatches: MergeBatchRunner, + failures: DeliveryFailureHandler, + @Optional() structuredLogger?: StructuredLoggerService, + @Optional() private readonly tenantSettingsService?: TenantSettingsService, + @Optional() private readonly eventNotificationResolver?: EventNotificationResolver, + ) { + super(mergeBatches, failures, structuredLogger) } /** - * The sender and header of the event this request came from, or null when it came from none. + * The sender and header for this job. An event-sourced send carries its own, which have no home + * on the request payload - the request records which event it came from and they are read back + * here. Every other send uses the tenant's configured sender, or none (the adapter's default). */ - private static async resolveEventSettings( + private async resolveSendSettings(notifyId: string, tenantId: string): Promise { + const eventSettings = await this.resolveEventSettings(notifyId, tenantId) + const fromAddress = + eventSettings?.senderEmail ?? + (await this.tenantSettingsService?.getSenderAddress(tenantId)) ?? + null + const headerOverride = eventSettings?.useCustomHeader + ? { logoId: eventSettings.headerLogoId, title: eventSettings.headerTitle } + : undefined + return { fromAddress, headerOverride } + } + + private async resolveEventSettings( notifyId: string, tenantId: string, - notificationService: NotificationService, - eventNotificationResolver: EventNotificationResolver | undefined, - logger: Logger, ): Promise { - if (!eventNotificationResolver) return null + if (!this.eventNotificationResolver) return null try { - const notification = await notificationService.findOne(notifyId, tenantId) + const notification = await this.notificationService.findOne(notifyId, tenantId) if (!notification?.eventId) return null - return await eventNotificationResolver.findEmailSendSettings(notification.eventId) + return await this.eventNotificationResolver.findEmailSendSettings(notification.eventId) } catch (error) { - logger.warn( + this.logger.warn( `[${notifyId}] Could not read event send settings, falling back to the tenant's: ${ error instanceof Error ? error.message : String(error) }`, @@ -122,731 +144,279 @@ export class EmailDeliveryWorker { } } - private static toHeaderOverride( - settings: EventEmailSendSettings | null, - ): EmailHeaderOverride | undefined { - if (!settings?.useCustomHeader) return undefined - - return { logoId: settings.headerLogoId, title: settings.headerTitle } - } - - /** Initialize the email delivery worker on a queue */ - static initialize({ - emailQueue, - notificationService, - templatesRepository, - templatesService, - inlineRenderingService, - attachmentResolverService, - emailAdapter, - requestDetailService, - concurrency = 2, - structuredLogger, - tenantSettingsService, - eventNotificationResolver, - }: EmailDeliveryWorkerDeps): void { - const logger = new Logger(EmailDeliveryWorker.name) - const workerContext = EmailDeliveryWorker.name - - logger.log(`Registering email delivery worker processor (concurrency=${concurrency})`) - - // Register the job processor with configurable concurrency - // Note: Don't await process() - it sets up listeners and never resolves - emailQueue.process(concurrency, async (job: Bull.Job) => { - const { notifyId, tenantId, payload, request } = job.data - const startedAt = Date.now() - - logger.debug(`[${notifyId}] Processing email delivery job for tenant=${tenantId}`) - - try { - // Validate DeliveryJobPayload structure - if (!notifyId || typeof notifyId !== 'string') { - throw new Error('Invalid delivery job: notifyId is missing or invalid') - } - if (!tenantId || typeof tenantId !== 'string') { - throw new Error('Invalid delivery job: tenantId is missing or invalid') - } - - // An event-sourced send carries its own sender and header, which have no home on the - // request payload - the request records which event it came from and they are read back - // here. Null for every other send, which leaves the tenant's own settings in place. - // - // Resolved before the merge branch below because an event test send can be a merge: the - // test send screen takes merge rows, so a merge does come from an event and needs the - // event's header as much as a single send does. That costs one read per batch, which is - // a read per 100 recipients rather than per recipient. - const eventSettings = await EmailDeliveryWorker.resolveEventSettings( - notifyId, - tenantId, - notificationService, - eventNotificationResolver, - logger, - ) - - // The tenant's configured sender, resolved once for every recipient this job sends to. - // Null when unset - or when the service is absent, as it is in a test context that boots - // the worker without the settings module - which leaves the adapter on `ches.from`. - const fromAddress = - eventSettings?.senderEmail ?? - (await tenantSettingsService?.getSenderAddress(tenantId)) ?? - null - const headerOverride = EmailDeliveryWorker.toHeaderOverride(eventSettings) - - // Mail merge batch: resolve the template once, then render + send per recipient - // individually. - if (job.data.mailMerge && job.data.mailMergeData && job.data.batchId) { - return await EmailDeliveryWorker.processMailMergeBatch( - job.data.batchId, - job.data.mailMergeData, - notifyId, - tenantId, - logger, - templatesRepository, - templatesService, - inlineRenderingService, - emailAdapter, - requestDetailService, - notificationService, - fromAddress, - headerOverride, - batchProgressReporter(job), - structuredLogger, - ) - } - - // Single delivery path: emit a structured lifecycle "start" event. - structuredLogger?.logNotificationStart(notifyId, tenantId, 'email', workerContext) - - // Validate job data - if (!payload || typeof payload !== 'object') { - throw new Error('Invalid delivery job: email payload is missing or invalid') - } - - // Cast payload to email channel type for type safety - let emailPayload = payload as - | NotifyEmailChannel - | ProcessedNotifyEmailChannel - | ResolvedEmailDeliveryPayload - - if ( - !emailPayload.recipients || - !emailPayload.recipients.to || - !Array.isArray(emailPayload.recipients.to) || - emailPayload.recipients.to.length === 0 - ) { - throw new Error('Invalid email payload: recipient email address is missing or invalid') - } - - // Resolve template if the email content carries a templateId. - // Do this BEFORE updating status to SENDING so that errors don't leave notification stuck in SENDING state - const emailTemplateId = emailPayload.content?.templateId - - // Inline subject/body are only required when the content is NOT template-based; a template - // supplies them during resolution below (and they're re-validated after rendering). - if (!emailTemplateId) { - if (!emailPayload.content?.subject || typeof emailPayload.content.subject !== 'string') { - throw new Error('Invalid email payload: subject is missing or invalid') - } - - if (!emailPayload.content?.body || typeof emailPayload.content.body !== 'string') { - throw new Error('Invalid email payload: body is missing or invalid') - } - } - - if ((job.attemptsMade ?? 0) > 0) { - await requestDetailService.resetForRetry(notifyId) - } - - // Checked on every run, not only on a counted retry: Bull re-runs a job whose pod stopped - // mid-send (a rolling deploy, an HPA scale-down) without incrementing attemptsMade. A - // single email is one message to all of its recipients, so it was delivered in full or - // not at all. - const alreadySent = await requestDetailService.findSentAddresses(notifyId) - if ( - alreadySent.size > 0 && - emailPayload.recipients.to.every((address) => alreadySent.has(address)) - ) { - logger.log(`[${notifyId}] Delivered on an earlier attempt; nothing to re-send`) - await notificationService.update(notifyId, tenantId, { - status: NotificationStatus.COMPLETED, - updatedBy: 'system', - }) - return { success: true, notifyId } - } - - if (emailTemplateId) { - logger.debug(`[${notifyId}] Resolving template: ${emailTemplateId}`) - try { - const template = await templatesRepository.findById(tenantId, emailTemplateId) - if (!template) { - throw new NotFoundException( - `Template '${emailTemplateId}' not found for tenant '${tenantId}'`, - ) - } - - // Merge template content into email payload if channel matches - if (template.channelCode === 'EMAIL') { - // Render the template with personalisation data from request.params - // Normalize legacy body types before entering the markdown-only render path. - const renderedTemplate = await templatesService.renderTemplateContent( - template, - { ...request?.params, ...emailPayload.params }, - EmailDeliveryWorker.normalizeTemplateBodyType(emailPayload.content?.bodyType), - ) - const rendered = await templatesService.applyEmailLayout( - template, - renderedTemplate, - headerOverride, - ) - - emailPayload = { - ...emailPayload, - content: { - ...emailPayload.content, - subject: rendered.subject || emailPayload.content?.subject, - body: rendered.body, - bodyType: rendered.bodyType, - }, - } - logger.debug(`[${notifyId}] Template resolved and rendered into email payload`) - } - } catch (templateError) { - logger.error( - `[${notifyId}] Failed to resolve template: ${(templateError as Error).message}`, + /** + * Render, brand and send each merge recipient their own email. An event test send can be a + * merge, and needs the event's sender and header as much as a single send does - one read per + * batch, not per recipient. + */ + protected async mergeSender(job: Job): Promise { + const { notifyId, tenantId } = job.data + const settings = await this.resolveSendSettings(notifyId, tenantId) + return { + channel: EMAIL_CHANNEL, + hasInlineContent: (content) => !!(content && (content.subject || content.body)), + send: async (recipient, params, { template, inline }) => { + const content = template + ? await this.templatesService.applyEmailLayout( + template, + await this.templatesService.renderTemplateContent(template, params), + settings.headerOverride, ) - throw templateError - } - } else if (emailPayload.content?.renderer) { - // Handle inline rendering if renderer is specified and no templateId - logger.debug( - `[${notifyId}] Rendering inline content with renderer: ${emailPayload.content.renderer}`, - ) - try { - const rendered = await inlineRenderingService.renderEmail(emailPayload.content, { - ...request?.params, - ...emailPayload.params, - }) - - emailPayload = { - ...emailPayload, - content: { - ...emailPayload.content, - subject: rendered.subject, - body: rendered.body, + : await this.templatesService.applyEmailLayout( + { tenantId, channelCode: 'EMAIL' }, + { + ...(await this.inlineRenderingService.renderEmail(inline, params)), + bodyType: inline.bodyType ?? 'text', }, - } - logger.debug(`[${notifyId}] Inline content rendered successfully`) - } catch (renderError) { - logger.error( - `[${notifyId}] Failed to render inline content: ${(renderError as Error).message}`, - ) - throw renderError - } - } - - if ( - !emailPayload.recipients || - !emailPayload.recipients.to || - !Array.isArray(emailPayload.recipients.to) || - emailPayload.recipients.to.length === 0 - ) { - throw new Error('Invalid email payload: recipient email address is missing or invalid') - } - - // Template sends are branded above; inline and pre-rendered sends need the same layout. - if (!emailTemplateId) { - const branded = await templatesService.applyEmailLayout( - { tenantId, channelCode: 'EMAIL' }, - { - subject: emailPayload.content?.subject, - body: emailPayload.content?.body, - bodyType: emailPayload.content?.bodyType ?? 'text', - }, - headerOverride, - ) - emailPayload = { ...emailPayload, content: { ...emailPayload.content, ...branded } } - } - - if (!emailPayload.content?.subject || typeof emailPayload.content.subject !== 'string') { - throw new Error('Invalid email payload: subject is missing or invalid') - } - - if (!emailPayload.content?.body || typeof emailPayload.content.body !== 'string') { - throw new Error('Invalid email payload: body is missing or invalid') - } - - if ( - emailPayload.attachments && - !EmailDeliveryWorker.hasAttachmentReferences(emailPayload.attachments) - ) { - throw new Error('Invalid processed email attachment reference payload') - } - - if (EmailDeliveryWorker.hasAttachmentReferences(emailPayload.attachments)) { - if (!attachmentResolverService) { - throw new Error( - 'AttachmentResolverService is not available to resolve stored email attachments', + settings.headerOverride, ) - } - - logger.debug( - `[${notifyId}] Resolving stored email attachments: ${JSON.stringify({ - tenantId, - storedAttachmentCount: emailPayload.attachments.length, - })}`, - ) - - const resolvedAttachments = await attachmentResolverService.resolveEmailAttachments( - tenantId, - emailPayload.attachments, - ) - - emailPayload = { - ...emailPayload, - attachments: resolvedAttachments as SendEmailOptions['attachments'], - } as ResolvedEmailDeliveryPayload - } - - // Update status to SENDING (only after all validations pass) - await notificationService.update(notifyId, tenantId, { - status: NotificationStatus.SENDING, - updatedBy: 'system', - }) - await requestDetailService.updateStatus(notifyId, NotificationStatus.SENDING, { - preserveCompleted: true, - }) - logger.debug(`[${notifyId}] Updated notification status to SENDING`) - - // Send email using the injected adapter - const result = await EmailDeliveryWorker.sendEmail( - emailPayload, - logger, + + const { subject, body, bodyType } = content + const result = await this.sendEmail( + { recipients: { to: [recipient.address] }, content: { subject, body, bodyType } } as any, notifyId, - emailAdapter, - fromAddress, + settings.fromAddress, ) + return result.externalId + }, + } + } - logger.debug(`[${notifyId}] Email sent successfully: ${JSON.stringify(result)}`) + /** A plain send: one email to every recipient on the job. */ + protected async deliver(job: Job, startedAt: number): Promise { + const { notifyId, tenantId } = job.data + const settings = await this.resolveSendSettings(notifyId, tenantId) + const payload = this.validatedPayload(job.data.payload) - // Request has made it to the smtp gateway, update request detail records as sent - await requestDetailService.markSent(notifyId, result.externalId) + if ((job.attemptsMade ?? 0) > 0) { + await this.requestDetailService.resetForRetry(notifyId) + } - // Update status to COMPLETED - await notificationService.update(notifyId, tenantId, { - status: NotificationStatus.COMPLETED, - updatedBy: 'system', - }) - logger.log(`[${notifyId}] Notification marked as COMPLETED`) - structuredLogger?.logNotificationSuccess( - notifyId, - tenantId, - 'email', - result.externalId, - Date.now() - startedAt, - workerContext, - ) + // Checked on every run, not only on a counted retry: a job whose pod stopped mid-send (a + // rolling deploy, an HPA scale-down) is re-run without incrementing attemptsMade. A single + // email is one message to all of its recipients, so it was delivered in full or not at all. + const alreadySent = await this.requestDetailService.findSentAddresses(notifyId) + if ( + alreadySent.size > 0 && + payload.recipients.to.every((address) => alreadySent.has(address)) + ) { + this.logger.log(`[${notifyId}] Delivered on an earlier attempt; nothing to re-send`) + await this.notificationService.update(notifyId, tenantId, { + status: NotificationStatus.COMPLETED, + updatedBy: 'system', + }) + return { success: true, notifyId } + } - return { success: true, externalId: result.externalId, provider: result.provider } - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error) - const attempt = (job.attemptsMade || 0) + 1 - const finalAttempt = (job.attemptsMade || 0) >= (job.opts.attempts || 3) - 1 - logger.error( - `[${notifyId}] Failed to send email delivery job (attempt ${attempt}/3): ${errorMessage}`, - error instanceof Error ? error.stack : '', - ) + // Rendered before SENDING, so a render error does not leave the notification stuck there. + const rendered = await this.render(job.data, payload, settings.headerOverride) + const resolved = await this.resolveAttachments(notifyId, tenantId, rendered) - // CHES could not take it right now. Nothing is marked failed, even on the last attempt: - // the recipients stay owed, and the delivery reconciler re-queues the job once Bull's - // retries are spent, until its own attempts run out. - if (isTransientDeliveryError(error)) throw error - - // A merge batch settles only itself: its sent rows, and every other batch, are not its - // to fail. - const batchId = job.data.mailMerge ? job.data.batchId : undefined - if (batchId) { - if (finalAttempt || EmailDeliveryWorker.isPermanentValidationError(error)) { - await requestDetailService.markBatchUnsentFailed(notifyId, batchId, errorMessage) - await EmailDeliveryWorker.settleMergeRequest( - notifyId, - tenantId, - requestDetailService, - notificationService, - logger, - structuredLogger, - ) - job.discard() - logger.error(`[${notifyId}] Batch ${batchId} failed; its unsent recipients are FAILED`) - } - throw error - } - - if (EmailDeliveryWorker.isPermanentValidationError(error)) { - await notificationService.update(notifyId, tenantId, { - status: NotificationStatus.FAILED, - updatedBy: 'system', - errorReason: errorMessage, - }) - await requestDetailService.markFailed(notifyId, errorMessage) - job.discard() - logger.error( - `[${notifyId}] Notification marked as FAILED after permanent validation error. Error: ${errorMessage}`, - ) - structuredLogger?.logNotificationFailure( - notifyId, - tenantId, - 'email', - error, - Date.now() - startedAt, - workerContext, - ) - throw error - } - - // Update status to FAILED only on final attempt (when no retries left) - if (finalAttempt) { - await notificationService.update(notifyId, tenantId, { - status: NotificationStatus.FAILED, - updatedBy: 'system', - errorReason: errorMessage, - }) - await requestDetailService.markFailed(notifyId, errorMessage) - logger.error( - `[${notifyId}] Notification marked as FAILED after 3 attempts. Error: ${errorMessage}`, - ) - structuredLogger?.logNotificationFailure( - notifyId, - tenantId, - 'email', - error instanceof Error ? error : errorMessage, - Date.now() - startedAt, - workerContext, - ) - } - - // Re-throw to trigger BullMQ retry logic - throw error - } + await this.notificationService.update(notifyId, tenantId, { + status: NotificationStatus.SENDING, + updatedBy: 'system', }) - - // Event listeners for job lifecycle - emailQueue.on('completed', (job: Bull.Job) => { - const { notifyId } = job.data - logger.debug(`[${notifyId}] Email delivery job completed`) - structuredLogger?.logQueueOperation( - 'complete', - emailQueue.name, - job.id?.toString(), - notifyId, - { - channel: 'email', - context: workerContext, - }, - ) + await this.requestDetailService.updateStatus(notifyId, NotificationStatus.SENDING, { + preserveCompleted: true, }) + this.logger.debug(`[${notifyId}] Updated notification status to SENDING`) - emailQueue.on('failed', async (job: Bull.Job, err: Error) => { - const { notifyId, tenantId } = job.data - logger.error( - `[${notifyId}] Email delivery job failed (attempt ${job.attemptsMade}/${job.opts.attempts}): error=${err.message}`, - ) - structuredLogger?.logQueueOperation('failed', emailQueue.name, job.id?.toString(), notifyId, { - channel: 'email', - context: workerContext, - error: err.message, - attemptsMade: job.attemptsMade, - maxAttempts: job.opts.attempts, - }) + const result = await this.sendEmail(resolved, notifyId, settings.fromAddress) + this.logger.debug(`[${notifyId}] Email sent successfully: ${JSON.stringify(result)}`) - // Safety net: once Bull has exhausted all retries the job is permanently failed, so the - // notification must never be left stuck in SENDING. Mark it FAILED here (idempotent) so a - // send error always reaches a terminal status — even if the in-processor final-attempt - // handler didn't run (e.g. a job enqueued without a retry count, or a different worker - // processed the last attempt). - // Not for a transient error - the recipients stay owed for the reconciler - nor for a merge - // batch, which the processor settled itself, scoped to that batch. - if (isTransientDeliveryError(err) || (job.data.mailMerge && job.data.batchId)) return - const attemptsMade = job.attemptsMade ?? 0 - const maxAttempts = job.opts.attempts ?? 1 - if (attemptsMade >= maxAttempts) { - try { - await notificationService.update(notifyId, tenantId, { - status: NotificationStatus.FAILED, - updatedBy: 'system', - errorReason: err.message, - }) - await requestDetailService.markFailed(notifyId, err.message) - logger.error( - `[${notifyId}] Notification marked as FAILED (delivery job exhausted after ${attemptsMade} attempt(s))`, - ) - } catch (markError) { - logger.error( - `[${notifyId}] Failed to mark notification FAILED after job exhaustion: ${ - markError instanceof Error ? markError.message : String(markError) - }`, - ) - } - } + // It has reached the SMTP gateway: the recipients are sent. + await this.requestDetailService.markSent(notifyId, result.externalId) + await this.notificationService.update(notifyId, tenantId, { + status: NotificationStatus.COMPLETED, + updatedBy: 'system', }) - - logger.log('Email delivery worker initialized') - } - - /** - * Process one batch of a mail merge email send. - * - * Resolves the template ONCE and renders it ONCE with the global params (no per-recipient - * personalization), then calls the email adapter once per individual address reusing that - * rendered content. Per-recipient results update that recipient's detail row. Individual - * recipient failures are recorded but do NOT throw, so a batch retry will not re-send - * already-delivered addresses; only systemic errors (e.g. template not found) throw to trigger - * a BullMQ retry of the whole batch. - */ - private static async processMailMergeBatch( - batchId: string, - mailMergeData: MailMergeJobData, - notifyId: string, - tenantId: string, - logger: Logger, - templatesRepository: TemplatesRepository, - templatesService: TemplatesService, - inlineRenderingService: InlineRenderingService, - emailAdapter: IEmailTransport, - requestDetailService: NotificationRequestDetailService, - notificationService: NotificationService, - fromAddress: string | null, - headerOverride: EmailHeaderOverride | undefined, - reportProgress?: BatchProgressReporter, - structuredLogger?: StructuredLoggerService, - ): Promise<{ success: boolean; batchId: string; sent: number; failed: number }> { - const { content, params } = mailMergeData - const { recipients, alreadySent } = await loadBatchRecipients( - requestDetailService, + this.logger.log(`[${notifyId}] Notification marked as COMPLETED`) + this.structuredLogger?.logNotificationSuccess( notifyId, - batchId, - mailMergeData.recipients, + tenantId, + 'email', + result.externalId, + Date.now() - startedAt, + EMAIL_CHANNEL.context, ) - const templateId = content?.templateId - const hasInlineContent = !!(content && (content.subject || content.body)) - // A mail merge send renders from exactly one source: a server template or inline content. - if (templateId && hasInlineContent) { - throw new Error('Mail merge requires either a templateId or inline content, not both') + return { success: true, externalId: result.externalId, provider: result.provider } + } + + private validatedPayload(payload: DeliveryJobPayload['payload']): EmailPayload { + if (!payload || typeof payload !== 'object') { + throw new Error('Invalid delivery job: email payload is missing or invalid') } - if (!templateId && !hasInlineContent) { - throw new Error('Mail merge requires either a templateId or inline content') + const emailPayload = payload as EmailPayload + if (!hasRecipients(emailPayload)) { + throw new Error('Invalid email payload: recipient email address is missing or invalid') } + // A template supplies subject and body when rendered, so only inline content needs them now. + if (!emailPayload.content?.templateId) assertContent(emailPayload) + return emailPayload + } + + /** Render the payload's content - a server template or inline content - and brand it. */ + private async render( + { notifyId, tenantId, request }: DeliveryJobPayload, + payload: EmailPayload, + headerOverride: EmailHeaderOverride | undefined, + ): Promise { + const templateId = payload.content?.templateId + const params = { ...request?.params, ...payload.params } + let rendered = payload - // Resolve the template once for the whole batch (systemic error if missing/wrong channel) - const template = templateId ? await templatesRepository.findById(tenantId, templateId) : null if (templateId) { - if (!template) { - throw new NotFoundException(`Template '${templateId}' not found for tenant '${tenantId}'`) - } - if (template.channelCode !== 'EMAIL') { - throw new Error(`Template '${templateId}' is not an EMAIL template`) + rendered = await this.renderTemplate( + notifyId, + tenantId, + templateId, + payload, + params, + headerOverride, + ) + } else { + if (payload.content?.renderer) { + rendered = await this.renderInline(notifyId, payload, params) } - } - - // Inline content uses handlebars by default so mail-merge variables are substituted per recipient. - const inlineContent = content - ? { ...content, renderer: content.renderer ?? ('handlebars' as const) } - : null - - logger.debug( - `[${notifyId}] Processing mail merge batch ${batchId}: ${recipients.length} recipient(s)`, - ) - - // A retried batch (a pod replaced mid-send, a crash, a provider error) starts again from - // the top. Addresses an earlier attempt delivered keep their rows and are skipped, or they - // would get the email twice. Mirrors SmsDeliveryWorker.processMergeBatch. - const pending = alreadySent.size - ? recipients.filter((recipient) => !alreadySent.has(recipient.address)) - : recipients - - if (alreadySent.size > 0) { - logger.log( - `[${notifyId}] Batch ${batchId}: skipping ${alreadySent.size} recipient(s) delivered on an earlier attempt`, + // Template sends are branded as they render; inline and pre-rendered sends are branded here. + const branded = await this.templatesService.applyEmailLayout( + { tenantId, channelCode: 'EMAIL' }, + { + subject: rendered.content?.subject, + body: rendered.content?.body, + bodyType: rendered.content?.bodyType ?? 'text', + }, + headerOverride, ) + rendered = { ...rendered, content: { ...rendered.content, ...branded } } } - let sent = 0 - let failed = 0 - const progress = () => ({ sent: alreadySent.size + sent, failed, total: recipients.length }) - reportProgress?.(progress()) - - for (const recipient of pending) { - try { - // Merge global params with per-recipient params; per-recipient takes precedence - const mergedParams = { ...(params || {}), ...recipient.params } - - let subject: string | undefined - let body: string - let bodyType: 'text' | 'markdown' | 'html' | undefined - if (template) { - const renderedTemplate = await templatesService.renderTemplateContent( - template, - mergedParams, - ) - const rendered = await templatesService.applyEmailLayout( - template, - renderedTemplate, - headerOverride, - ) - subject = rendered.subject - body = rendered.body - bodyType = rendered.bodyType - } else { - const rendered = await inlineRenderingService.renderEmail(inlineContent!, mergedParams) - const branded = await templatesService.applyEmailLayout( - { tenantId, channelCode: 'EMAIL' }, - { ...rendered, bodyType: inlineContent!.bodyType ?? 'text' }, - headerOverride, - ) - subject = branded.subject - body = branded.body - bodyType = branded.bodyType - } - - const emailPayload = { - recipients: { to: [recipient.address] }, - content: { subject, body, bodyType }, - } - - const result = await EmailDeliveryWorker.sendEmail( - emailPayload as any, - logger, - notifyId, - emailAdapter, - fromAddress, - ) + assertContent(rendered) + return rendered + } - await requestDetailService.markRecipientSent( - notifyId, - batchId, - recipient.address, - result.externalId, - ) - sent++ - } catch (recipientError) { - // CHES, not this recipient: stop the batch and leave the rest owed. The job is retried, - // skipping whoever was sent, and the circuit breaker holds the retry until CHES is back. - if (isTransientDeliveryError(recipientError)) { - logger.warn( - `[${notifyId}] Batch ${batchId} paused after ${sent} sent: ${recipientError.message}`, - ) - throw recipientError - } - const errorMessage = - recipientError instanceof Error ? recipientError.message : String(recipientError) - logger.error( - `[${notifyId}] Mail merge recipient send failed (batch=${batchId}, address=${recipient.address}): ${errorMessage}`, - ) - await requestDetailService.markRecipientFailed( - notifyId, - batchId, - recipient.address, - errorMessage, - ) - failed++ + private async renderTemplate( + notifyId: string, + tenantId: string, + templateId: string, + payload: EmailPayload, + params: Record, + headerOverride: EmailHeaderOverride | undefined, + ): Promise { + this.logger.debug(`[${notifyId}] Resolving template: ${templateId}`) + try { + const template = await this.templatesRepository.findById(tenantId, templateId) + if (!template) { + throw new NotFoundException(`Template '${templateId}' not found for tenant '${tenantId}'`) + } + // A template of another channel leaves the payload's own content to be validated. + if (template.channelCode !== 'EMAIL') return payload + + // Legacy body types are normalized before the markdown-only render path. + const content = await this.templatesService.renderTemplateContent( + template, + params, + normalizeTemplateBodyType(payload.content?.bodyType), + ) + const rendered = await this.templatesService.applyEmailLayout( + template, + content, + headerOverride, + ) + this.logger.debug(`[${notifyId}] Template resolved and rendered into email payload`) + return { + ...payload, + content: { + ...payload.content, + subject: rendered.subject || payload.content?.subject, + body: rendered.body, + bodyType: rendered.bodyType, + }, } - reportProgress?.(progress()) + } catch (error) { + this.logger.error(`[${notifyId}] Failed to resolve template: ${(error as Error).message}`) + throw error } + } - logger.log(`[${notifyId}] Mail merge batch ${batchId} complete: sent=${sent}, failed=${failed}`) - await EmailDeliveryWorker.settleMergeRequest( - notifyId, - tenantId, - requestDetailService, - notificationService, - logger, - structuredLogger, + private async renderInline( + notifyId: string, + payload: EmailPayload, + params: Record, + ): Promise { + this.logger.debug( + `[${notifyId}] Rendering inline content with renderer: ${payload.content.renderer}`, ) - - return { success: true, batchId, sent, failed } + try { + const rendered = await this.inlineRenderingService.renderEmail(payload.content, params) + this.logger.debug(`[${notifyId}] Inline content rendered successfully`) + return { + ...payload, + content: { ...payload.content, subject: rendered.subject, body: rendered.body }, + } + } catch (error) { + this.logger.error( + `[${notifyId}] Failed to render inline content: ${(error as Error).message}`, + ) + throw error + } } - /** - * Settle the parent request once no recipients remain pending across all batches. - * all sent → COMPLETED; some sent + some failed → PARTIALLY_COMPLETED; all failed → FAILED. - * Anything short of COMPLETED is logged as one delivery failure for the whole merge, which is - * what the EmailSendingFailures alert counts. - */ - private static async settleMergeRequest( + /** Swap stored attachment references for their content. */ + private async resolveAttachments( notifyId: string, tenantId: string, - requestDetailService: NotificationRequestDetailService, - notificationService: NotificationService, - logger: Logger, - structuredLogger?: StructuredLoggerService, - ): Promise { - const pendingRemaining = await requestDetailService.countByStatus(notifyId, 'pending') - if (pendingRemaining === 0) { - const failedRemaining = await requestDetailService.countByStatus(notifyId, 'failed') - const sentRemaining = await requestDetailService.countByStatus(notifyId, 'sent') - let finalStatus: NotificationStatus - if (failedRemaining === 0) { - finalStatus = NotificationStatus.COMPLETED - } else if (sentRemaining > 0) { - finalStatus = NotificationStatus.PARTIALLY_COMPLETED - } else { - finalStatus = NotificationStatus.FAILED - } - await notificationService.update(notifyId, tenantId, { - status: finalStatus, - updatedBy: 'email-delivery-worker', - }) - logger.log( - `[${notifyId}] All mail merge batches complete; parent marked ${finalStatus.toUpperCase()}`, + payload: EmailPayload, + ): Promise { + if (!payload.attachments) return payload + if (!hasAttachmentReferences(payload.attachments)) { + throw new Error('Invalid processed email attachment reference payload') + } + if (!this.attachmentResolverService) { + throw new Error( + 'AttachmentResolverService is not available to resolve stored email attachments', ) - if (finalStatus !== NotificationStatus.COMPLETED) { - structuredLogger?.logNotificationFailure( - notifyId, - tenantId, - 'email', - `${failedRemaining} of ${failedRemaining + sentRemaining} merge recipients failed`, - undefined, - EmailDeliveryWorker.name, - ) - } } + + this.logger.debug( + `[${notifyId}] Resolving stored email attachments: ${JSON.stringify({ + tenantId, + storedAttachmentCount: payload.attachments.length, + })}`, + ) + const attachments = await this.attachmentResolverService.resolveEmailAttachments( + tenantId, + payload.attachments, + ) + return { + ...payload, + attachments: attachments as SendEmailOptions['attachments'], + } as ResolvedEmailDeliveryPayload } - /** - * Send email via adapter - * @param payload Email payload - * @param logger Logger instance - * @param notifyId Notification ID for tracing - * @param emailAdapter Email transport adapter - * @returns Promise with send result - */ - private static async sendEmail( - payload: NotifyEmailChannel | ProcessedNotifyEmailChannel | ResolvedEmailDeliveryPayload, - logger: Logger, + private async sendEmail( + payload: EmailPayload, notifyId: string, - emailAdapter: IEmailTransport, fromAddress: string | null, ): Promise<{ externalId: string; provider: string }> { - const attachmentCount = Array.isArray((payload as { attachments?: unknown[] }).attachments) - ? ((payload as { attachments?: unknown[] }).attachments?.length ?? 0) - : 0 - - logger.debug( - `[${notifyId}] Sending email via ${emailAdapter.name} adapter: ${JSON.stringify({ + const attachmentCount = Array.isArray(payload.attachments) ? payload.attachments.length : 0 + this.logger.debug( + `[${notifyId}] Sending email via ${this.emailAdapter.name} adapter: ${JSON.stringify({ attachmentCount, })}`, ) - // Only set `from` when the tenant configured one: the adapters read their own default when - // the field is absent, and an explicit undefined would not be the same thing. - const result = await emailAdapter.send( + // Only set `from` when one is configured: the adapters read their own default when the field + // is absent, and an explicit undefined would not be the same thing. + const result = await this.emailAdapter.send( (fromAddress ? { ...payload, from: fromAddress } : payload) as any, ) return { externalId: - result.messageId || result.providerResponse || `${emailAdapter.name}-${Date.now()}`, - provider: emailAdapter.name, + result.messageId || result.providerResponse || `${this.emailAdapter.name}-${Date.now()}`, + provider: this.emailAdapter.name, } } } diff --git a/backend/src/queue/workers/ingestion.worker.spec.ts b/backend/src/queue/workers/ingestion.worker.spec.ts index 6e30cd82d..b3f75cd22 100644 --- a/backend/src/queue/workers/ingestion.worker.spec.ts +++ b/backend/src/queue/workers/ingestion.worker.spec.ts @@ -1,5 +1,5 @@ import { Logger } from '@nestjs/common' -import Bull from 'bull' +import type { Job, Queue } from 'bullmq' import { vi, describe, it, expect, beforeEach } from 'vitest' import { IngestionWorker } from './ingestion.worker' import { IngestionJobPayload, DeliveryJobPayload } from '../queue.types' @@ -7,18 +7,50 @@ import { NotificationChannel } from '../../enum/notification-channel.enum' import { NotificationStatus } from '../../enum/notification-status.enum' import { AttachmentService } from '../../api/attachment/attachment.service' import { FAILED_JOB_RETENTION } from '../job-retention' +import { BACKOFF_STRATEGY } from '../job-options' +import { QueueName } from '../../enum/queue-name.enum' +import { PhoneNumberService } from '../../api/notify/services/phone-number.service' +import type { ConfigService } from '@nestjs/config' +import type { ClamavService } from '../../services/clamav.service' +import type { NotificationService } from '../../api/notification/notification.service' +import type { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' describe('IngestionWorker', () => { - let mockIngestionQueue: Partial> - let mockEmailQueue: Partial> - let mockSmsQueue: Partial> + let mockEmailQueue: Partial> + let mockSmsQueue: Partial> let mockNotificationService: any let mockRequestDetailService: any let mockConfigService: any let mockClamavService: any let mockAttachmentService: any - let processHandler: (job: Bull.Job) => Promise - let failedCallback: (job: Bull.Job, err: Error) => void + let processHandler: (job: Job) => Promise + let failedCallback: (job: Job, err: Error) => void + + /** The worker as QueueModule builds it, with its handlers captured for the tests to call. */ + function initialize(deps: { + emailQueue: Queue + smsQueue: Queue + notificationService: NotificationService + requestDetailService: NotificationRequestDetailService + configService: ConfigService + clamavService?: ClamavService + concurrency?: number + attachmentService?: AttachmentService + phoneNumberService?: PhoneNumberService + }) { + const worker = new IngestionWorker( + deps.emailQueue, + deps.smsQueue, + deps.notificationService, + deps.requestDetailService, + deps.configService, + deps.phoneNumberService ?? new PhoneNumberService(), + deps.clamavService, + deps.attachmentService, + ) + processHandler = (job) => worker.process(job) + failedCallback = (job, err) => worker.onFailed(job, err) + } beforeEach(() => { mockNotificationService = { @@ -58,24 +90,13 @@ describe('IngestionWorker', () => { } as any), } - mockIngestionQueue = { - process: vi.fn().mockImplementation((...args) => { - const handler = typeof args[0] === 'function' ? args[0] : args[1] - processHandler = handler - return Promise.resolve() - }), - on: vi.fn().mockImplementation((event, callback) => { - if (event === 'failed') { - failedCallback = callback - } - }), - } - mockEmailQueue = { + name: QueueName.EMAIL_DELIVERY, add: vi.fn().mockResolvedValue({ id: 'email-job-1' }), } mockSmsQueue = { + name: QueueName.SMS_DELIVERY, add: vi.fn().mockResolvedValue({ id: 'sms-job-1' }), } @@ -90,39 +111,10 @@ describe('IngestionWorker', () => { }) describe('initialize', () => { - it('should register a process handler on the ingestion queue', () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, - notificationService: mockNotificationService, - requestDetailService: mockRequestDetailService, - configService: mockConfigService, - clamavService: mockClamavService, - }) - - expect(mockIngestionQueue.process).toHaveBeenCalled() - }) - - it('should register event listeners on the ingestion queue', () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, - notificationService: mockNotificationService, - requestDetailService: mockRequestDetailService, - configService: mockConfigService, - clamavService: mockClamavService, - }) - - expect(mockIngestionQueue.on).toHaveBeenCalledWith('failed', expect.any(Function)) - }) - it('should process email delivery job when email channel is requested', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, @@ -141,10 +133,11 @@ describe('IngestionWorker', () => { }, requestedAt: new Date().toISOString(), }, - } as Bull.Job) + } as Job) expect(result).toEqual({ success: true, deliveryJobsQueued: 1 }) expect(mockEmailQueue.add).toHaveBeenCalledWith( + QueueName.EMAIL_DELIVERY, expect.objectContaining({ notifyId: 'notify-123', channel: NotificationChannel.EMAIL, @@ -157,17 +150,16 @@ describe('IngestionWorker', () => { }) it('should process SMS delivery job when SMS channel is requested', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, clamavService: mockClamavService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-456', tenantId: 'tenant-456', @@ -178,10 +170,11 @@ describe('IngestionWorker', () => { }, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result).toEqual({ success: true, deliveryJobsQueued: 1 }) expect(mockSmsQueue.add).toHaveBeenCalledWith( + QueueName.SMS_DELIVERY, expect.objectContaining({ notifyId: 'notify-456', channel: NotificationChannel.SMS, @@ -193,17 +186,16 @@ describe('IngestionWorker', () => { }) it('normalizes an SMS recipient once before persistence and delivery fan-out', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, clamavService: mockClamavService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-normalized-sms', tenantId: 'tenant-456', @@ -217,7 +209,7 @@ describe('IngestionWorker', () => { }, } - await processHandler(job as Bull.Job) + await processHandler(job as Job) expect(mockRequestDetailService.createPending).toHaveBeenCalledWith( 'notify-normalized-sms', @@ -229,6 +221,7 @@ describe('IngestionWorker', () => { 'tenant-456', ) expect(mockSmsQueue.add).toHaveBeenCalledWith( + QueueName.SMS_DELIVERY, expect.objectContaining({ request: expect.objectContaining({ sms: expect.objectContaining({ recipients: { to: ['+12505551234'] } }), @@ -240,17 +233,16 @@ describe('IngestionWorker', () => { }) it('should process both email and SMS delivery jobs when both channels are requested', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, clamavService: mockClamavService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-789', tenantId: 'tenant-789', @@ -265,7 +257,7 @@ describe('IngestionWorker', () => { }, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result).toEqual({ success: true, deliveryJobsQueued: 2 }) expect(mockEmailQueue.add).toHaveBeenCalled() @@ -273,10 +265,9 @@ describe('IngestionWorker', () => { }) it('should add delay for scheduled sends', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, @@ -286,7 +277,7 @@ describe('IngestionWorker', () => { // Schedule for 1 minute from now const scheduledFor = new Date(Date.now() + 60000).toISOString() - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-scheduled', tenantId: 'tenant-scheduled', @@ -301,9 +292,10 @@ describe('IngestionWorker', () => { }, } - await processHandler(job as Bull.Job) + await processHandler(job as Job) expect(mockEmailQueue.add).toHaveBeenCalledWith( + QueueName.EMAIL_DELIVERY, expect.any(Object), expect.objectContaining({ delay: expect.any(Number), @@ -311,16 +303,15 @@ describe('IngestionWorker', () => { ) // Check that delay is positive (close to 60000ms) - const callArgs = (mockEmailQueue.add as any).mock.calls[0][1] + const callArgs = (mockEmailQueue.add as any).mock.calls[0][2] expect(callArgs.delay).toBeGreaterThan(50000) expect(callArgs.delay).toBeLessThanOrEqual(60000) }) it('should update status to processing for scheduled notifications', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, @@ -329,7 +320,7 @@ describe('IngestionWorker', () => { const futureDate = new Date(Date.now() + 60000).toISOString() - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-scheduled', tenantId: 'tenant-scheduled', @@ -344,7 +335,7 @@ describe('IngestionWorker', () => { }, } - await processHandler(job as Bull.Job) + await processHandler(job as Job) expect(mockNotificationService.update).toHaveBeenCalledWith( 'notify-scheduled', @@ -357,17 +348,16 @@ describe('IngestionWorker', () => { }) it('should throw error when no channels are specified', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, clamavService: mockClamavService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-no-channel', tenantId: 'tenant-no-channel', @@ -376,23 +366,22 @@ describe('IngestionWorker', () => { }, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'No delivery channels specified - this should have been caught by validateBusinessRules', ) }) it('should throw error when request is missing', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, clamavService: mockClamavService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-no-request', tenantId: 'tenant-no-request', @@ -401,23 +390,22 @@ describe('IngestionWorker', () => { }, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid request: request payload is missing or invalid', ) }) it('should throw error when notifyId is missing', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, clamavService: mockClamavService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: undefined as any, tenantId: 'tenant-123', @@ -431,23 +419,22 @@ describe('IngestionWorker', () => { }, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid ingestion job: notifyId is missing or invalid', ) }) it('should throw error when tenantId is missing', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, clamavService: mockClamavService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', tenantId: null as any, @@ -461,23 +448,22 @@ describe('IngestionWorker', () => { }, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid ingestion job: tenantId is missing or invalid', ) }) it('should throw error when requestedAt is missing', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, clamavService: mockClamavService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', tenantId: 'tenant-123', @@ -491,37 +477,22 @@ describe('IngestionWorker', () => { }, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid ingestion job: requestedAt is missing or invalid', ) }) - it('should register event listeners successfully', () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, - notificationService: mockNotificationService, - requestDetailService: mockRequestDetailService, - configService: mockConfigService, - clamavService: mockClamavService, - }) - - expect(mockIngestionQueue.on).toHaveBeenCalledWith('failed', expect.any(Function)) - }) - it('should call failed callback on job error', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, clamavService: mockClamavService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-failed', tenantId: 'tenant-failed', @@ -534,12 +505,12 @@ describe('IngestionWorker', () => { const error = new Error('Test error') try { - await processHandler(job as Bull.Job) + await processHandler(job as Job) } catch { // Expected } - failedCallback(job as Bull.Job, error) + failedCallback(job as Job, error) // The logger.error should be called - check that it was called at all expect(Logger.prototype.error).toHaveBeenCalled() @@ -551,10 +522,9 @@ describe('IngestionWorker', () => { }) it('should include job data in delivery payload', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, @@ -565,7 +535,7 @@ describe('IngestionWorker', () => { recipients: { to: ['test@example.com'] }, content: { subject: 'Test Subject', body: 'Test body' }, } - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-data', tenantId: 'tenant-data', @@ -576,9 +546,10 @@ describe('IngestionWorker', () => { }, } - await processHandler(job as Bull.Job) + await processHandler(job as Job) expect(mockEmailQueue.add).toHaveBeenCalledWith( + QueueName.EMAIL_DELIVERY, expect.objectContaining({ notifyId: 'notify-data', tenantId: 'tenant-data', @@ -591,17 +562,16 @@ describe('IngestionWorker', () => { }) it('should set retry and backoff configuration on delivery jobs', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, clamavService: mockClamavService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-retry', tenantId: 'tenant-retry', @@ -615,14 +585,15 @@ describe('IngestionWorker', () => { }, } - await processHandler(job as Bull.Job) + await processHandler(job as Job) expect(mockEmailQueue.add).toHaveBeenCalledWith( + QueueName.EMAIL_DELIVERY, expect.any(Object), expect.objectContaining({ attempts: 3, backoff: { - type: 'exponential', + type: BACKOFF_STRATEGY, delay: 2000, }, removeOnComplete: true, @@ -636,10 +607,9 @@ describe('IngestionWorker', () => { new Error('ClamAV is unavailable while fail-closed mode is enabled'), ) - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, @@ -648,7 +618,7 @@ describe('IngestionWorker', () => { attachmentService: mockAttachmentService as AttachmentService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-attachment-retry', tenantId: 'tenant-attachment-retry', @@ -667,7 +637,7 @@ describe('IngestionWorker', () => { }, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Attachment scan failed: ClamAV is unavailable while fail-closed mode is enabled', ) @@ -683,10 +653,9 @@ describe('IngestionWorker', () => { }) it('should scan stored attachment references using attachmentId and tenantId', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, @@ -708,7 +677,7 @@ describe('IngestionWorker', () => { }, requestedAt: new Date().toISOString(), }, - } as any as Bull.Job) + } as any as Job) expect(result).toEqual({ success: true, deliveryJobsQueued: 1 }) expect(mockAttachmentService.downloadAttachmentByIdAndTenantId).toHaveBeenCalledWith( @@ -726,10 +695,9 @@ describe('IngestionWorker', () => { new Error('Attachment not found'), ) - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, @@ -752,7 +720,7 @@ describe('IngestionWorker', () => { }, requestedAt: new Date().toISOString(), }, - } as any as Bull.Job), + } as any as Job), ).rejects.toThrow('Attachment scan failed: Attachment not found') expect(mockEmailQueue.add).not.toHaveBeenCalled() @@ -767,10 +735,9 @@ describe('IngestionWorker', () => { }) it('should fail when processed attachments are not attachmentId references', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, @@ -791,7 +758,7 @@ describe('IngestionWorker', () => { }, requestedAt: new Date().toISOString(), }, - } as any as Bull.Job), + } as any as Job), ).rejects.toThrow('Invalid processed attachment reference payload') expect(mockAttachmentService.downloadAttachmentByIdAndTenantId).not.toHaveBeenCalled() @@ -805,10 +772,9 @@ describe('IngestionWorker', () => { mockRequestDetailService.countInFlight.mockImplementation((_id: string, channel: string) => Promise.resolve(channel === NotificationChannel.EMAIL ? 0 : 1), ) - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, @@ -828,7 +794,7 @@ describe('IngestionWorker', () => { sms: { recipients: { to: ['+12505550123'] }, content: { body: 'B' } }, } as any, }, - } as Bull.Job) + } as Job) expect(mockRequestDetailService.createPending).not.toHaveBeenCalled() expect(mockEmailQueue.add).not.toHaveBeenCalled() @@ -840,10 +806,9 @@ describe('IngestionWorker', () => { describe('bulk email fan-out', () => { it('should fan out a bulk job into one batch and update status to PROCESSING', async () => { - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, @@ -854,7 +819,7 @@ describe('IngestionWorker', () => { { address: 'alice@example.com', params: {} }, { address: 'bob@example.com', params: {} }, ] - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-bulk', tenantId: 'tenant-bulk', @@ -869,7 +834,7 @@ describe('IngestionWorker', () => { }, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result).toEqual({ success: true, deliveryJobsQueued: 1 }) expect(mockRequestDetailService.createMergePending).toHaveBeenCalledTimes(1) @@ -883,6 +848,7 @@ describe('IngestionWorker', () => { ) expect(mockEmailQueue.add).toHaveBeenCalledTimes(1) expect(mockEmailQueue.add).toHaveBeenCalledWith( + QueueName.EMAIL_DELIVERY, expect.objectContaining({ notifyId: 'notify-bulk', tenantId: 'tenant-bulk', @@ -897,7 +863,7 @@ describe('IngestionWorker', () => { removeOnFail: FAILED_JOB_RETENTION, attempts: 3, backoff: { - type: 'exponential', + type: BACKOFF_STRATEGY, delay: 2000, }, }), @@ -914,17 +880,16 @@ describe('IngestionWorker', () => { return undefined }) - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, clamavService: mockClamavService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-bulk-multi', tenantId: 'tenant-bulk', @@ -943,7 +908,7 @@ describe('IngestionWorker', () => { }, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) // 3 addresses, batchSize=2 → 2 batches expect(result).toEqual({ success: true, deliveryJobsQueued: 2 }) @@ -967,7 +932,7 @@ describe('IngestionWorker', () => { 'tenant-bulk', ) // Jobs carry the batch id, not its recipients. - for (const [payload] of mockEmailQueue.add.mock.calls) { + for (const [, payload] of mockEmailQueue.add.mock.calls) { expect(payload.mailMergeData).not.toHaveProperty('recipients') } }) @@ -979,17 +944,16 @@ describe('IngestionWorker', () => { params: {}, })) - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, clamavService: mockClamavService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-bulk-default', tenantId: 'tenant-bulk', @@ -1004,7 +968,7 @@ describe('IngestionWorker', () => { }, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) // 150 addresses with default batchSize=25 → 6 batches expect(result).toEqual({ success: true, deliveryJobsQueued: 6 }) @@ -1033,10 +997,9 @@ describe('IngestionWorker', () => { new Set(['blocked@example.com']), ) - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, @@ -1052,7 +1015,7 @@ describe('IngestionWorker', () => { mailMerge: true, mailMergeData: { content: { templateId: 'template-uuid' }, params: {} }, }, - } as Bull.Job) + } as Job) expect(result).toEqual({ success: true, deliveryJobsQueued: 1 }) expect(mockNotificationService.findOne).toHaveBeenCalledWith('notify-db', 'tenant-bulk') @@ -1076,10 +1039,9 @@ describe('IngestionWorker', () => { key === 'queue.batchSize' ? 2 : undefined, ) - IngestionWorker.initialize({ - ingestionQueue: mockIngestionQueue as Bull.Queue, - emailQueue: mockEmailQueue as Bull.Queue, - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ + emailQueue: mockEmailQueue as Queue, + smsQueue: mockSmsQueue as Queue, notificationService: mockNotificationService, requestDetailService: mockRequestDetailService, configService: mockConfigService, @@ -1102,9 +1064,9 @@ describe('IngestionWorker', () => { ], }, }, - } as Bull.Job) + } as Job) - // Batch 0 already had rows; only batch 1 is written. Both are (re)queued - Bull ignores a + // Batch 0 already had rows; only batch 1 is written. Both are (re)queued - BullMQ ignores a // jobId it already holds. expect(mockRequestDetailService.createMergePending).toHaveBeenCalledTimes(1) expect(mockRequestDetailService.createMergePending).toHaveBeenCalledWith( diff --git a/backend/src/queue/workers/ingestion.worker.ts b/backend/src/queue/workers/ingestion.worker.ts index 66c28cc8e..90abac9da 100644 --- a/backend/src/queue/workers/ingestion.worker.ts +++ b/backend/src/queue/workers/ingestion.worker.ts @@ -1,7 +1,9 @@ -import { Logger } from '@nestjs/common' -import Bull from 'bull' +import { Inject, Injectable, Logger, Optional } from '@nestjs/common' +import type { Job, Queue } from 'bullmq' import { ConfigService } from '@nestjs/config' import { IngestionJobPayload, DeliveryJobPayload } from '../queue.types' +import { QueueProcessor } from './queue-processor' +import { QueueName } from '../../enum/queue-name.enum' import { NotificationChannel } from '../../enum/notification-channel.enum' import { NotificationStatus } from '../../enum/notification-status.enum' import { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' @@ -11,72 +13,226 @@ import { QuarantineDetails } from '../../api/notification/entities/notification- import { AttachmentService } from '../../api/attachment/attachment.service' import { PhoneNumberService } from '../../api/notify/services/phone-number.service' import { mergeJobDataFromRequest } from '../merge-batch-builder' -import { FAILED_JOB_RETENTION } from '../job-retention' - -export interface IngestionWorkerDeps { - /** The Bull queue instance for ingestion jobs */ - ingestionQueue: Bull.Queue - /** Queue for email delivery jobs */ - emailQueue: Bull.Queue - /** Queue for SMS delivery jobs */ - smsQueue: Bull.Queue - /** Service for database updates */ - notificationService: NotificationService - requestDetailService: NotificationRequestDetailService - /** Configuration service for queue settings */ - configService: ConfigService - /** Service for malware scanning */ - clamavService?: ClamavService - /** Number of jobs to process in parallel (default: 1) */ - concurrency?: number - attachmentService?: AttachmentService - phoneNumberService?: PhoneNumberService +import { DELIVERY_JOB_OPTIONS } from '../job-options' + +type IngestionRequest = IngestionJobPayload['request'] + +function hasAttachmentReferences( + attachments: unknown, +): attachments is Array<{ attachmentId: string }> { + return ( + Array.isArray(attachments) && + attachments.every( + (attachment) => + attachment && + typeof attachment === 'object' && + typeof (attachment as { attachmentId?: unknown }).attachmentId === 'string', + ) + ) +} + +function assertIngestionJob({ notifyId, tenantId, requestedAt, request }: IngestionJobPayload) { + if (!notifyId || typeof notifyId !== 'string') { + throw new Error('Invalid ingestion job: notifyId is missing or invalid') + } + if (!tenantId || typeof tenantId !== 'string') { + throw new Error('Invalid ingestion job: tenantId is missing or invalid') + } + if (!requestedAt || typeof requestedAt !== 'string') { + throw new Error('Invalid ingestion job: requestedAt is missing or invalid') + } + if (!request || typeof request !== 'object') { + throw new Error('Invalid request: request payload is missing or invalid') + } } /** * Ingestion Worker * - * Orchestrates notification processing: - * 1. Receives notification requests from the ingestion queue - * 2. Validates and determines which channels are requested (email, SMS) - * 3. Fans out to channel-specific delivery queues - * 4. Updates notification status in database + * Turns an accepted request into delivery work: writes one detail row per recipient, scans + * attachments, and fans out to the channel delivery queues - one job per channel for a plain + * send, one per BATCH_SIZE recipients for a merge. A scheduled send's delay is on this job, so + * delivery is queued only once it is due. * - * Idempotency: Job key is notifyId, preventing duplicate processing - * Tracing: All operations logged with correlationId for end-to-end visibility + * Idempotency: the job key is the notifyId, and a re-run writes no rows twice. */ -export class IngestionWorker { +@Injectable() +export class IngestionWorker implements QueueProcessor { + readonly queueName = QueueName.INGESTION + readonly concurrency = { configKey: 'ingestionWorkerConcurrency', default: 1 } private readonly logger = new Logger(IngestionWorker.name) + constructor( + @Inject(QueueName.EMAIL_DELIVERY) private readonly emailQueue: Queue, + @Inject(QueueName.SMS_DELIVERY) private readonly smsQueue: Queue, + private readonly notificationService: NotificationService, + private readonly requestDetailService: NotificationRequestDetailService, + private readonly configService: ConfigService, + private readonly phoneNumberService: PhoneNumberService, + @Optional() private readonly clamavService?: ClamavService, + @Optional() private readonly attachmentService?: AttachmentService, + ) {} + + async process(job: Job): Promise { + const { notifyId, tenantId } = job.data + this.logger.debug(`[${notifyId}] Processing ingestion job for tenant=${tenantId}`) + + // Set once ingestion finds rows an earlier run wrote; read by both the success and error paths. + let resumed = false + try { + this.logger.log(`[${notifyId}] [IngestionWorker] Starting to process job ${job.id}`) + assertIngestionJob(job.data) + + if (job.data.mailMerge && job.data.mailMergeData) { + return await this.fanOutMerge(job.data) + } + + const request = this.normalizeSmsRecipients(job.data.request) + // A re-run (a queue retry, or recovery by the delivery reconciler) finds its rows already + // written; writing them again would double every recipient. + resumed = (await this.requestDetailService.countUnbatched(notifyId)) > 0 + if (!resumed) { + await this.requestDetailService.createPending(notifyId, request, tenantId) + } + + if (await this.quarantined(notifyId, tenantId, request)) { + return { success: false, reason: 'Quarantined due to malware detection' } + } + + const queued = await this.fanOut(job.data, request, resumed) + + await this.notificationService.update(notifyId, tenantId, { + status: NotificationStatus.PROCESSING, + updatedBy: 'ingestion-worker', + }) + // Not on a re-run: some rows are already sent or failed, and a blanket update would mark + // them in flight again - and the reconciler would then send them twice. + if (!resumed) { + await this.requestDetailService.updateStatus(notifyId, NotificationStatus.PROCESSING) + } + + return { success: true, deliveryJobsQueued: queued } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error) + this.logger.error( + `[${notifyId}] Failed to process ingestion job: ${errorMessage}`, + error instanceof Error ? error.stack : '', + ) + + await this.notificationService.update(notifyId, tenantId, { + status: NotificationStatus.FAILED, + updatedBy: 'ingestion-worker', + }) + await this.requestDetailService.updateStatus(notifyId, NotificationStatus.FAILED, { + preserveCompleted: resumed, + }) + + // Rethrown for the queue's retry policy. + throw error + } + } + + onCompleted(job: Job): void { + this.logger.debug(`[${job.data.notifyId}] Ingestion job completed`) + } + + onFailed(job: Job, error: Error): void { + this.logger.error( + `[${job.data.notifyId}] Ingestion job failed (attempt ${job.attemptsMade}/${job.opts.attempts}): error=${error.message}`, + ) + } + /** * Every recipient of a merge, from the request as stored at acceptance, less those the * safelist blocked then (recorded as `blocked` rows). Read here rather than carried on the * job, so a large merge is not held in Redis. */ - static async loadMergeRecipients( + private async loadMergeRecipients( notifyId: string, tenantId: string, channel: NotificationChannel, - notificationService: NotificationService, - requestDetailService: NotificationRequestDetailService, ): Promise }>> { const [stored, blocked] = await Promise.all([ - notificationService.findOne(notifyId, tenantId), - requestDetailService.findAddressesByStatus(notifyId, 'blocked'), + this.notificationService.findOne(notifyId, tenantId), + this.requestDetailService.findAddressesByStatus(notifyId, 'blocked'), ]) return mergeJobDataFromRequest( stored.payload, channel, (mergeArray, mergeChannel) => - notificationService.parseMailMergeRecipients(mergeArray, mergeChannel), + this.notificationService.parseMailMergeRecipients(mergeArray, mergeChannel), (address) => !blocked.has(address), ).recipients! } - private static normalizeSmsRecipients( - request: IngestionJobPayload['request'], - phoneNumberService: PhoneNumberService, - ): IngestionJobPayload['request'] { + /** + * Split a merge's recipients into fixed-size batches and queue one delivery job per batch on + * the merge's channel. Each batch's detail rows are written here, tagged with its batchId. + */ + private async fanOutMerge(data: IngestionJobPayload): Promise { + const { notifyId, tenantId, request } = data + const { content, params } = data.mailMergeData! + const batchSize = this.configService?.get('queue.batchSize') || 25 + // Jobs queued before SMS merge existed carry no channel and are email. + const channel = data.mailMergeChannel ?? NotificationChannel.EMAIL + const recipients = + data.mailMergeData!.recipients ?? + (await this.loadMergeRecipients(notifyId, tenantId, channel)) + const queue = channel === NotificationChannel.SMS ? this.smsQueue : this.emailQueue + + this.logger.log( + `[${notifyId}] Processing ${channel} merge job: ${recipients.length} recipient(s), batchSize=${batchSize}`, + ) + + let batchIndex = 0 + // In order, one batch at a time: each batch's rows are written before its job is queued. + for (let start = 0; start < recipients.length; start += batchSize, batchIndex++) { + const chunk = recipients.slice(start, start + batchSize) + // {notification_request id}-{channel}-{index}, reused as the delivery jobId so a failed + // batch is easy to identify and retry. + const batchId = `${notifyId}-${channel}-${batchIndex}` + + // A retried ingestion job re-runs this loop. Rows already written for a batch mean an + // earlier run created it; writing them again would send those recipients twice. + if ((await this.requestDetailService.countBatch(notifyId, batchId)) === 0) { + await this.requestDetailService.createMergePending( + notifyId, + batchId, + chunk, + channel, + tenantId, + ) + } + + const deliveryPayload: DeliveryJobPayload = { + notifyId, + tenantId, + channel, + request, + payload: {} as any, + attempt: 0, + mailMerge: true, + batchId, + // Recipients stay in Postgres; the worker reads this batch's detail rows. + mailMergeData: { content, params }, + } + await queue.add(queue.name, deliveryPayload, { jobId: batchId, ...DELIVERY_JOB_OPTIONS }) + + this.logger.log( + `[${notifyId}] Queued mail merge batch ${batchIndex} (batchId=${batchId}, recipients=${chunk.length})`, + ) + } + + await this.notificationService.update(notifyId, tenantId, { + status: NotificationStatus.PROCESSING, + updatedBy: 'ingestion-worker', + }) + + this.logger.log(`[${notifyId}] Mail merge email job fanned out into ${batchIndex} batch(es)`) + return { success: true, deliveryJobsQueued: batchIndex } + } + + private normalizeSmsRecipients(request: IngestionRequest): IngestionRequest { if (!request.sms?.recipients?.to) return request return { @@ -86,393 +242,192 @@ export class IngestionWorker { recipients: { ...request.sms.recipients, to: request.sms.recipients.to.map( - (recipient) => phoneNumberService.normalize(recipient) ?? recipient, + (recipient) => this.phoneNumberService.normalize(recipient) ?? recipient, ), }, }, - } as IngestionJobPayload['request'] + } as IngestionRequest } - private static hasAttachmentReferences( - attachments: unknown, - ): attachments is Array<{ attachmentId: string }> { - return ( - Array.isArray(attachments) && - attachments.every( - (attachment) => - attachment && - typeof attachment === 'object' && - typeof (attachment as { attachmentId?: unknown }).attachmentId === 'string', - ) - ) - } - - /** Initialize the ingestion worker on a queue */ - static initialize({ - ingestionQueue, - emailQueue, - smsQueue, - notificationService, - requestDetailService, - configService, - clamavService, - concurrency = 1, - attachmentService, - phoneNumberService = new PhoneNumberService(), - }: IngestionWorkerDeps): void { - const logger = new Logger(IngestionWorker.name) - - logger.log(`Registering ingestion worker processor (concurrency=${concurrency})`) - logger.log( - `Queue check - ingestion: ${!!ingestionQueue}, email: ${!!emailQueue}, sms: ${!!smsQueue}`, - ) + /** + * Scan every attachment for malware. An infected one quarantines the request and nothing is + * delivered; a scan that cannot run throws, so the job is retried rather than sent unscanned. + */ + private async quarantined( + notifyId: string, + tenantId: string, + request: IngestionRequest, + ): Promise { + const attachments = [ + ...(request.email?.attachments ?? []), + ...(request.sms?.attachments ?? []), + ...(request.msgApp?.attachments ?? []), + ] + if (attachments.length === 0) return false + + if (!this.clamavService) { + throw new Error('Attachment scan service unavailable') + } + if (!hasAttachmentReferences(attachments)) { + throw new Error('Invalid processed attachment reference payload') + } + if (!this.attachmentService) { + throw new Error('Attachment service unavailable') + } + + this.logger.log(`[${notifyId}] Scanning ${attachments.length} attachment(s) for malware`) + // One at a time: each is downloaded in full to scan, and the first infected one ends the scan. + for (const { attachmentId } of attachments) { + const quarantine = await this.scanAttachment(notifyId, tenantId, attachmentId) + if (quarantine) { + await this.quarantine(notifyId, tenantId, quarantine) + return true + } + } - // Register the job processor with configurable concurrency - ingestionQueue.process(concurrency, async (job: Bull.Job) => { - const { notifyId, tenantId, request, scheduledFor, requestedAt } = job.data - - logger.debug(`[${notifyId}] Processing ingestion job for tenant=${tenantId}`) - - // Set once ingestion finds rows an earlier run wrote; read by both the success and error paths. - let resumed = false - try { - logger.log(`[${notifyId}] [IngestionWorker] Starting to process job ${job.id}`) - - // Validate IngestionJobPayload structure - if (!notifyId || typeof notifyId !== 'string') { - throw new Error('Invalid ingestion job: notifyId is missing or invalid') - } - if (!tenantId || typeof tenantId !== 'string') { - throw new Error('Invalid ingestion job: tenantId is missing or invalid') - } - if (!requestedAt || typeof requestedAt !== 'string') { - throw new Error('Invalid ingestion job: requestedAt is missing or invalid') - } - - // Validate request structure - if (!request || typeof request !== 'object') { - throw new Error('Invalid request: request payload is missing or invalid') - } - let processedRequest - - // Mail merge email send: split recipients into fixed-size batches and fan out one - // email-delivery job per batch. Detail rows are created here, tagged with a batchId. - if (job.data.mailMerge && job.data.mailMergeData) { - const { content, params } = job.data.mailMergeData - const batchSize = configService?.get('queue.batchSize') || 25 - // Older jobs queued before SMS merge existed carry no channel and are email. - const mergeChannel = job.data.mailMergeChannel ?? NotificationChannel.EMAIL - const recipients = - job.data.mailMergeData.recipients ?? - (await IngestionWorker.loadMergeRecipients( - notifyId, - tenantId, - mergeChannel, - notificationService, - requestDetailService, - )) - const isSmsMerge = mergeChannel === NotificationChannel.SMS - const mergeQueue = isSmsMerge ? smsQueue : emailQueue - - logger.log( - `[${notifyId}] Processing ${mergeChannel} merge job: ${recipients.length} recipient(s), batchSize=${batchSize}`, - ) - - let batchIndex = 0 - for (let start = 0; start < recipients.length; start += batchSize) { - const chunk = recipients.slice(start, start + batchSize) - // Format: {notification_request id}-{channel}-{index}; reused as the delivery jobId - // so a failed batch is easy to identify and retry. - const batchId = `${notifyId}-${mergeChannel}-${batchIndex}` - - // A retried ingestion job re-runs this loop. Rows already written for a batch mean an - // earlier run created it; writing them again would send those recipients twice. - if ((await requestDetailService.countBatch(notifyId, batchId)) === 0) { - await requestDetailService.createMergePending( - notifyId, - batchId, - chunk, - mergeChannel, - tenantId, - ) - } - - const deliveryPayload: DeliveryJobPayload = { - notifyId, - tenantId, - channel: mergeChannel, - request, - payload: {} as any, - attempt: 0, - mailMerge: true, - batchId, - // Recipients stay in Postgres; the worker reads this batch's detail rows. - mailMergeData: { content, params }, - } - - await mergeQueue.add(deliveryPayload, { - jobId: batchId, - removeOnComplete: true, - removeOnFail: FAILED_JOB_RETENTION, - attempts: 3, - backoff: { type: 'exponential', delay: 2000 }, - }) - - logger.log( - `[${notifyId}] Queued mail merge batch ${batchIndex} (batchId=${batchId}, recipients=${chunk.length})`, - ) - batchIndex++ - } - - await notificationService.update(notifyId, tenantId, { - status: NotificationStatus.PROCESSING, - updatedBy: 'ingestion-worker', - }) - - logger.log(`[${notifyId}] Mail merge email job fanned out into ${batchIndex} batch(es)`) - return { success: true, deliveryJobsQueued: batchIndex } - } else { - processedRequest = IngestionWorker.normalizeSmsRecipients(request, phoneNumberService) - - // A re-run (Bull retry, or recovery by the delivery reconciler) finds its rows already - // written; writing them again would double every recipient. - resumed = (await requestDetailService.countUnbatched(notifyId)) > 0 - if (!resumed) { - await requestDetailService.createPending(notifyId, processedRequest, tenantId) - } - } - - const channelAttachments = [ - ...(processedRequest.email?.attachments ?? []), - ...(processedRequest.sms?.attachments ?? []), - ...(processedRequest.msgApp?.attachments ?? []), - ] - - if (channelAttachments.length > 0) { - if (!clamavService) { - throw new Error('Attachment scan service unavailable') - } - if (!IngestionWorker.hasAttachmentReferences(channelAttachments)) { - throw new Error('Invalid processed attachment reference payload') - } - if (!attachmentService) { - throw new Error('Attachment service unavailable') - } - - logger.log( - `[${notifyId}] Scanning ${channelAttachments.length} attachment(s) for malware`, - ) - - for (const attachment of channelAttachments) { - try { - const downloadedAttachment = - await attachmentService.downloadAttachmentByIdAndTenantId( - attachment.attachmentId, - tenantId, - ) - const buffer = downloadedAttachment.content - const attachmentFilename = downloadedAttachment.filename - - logger.debug( - `[${notifyId}] Scanning attachment: ${JSON.stringify({ - tenantId, - attachmentId: attachment.attachmentId, - filename: attachmentFilename, - sizeBytes: buffer.length, - })}`, - ) - - // Scan buffer for malware using CLAMD protocol - const scanResult = await clamavService.scanBuffer(buffer, attachmentFilename) - - if (scanResult.isInfected) { - // Malware detected - quarantine the notification - logger.warn( - `[${notifyId}] SECURITY: Malware detected in attachment: ${JSON.stringify({ - tenantId, - attachmentId: attachment.attachmentId, - filename: attachmentFilename, - viruses: scanResult.quarantineInfo?.viruses || [], - })}`, - ) - - // Create quarantine details from scan result - const quarantineDetails: QuarantineDetails = { - viruses: scanResult.quarantineInfo?.viruses || [], - filename: attachmentFilename, - scannedAt: scanResult.quarantineInfo?.scannedAt - ? scanResult.quarantineInfo.scannedAt.toISOString() - : new Date().toISOString(), - reason: 'Malware detected during attachment scanning', - } - - // Update notification to QUARANTINED status - await notificationService.update(notifyId, tenantId, { - status: NotificationStatus.QUARANTINED, - quarantineDetails, - updatedBy: 'ingestion-worker-scanner', - }) - await requestDetailService.updateStatus(notifyId, NotificationStatus.QUARANTINED) - - logger.log( - `[${notifyId}] Notification marked as QUARANTINED due to malware detection`, - ) - - // Stop processing - do not queue delivery jobs - return { success: false, reason: 'Quarantined due to malware detection' } - } - - logger.debug( - `[${notifyId}] Attachment ${attachmentFilename || 'unnamed'} passed malware scan`, - ) - } catch (scanError) { - const errorMsg = scanError instanceof Error ? scanError.message : String(scanError) - logger.error( - `[${notifyId}] Attachment scan error: ${errorMsg}. Strict mode: retrying job.`, - ) - // Strict mode: throw error to trigger BullMQ retry if ClamAV is unavailable - throw new Error(`Attachment scan failed: ${errorMsg}`, { cause: scanError }) - } - } - - logger.log(`[${notifyId}] All attachments passed malware scanning`) - } - - // Determine channels and fan-out to delivery queues - const deliveryJobs: Array<{ - queue: Bull.Queue - channel: NotificationChannel - payload: any - }> = [] - - // Email channel - if (processedRequest.email) { - logger.log(`[${notifyId}] Adding email delivery job`) - deliveryJobs.push({ - queue: emailQueue, - channel: NotificationChannel.EMAIL, - payload: processedRequest.email, - }) - } - - // SMS channel - if (processedRequest.sms) { - logger.log(`[${notifyId}] Adding SMS delivery job`) - deliveryJobs.push({ - queue: smsQueue, - channel: NotificationChannel.SMS, - payload: processedRequest.sms, - }) - } - - // Note: At least one channel is guaranteed by validateBusinessRules() - // which runs before this job is queued, so this should never be empty - if (deliveryJobs.length === 0) { - throw new Error( - 'No delivery channels specified - this should have been caught by validateBusinessRules', - ) - } - - // Queue delivery jobs with idempotency and tracing - for (const { queue, channel, payload } of deliveryJobs) { - // On a re-run, a channel whose recipients are all sent or failed is done; queueing it - // again would re-send it while recovering the other channel. - if (resumed && (await requestDetailService.countInFlight(notifyId, channel)) === 0) { - logger.log(`[${notifyId}] Skipping ${channel}: delivered on an earlier run`) - continue - } - - const deliveryPayload: DeliveryJobPayload = { - notifyId, - tenantId, - channel, - request: processedRequest, - payload, - attempt: 0, - } - - // Add job with notifyId_channel as key for idempotency - // (prevents duplicate delivery if ingestion job is retried) - const jobKey = `${notifyId}_${channel}` - - // Calculate delay for scheduled sends - const delay = scheduledFor ? new Date(scheduledFor).getTime() - Date.now() : 0 - const isScheduled = delay > 0 - - await queue.add(deliveryPayload, { - jobId: jobKey, - delay: Math.max(0, delay), // BullMQ ignores negative delays - removeOnComplete: true, - removeOnFail: FAILED_JOB_RETENTION, - attempts: 3, // Retry up to 3 times - backoff: { - type: 'exponential', - delay: 2000, // Start with 2s, exponential backoff - }, - }) - - const scheduleInfo = isScheduled - ? ` (scheduled for ${new Date(scheduledFor).toISOString()})` - : '' - - logger.log( - `[${notifyId}] Queued delivery job (channel=${channel}, key=${jobKey})${scheduleInfo}`, - ) - } - - logger.log( - `[${notifyId}] Successfully processed ingestion job, channels=${deliveryJobs.map((d) => d.channel).join(',')}`, - ) + this.logger.log(`[${notifyId}] All attachments passed malware scanning`) + return false + } - // Update notification_request status to PROCESSING in database - await notificationService.update(notifyId, tenantId, { - status: NotificationStatus.PROCESSING, - updatedBy: 'ingestion-worker', - }) - - // Not on a re-run: some rows are already sent or failed, and a blanket update would mark - // them in flight again - and the reconciler would then send them twice. - if (!resumed) { - await requestDetailService.updateStatus(notifyId, NotificationStatus.PROCESSING) - } - - return { success: true, deliveryJobsQueued: deliveryJobs.length } - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error) - logger.error( - `[${notifyId}] Failed to process ingestion job: ${errorMessage}`, - error instanceof Error ? error.stack : '', - ) + /** Scan one attachment; the quarantine record when it is infected, else null. */ + private async scanAttachment( + notifyId: string, + tenantId: string, + attachmentId: string, + ): Promise { + try { + const { content, filename } = await this.attachmentService!.downloadAttachmentByIdAndTenantId( + attachmentId, + tenantId, + ) + this.logger.debug( + `[${notifyId}] Scanning attachment: ${JSON.stringify({ + tenantId, + attachmentId, + filename, + sizeBytes: content.length, + })}`, + ) - // Update notification_request status to FAILED in database - await notificationService.update(notifyId, tenantId, { - status: NotificationStatus.FAILED, - updatedBy: 'ingestion-worker', - }) - await requestDetailService.updateStatus(notifyId, NotificationStatus.FAILED, { - preserveCompleted: resumed, - }) - - // Re-throw to trigger BullMQ retry logic - throw error + const scanResult = await this.clamavService!.scanBuffer(content, filename) + if (!scanResult.isInfected) { + this.logger.debug(`[${notifyId}] Attachment ${filename || 'unnamed'} passed malware scan`) + return null } - }) - logger.log('Ingestion worker processor registered successfully') + const viruses = scanResult.quarantineInfo?.viruses || [] + this.logger.warn( + `[${notifyId}] SECURITY: Malware detected in attachment: ${JSON.stringify({ + tenantId, + attachmentId, + filename, + viruses, + })}`, + ) + return { + viruses, + filename, + scannedAt: scanResult.quarantineInfo?.scannedAt + ? scanResult.quarantineInfo.scannedAt.toISOString() + : new Date().toISOString(), + reason: 'Malware detected during attachment scanning', + } + } catch (scanError) { + const errorMsg = scanError instanceof Error ? scanError.message : String(scanError) + this.logger.error( + `[${notifyId}] Attachment scan error: ${errorMsg}. Strict mode: retrying job.`, + ) + throw new Error(`Attachment scan failed: ${errorMsg}`, { cause: scanError }) + } + } - // Event listeners for job lifecycle - ingestionQueue.on('completed', (job: Bull.Job) => { - const { notifyId } = job.data - logger.debug(`[${notifyId}] Ingestion job completed`) + private async quarantine( + notifyId: string, + tenantId: string, + quarantineDetails: QuarantineDetails, + ): Promise { + await this.notificationService.update(notifyId, tenantId, { + status: NotificationStatus.QUARANTINED, + quarantineDetails, + updatedBy: 'ingestion-worker-scanner', }) + await this.requestDetailService.updateStatus(notifyId, NotificationStatus.QUARANTINED) + this.logger.log(`[${notifyId}] Notification marked as QUARANTINED due to malware detection`) + } - ingestionQueue.on('failed', (job: Bull.Job, err: Error) => { - const { notifyId } = job.data - logger.error( - `[${notifyId}] Ingestion job failed (attempt ${job.attemptsMade}/${job.opts.attempts}): error=${err.message}`, + /** Queue one delivery job per requested channel; the number queued. */ + private async fanOut( + { notifyId, tenantId, scheduledFor }: IngestionJobPayload, + request: IngestionRequest, + resumed: boolean, + ): Promise { + const deliveries: Array<{ + queue: Queue + channel: NotificationChannel + payload: any + }> = [] + if (request.email) { + this.logger.log(`[${notifyId}] Adding email delivery job`) + deliveries.push({ + queue: this.emailQueue, + channel: NotificationChannel.EMAIL, + payload: request.email, + }) + } + if (request.sms) { + this.logger.log(`[${notifyId}] Adding SMS delivery job`) + deliveries.push({ + queue: this.smsQueue, + channel: NotificationChannel.SMS, + payload: request.sms, + }) + } + // validateBusinessRules() guarantees a channel before the request is queued. + if (deliveries.length === 0) { + throw new Error( + 'No delivery channels specified - this should have been caught by validateBusinessRules', ) - }) + } + + // Calculated for scheduled sends; the queue ignores a delay of 0. + const delay = scheduledFor ? Math.max(0, new Date(scheduledFor).getTime() - Date.now()) : 0 + const scheduleInfo = + delay > 0 ? ` (scheduled for ${new Date(scheduledFor!).toISOString()})` : '' + + // Email and SMS are independent, so they are queued together. + const queueChannel = async ({ queue, channel, payload }: (typeof deliveries)[number]) => { + // On a re-run, a channel whose recipients are all sent or failed is done; queueing it + // again would re-send it while recovering the other channel. + if (resumed && (await this.requestDetailService.countInFlight(notifyId, channel)) === 0) { + this.logger.log(`[${notifyId}] Skipping ${channel}: delivered on an earlier run`) + return + } - logger.log('Ingestion worker initialized') + // notifyId_channel: a retried ingestion job cannot queue the same delivery twice. + const jobKey = `${notifyId}_${channel}` + const deliveryPayload: DeliveryJobPayload = { + notifyId, + tenantId, + channel, + request, + payload, + attempt: 0, + } + await queue.add(queue.name, deliveryPayload, { + jobId: jobKey, + delay, + ...DELIVERY_JOB_OPTIONS, + }) + + this.logger.log( + `[${notifyId}] Queued delivery job (channel=${channel}, key=${jobKey})${scheduleInfo}`, + ) + } + await Promise.all(deliveries.map(queueChannel)) + + this.logger.log( + `[${notifyId}] Successfully processed ingestion job, channels=${deliveries.map((d) => d.channel).join(',')}`, + ) + return deliveries.length } } diff --git a/backend/src/queue/workers/merge-batch-runner.spec.ts b/backend/src/queue/workers/merge-batch-runner.spec.ts new file mode 100644 index 000000000..8fb741f55 --- /dev/null +++ b/backend/src/queue/workers/merge-batch-runner.spec.ts @@ -0,0 +1,112 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { Logger } from '@nestjs/common' +import type { Job } from 'bullmq' +import { MergeBatchRunner, settledMergeStatus } from './merge-batch-runner' +import { EMAIL_CHANNEL, SMS_CHANNEL } from './delivery-channel' +import { NotificationStatus } from '../../enum/notification-status.enum' +import type { DeliveryJobPayload } from '../queue.types' +import type { NotificationService } from '../../api/notification/notification.service' +import type { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' +import type { TemplatesRepository } from '../../api/templates/templates.repository' + +describe('settledMergeStatus', () => { + it('waits while any recipient is pending', () => { + expect(settledMergeStatus({ pending: 1, sent: 9 })).toBeNull() + }) + + it.each([ + [{ sent: 3 }, NotificationStatus.COMPLETED], + [{ sent: 3, blocked: 1 }, NotificationStatus.COMPLETED], + [{ sent: 2, failed: 1 }, NotificationStatus.PARTIALLY_COMPLETED], + [{ failed: 3 }, NotificationStatus.FAILED], + [{}, NotificationStatus.COMPLETED], + ])('settles %o as %s', (counts, status) => { + expect(settledMergeStatus(counts)).toBe(status) + }) +}) + +describe('MergeBatchRunner', () => { + const notificationService = { update: vi.fn() } + const requestDetailService = { + countByStatuses: vi.fn(), + findBatchRecipients: vi.fn(), + markRecipientSent: vi.fn(), + markRecipientFailed: vi.fn(), + markBatchUnsentFailed: vi.fn(), + } + const runner = new MergeBatchRunner( + notificationService as unknown as NotificationService, + requestDetailService as unknown as NotificationRequestDetailService, + { findById: vi.fn() } as unknown as TemplatesRepository, + ) + + beforeEach(() => { + vi.clearAllMocks() + vi.spyOn(Logger.prototype, 'log').mockImplementation(() => {}) + vi.spyOn(Logger.prototype, 'error').mockImplementation(() => {}) + }) + + it('settles the parent from one grouped count, attributed to the channel', async () => { + requestDetailService.countByStatuses.mockResolvedValue({ sent: 4, failed: 1 }) + + await runner.settle('notify-1', 'tenant-1', SMS_CHANNEL) + + expect(requestDetailService.countByStatuses).toHaveBeenCalledTimes(1) + expect(notificationService.update).toHaveBeenCalledWith('notify-1', 'tenant-1', { + status: NotificationStatus.PARTIALLY_COMPLETED, + updatedBy: 'sms-delivery-worker', + }) + }) + + it('records a failed recipient and carries on with the rest of the batch', async () => { + requestDetailService.findBatchRecipients.mockResolvedValue([ + { address: 'a@example.com', params: {}, status: 'pending' }, + { address: 'b@example.com', params: {}, status: 'pending' }, + { address: 'c@example.com', params: {}, status: 'sent' }, + ]) + requestDetailService.countByStatuses.mockResolvedValue({ sent: 2, failed: 1 }) + const send = vi + .fn() + .mockRejectedValueOnce(new Error('mailbox unavailable')) + .mockResolvedValueOnce('msg-2') + const job = { + id: 'notify-1-EMAIL-0', + data: { + notifyId: 'notify-1', + tenantId: 'tenant-1', + mailMerge: true, + batchId: 'notify-1-EMAIL-0', + mailMergeData: { + content: { subject: 'Hi', body: 'Hello {{name}}' }, + params: { name: 'all' }, + }, + }, + updateProgress: vi.fn().mockResolvedValue(undefined), + } as unknown as Job + + const result = await runner.run(job, { + channel: EMAIL_CHANNEL, + hasInlineContent: () => true, + send, + }) + + // The recipient delivered on an earlier attempt is skipped, not sent again. + expect(send).toHaveBeenCalledTimes(2) + expect(send).toHaveBeenCalledWith( + expect.objectContaining({ address: 'b@example.com' }), + { name: 'all' }, + { inline: expect.objectContaining({ renderer: 'handlebars' }) }, + ) + expect(requestDetailService.markRecipientFailed).toHaveBeenCalledWith( + 'notify-1', + 'notify-1-EMAIL-0', + 'a@example.com', + 'mailbox unavailable', + ) + expect(result).toEqual({ success: false, batchId: 'notify-1-EMAIL-0', sent: 1, failed: 1 }) + expect(notificationService.update).toHaveBeenCalledWith('notify-1', 'tenant-1', { + status: NotificationStatus.PARTIALLY_COMPLETED, + updatedBy: 'email-delivery-worker', + }) + }) +}) diff --git a/backend/src/queue/workers/merge-batch-runner.ts b/backend/src/queue/workers/merge-batch-runner.ts new file mode 100644 index 000000000..3246ec0f9 --- /dev/null +++ b/backend/src/queue/workers/merge-batch-runner.ts @@ -0,0 +1,246 @@ +import { Injectable, Logger, NotFoundException, Optional } from '@nestjs/common' +import { describeError } from '../../common/utils/describe-error' +import type { Job } from 'bullmq' +import { DeliveryJobPayload, MailMergeJobData } from '../queue.types' +import { batchProgressReporter } from '../batch-progress' +import { loadBatchRecipients } from './merge-batch-recipients' +import { capitalize, DeliveryChannel } from './delivery-channel' +import { isTransientDeliveryError } from '../../adapters/delivery-errors' +import { NotificationService } from '../../api/notification/notification.service' +import { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' +import { TemplatesRepository } from '../../api/templates/templates.repository' +import type { Template } from '../../api/templates/entities/template.entity' +import { NotificationStatus } from '../../enum/notification-status.enum' +import { StructuredLoggerService } from '../../common/logger' + +export type MergeRecipient = NonNullable[number] + +/** Inline merge content, with the renderer defaulted so per-recipient variables are substituted. */ +export type InlineMergeContent = NonNullable & { + renderer: NonNullable['renderer']> +} + +/** What a merge batch renders from: exactly one of a server template or inline content. */ +export type MergeSource = + | { template: Template; inline?: never } + | { template?: never; inline: InlineMergeContent } + +/** The channel-specific half of sending a merge batch. */ +export interface MergeSender { + channel: DeliveryChannel + /** Whether the merge's content carries inline content of this channel's shape. */ + hasInlineContent(content: MailMergeJobData['content']): boolean + /** Render for and send to one recipient; resolves the provider's id for the message. */ + send( + recipient: MergeRecipient, + params: Record, + source: MergeSource, + ): Promise +} + +/** A job carrying one batch of a merge rather than a plain send. */ +export function isMergeBatch(data: DeliveryJobPayload): boolean { + return !!(data.mailMerge && data.mailMergeData && data.batchId) +} + +export interface MergeBatchResult { + success: boolean + batchId: string + sent: number + failed: number +} + +/** + * Sends one batch of a merge and settles the parent request, for every delivery channel. + * + * A batch's recipients are sent one at a time. A failed recipient is recorded and skipped rather + * than thrown, so one bad row cannot strand those already delivered; a transient provider error + * stops the batch and leaves the rest owed; only systemic errors (a missing template) throw to + * retry the whole batch. A retried batch starts from the top, skipping addresses an earlier + * attempt delivered. + */ +@Injectable() +export class MergeBatchRunner { + private readonly logger = new Logger(MergeBatchRunner.name) + + constructor( + private readonly notificationService: NotificationService, + private readonly requestDetailService: NotificationRequestDetailService, + private readonly templatesRepository: TemplatesRepository, + @Optional() private readonly structuredLogger?: StructuredLoggerService, + ) {} + + async run(job: Job, sender: MergeSender): Promise { + const { notifyId, tenantId, batchId, mailMergeData } = job.data as DeliveryJobPayload & { + batchId: string + mailMergeData: MailMergeJobData + } + const { mergeLabel } = sender.channel + const { recipients, alreadySent } = await loadBatchRecipients( + this.requestDetailService, + notifyId, + batchId, + mailMergeData.recipients, + ) + const source = await this.resolveSource(tenantId, mailMergeData.content, sender) + + this.logger.debug( + `[${notifyId}] Processing ${mergeLabel} batch ${batchId}: ${recipients.length} recipient(s)`, + ) + if (alreadySent.size > 0) { + this.logger.log( + `[${notifyId}] Batch ${batchId}: skipping ${alreadySent.size} recipient(s) delivered on an earlier attempt`, + ) + } + const pending = recipients.filter((recipient) => !alreadySent.has(recipient.address)) + + const reportProgress = batchProgressReporter(job) + let sent = 0 + let failed = 0 + // Recipients delivered by an earlier attempt count as sent, so a retried batch resumes its bar. + const progress = () => ({ sent: alreadySent.size + sent, failed, total: recipients.length }) + reportProgress(progress()) + + // One at a time on purpose: progress is per recipient, and a transient error must stop the + // batch before anyone after it is attempted. + for (const recipient of pending) { + const delivered = await this.sendOne(job.data, recipient, source, sender, sent) + if (delivered) sent++ + else failed++ + reportProgress(progress()) + } + + this.logger.log( + `[${notifyId}] ${capitalize(mergeLabel)} batch ${batchId} complete: sent=${sent}, failed=${failed}`, + ) + await this.settle(notifyId, tenantId, sender.channel) + + return { success: failed === 0, batchId, sent, failed } + } + + /** + * Fail what one batch still owes - never its sent rows or another batch's - then settle the + * parent request. For a batch that can never succeed, or whose retries are spent; the caller + * fails the job so it is not retried. + */ + async failUnsent( + job: Job, + errorMessage: string, + channel: DeliveryChannel, + ): Promise { + const { notifyId, tenantId, batchId } = job.data + await this.requestDetailService.markBatchUnsentFailed(notifyId, batchId!, errorMessage) + await this.settle(notifyId, tenantId, channel) + this.logger.error(`[${notifyId}] Batch ${batchId} failed; its unsent recipients are FAILED`) + } + + /** + * Settle the parent request once no recipient is pending across all of its batches: + * all sent → COMPLETED; some sent and some failed → PARTIALLY_COMPLETED; all failed → FAILED. + * Anything short of COMPLETED is logged as one delivery failure for the whole merge, which is + * what the channel's sending-failures alert counts. + */ + async settle(notifyId: string, tenantId: string, channel: DeliveryChannel): Promise { + const counts = await this.requestDetailService.countByStatuses(notifyId) + const status = settledMergeStatus(counts) + if (!status) return + + await this.notificationService.update(notifyId, tenantId, { + status, + updatedBy: channel.updatedBy, + }) + this.logger.log( + `[${notifyId}] All ${channel.mergeLabel} batches complete; parent marked ${status.toUpperCase()}`, + ) + if (status !== NotificationStatus.COMPLETED) { + const failed = counts.failed ?? 0 + this.structuredLogger?.logNotificationFailure( + notifyId, + tenantId, + channel.channel, + `${failed} of ${failed + (counts.sent ?? 0)} merge recipients failed`, + undefined, + channel.context, + ) + } + } + + private async resolveSource( + tenantId: string, + content: MailMergeJobData['content'], + sender: MergeSender, + ): Promise { + const label = capitalize(sender.channel.mergeLabel) + const templateId = content?.templateId + const hasInlineContent = sender.hasInlineContent(content) + + if (templateId && hasInlineContent) { + throw new Error(`${label} requires either a templateId or inline content, not both`) + } + if (!templateId) { + if (!hasInlineContent) { + throw new Error(`${label} requires either a templateId or inline content`) + } + return { inline: { ...content!, renderer: content!.renderer ?? 'handlebars' } } + } + + const template = await this.templatesRepository.findById(tenantId, templateId) + if (!template) { + throw new NotFoundException(`Template '${templateId}' not found for tenant '${tenantId}'`) + } + const { templateCode } = sender.channel + if (template.channelCode !== templateCode) { + throw new Error(`Template '${templateId}' is not an ${templateCode} template`) + } + return { template } + } + + /** Send to one recipient and record the outcome; false when that recipient failed. */ + private async sendOne( + { notifyId, batchId, mailMergeData }: DeliveryJobPayload, + recipient: MergeRecipient, + source: MergeSource, + sender: MergeSender, + sentSoFar: number, + ): Promise { + // Per-recipient params take precedence over the global ones. + const params = { ...mailMergeData?.params, ...recipient.params } + try { + const externalId = await sender.send(recipient, params, source) + await this.requestDetailService.markRecipientSent( + notifyId, + batchId!, + recipient.address, + externalId, + ) + return true + } catch (error) { + // The provider, not this recipient: stop the batch and leave the rest owed. The job is + // retried, skipping whoever was sent, and the circuit breaker holds it until it is back. + if (isTransientDeliveryError(error)) { + this.logger.warn( + `[${notifyId}] Batch ${batchId} paused after ${sentSoFar} sent: ${error.message}`, + ) + throw error + } + const errorMessage = describeError(error) + this.logger.error( + `[${notifyId}] ${capitalize(sender.channel.mergeLabel)} recipient send failed (batch=${batchId}): ${errorMessage}`, + ) + await this.requestDetailService.markRecipientFailed( + notifyId, + batchId!, + recipient.address, + errorMessage, + ) + return false + } + } +} + +/** The parent status a merge settles at, or null while any recipient is still pending. */ +export function settledMergeStatus(counts: Record): NotificationStatus | null { + if (counts.pending) return null + if (!counts.failed) return NotificationStatus.COMPLETED + return counts.sent ? NotificationStatus.PARTIALLY_COMPLETED : NotificationStatus.FAILED +} diff --git a/backend/src/queue/workers/queue-processor.ts b/backend/src/queue/workers/queue-processor.ts new file mode 100644 index 000000000..36bcb3413 --- /dev/null +++ b/backend/src/queue/workers/queue-processor.ts @@ -0,0 +1,12 @@ +import type { Job } from 'bullmq' +import type { QueueName } from '../../enum/queue-name.enum' + +/** A worker QueueModule runs against one queue, at the concurrency config gives it. */ +export interface QueueProcessor { + readonly queueName: QueueName + /** Key under `queue.` in configuration.ts, and the default when it is unset. */ + readonly concurrency: { configKey: string; default: number } + process(job: Job): Promise + onCompleted?(job: Job): void + onFailed?(job: Job, error: Error): void | Promise +} diff --git a/backend/src/queue/workers/sms-delivery.worker.spec.ts b/backend/src/queue/workers/sms-delivery.worker.spec.ts index 2d5d2f532..e7b3ae489 100644 --- a/backend/src/queue/workers/sms-delivery.worker.spec.ts +++ b/backend/src/queue/workers/sms-delivery.worker.spec.ts @@ -1,23 +1,70 @@ import { BadRequestException, Logger } from '@nestjs/common' -import Bull from 'bull' +import type { Job } from 'bullmq' import { vi, describe, it, expect, beforeEach, afterEach } from 'vitest' +import { UnrecoverableError } from 'bullmq' import { SmsDeliveryWorker } from './sms-delivery.worker' import { DeliveryJobPayload } from '../queue.types' import { NotificationChannel } from '../../enum/notification-channel.enum' import { NotificationStatus } from '../../enum/notification-status.enum' import { ISmsTransport } from '../../adapters' import { TransientDeliveryError } from '../../adapters/delivery-errors' +import { MergeBatchRunner } from './merge-batch-runner' +import { DeliveryFailureHandler } from './delivery-failure-handler' +import { PhoneNumberService } from '../../api/notify/services/phone-number.service' +import type { NotificationService } from '../../api/notification/notification.service' +import type { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' +import type { TemplatesRepository } from '../../api/templates/templates.repository' +import type { TemplatesService } from '../../api/templates/templates.service' +import type { InlineRenderingService } from '../../services/rendering/inline-rendering.service' +import type { StructuredLoggerService } from '../../common/logger' describe('SmsDeliveryWorker', () => { - let mockSmsQueue: Partial> let mockNotificationService: any let mockTemplatesRepository: any let mockTemplatesService: any let mockInlineRenderingService: any let mockSmsAdapter: ISmsTransport let mockRequestDetailService: any - let processHandler: (job: Bull.Job) => Promise - let failedCallback: (job: Bull.Job, err: Error) => void + let processHandler: (job: Job) => Promise + let failedCallback: (job: Job, err: Error) => Promise + + /** The worker as QueueModule builds it, with its handlers captured for the tests to call. */ + function initialize(deps: { + notificationService: NotificationService + templatesRepository: TemplatesRepository + templatesService: TemplatesService + inlineRenderingService: InlineRenderingService + smsAdapter: ISmsTransport + requestDetailService: NotificationRequestDetailService + concurrency?: number + structuredLogger?: StructuredLoggerService + }) { + const mergeBatches = new MergeBatchRunner( + deps.notificationService, + deps.requestDetailService, + deps.templatesRepository, + deps.structuredLogger, + ) + const worker = new SmsDeliveryWorker( + deps.notificationService, + deps.requestDetailService, + deps.templatesRepository, + deps.templatesService, + deps.inlineRenderingService, + deps.smsAdapter, + new PhoneNumberService(), + mergeBatches, + new DeliveryFailureHandler( + deps.notificationService, + deps.requestDetailService, + mergeBatches, + deps.structuredLogger, + ), + deps.structuredLogger, + ) + processHandler = (job) => worker.process(job) + failedCallback = (job, err) => worker.onFailed(job, err) + } beforeEach(() => { // Mock the SMS adapter @@ -67,26 +114,11 @@ describe('SmsDeliveryWorker', () => { markRecipientSent: vi.fn().mockResolvedValue(undefined), markRecipientFailed: vi.fn().mockResolvedValue(undefined), markBatchUnsentFailed: vi.fn().mockResolvedValue(undefined), - countByStatus: vi.fn().mockResolvedValue(1), + countByStatuses: vi.fn().mockResolvedValue({ pending: 1 }), updateStatus: vi.fn().mockResolvedValue(undefined), findSentAddresses: vi.fn().mockResolvedValue(new Set()), } - // Mock the SMS queue - mockSmsQueue = { - process: vi.fn().mockImplementation((...args) => { - // Handle: process(concurrency, handler) - const handler = typeof args[0] === 'function' ? args[0] : args[1] - processHandler = handler - }), - on: vi.fn().mockImplementation((event, callback) => { - if (event === 'failed') { - failedCallback = callback - } - // Don't capture 'completed' event - only tracking failures in tests - }), - } - vi.spyOn(Logger.prototype, 'log').mockImplementation(() => {}) vi.spyOn(Logger.prototype, 'debug').mockImplementation(() => {}) vi.spyOn(Logger.prototype, 'error').mockImplementation(() => {}) @@ -96,37 +128,6 @@ describe('SmsDeliveryWorker', () => { vi.clearAllMocks() }) - describe('initialize', () => { - it('should register a process handler on the SMS queue', () => { - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, - notificationService: mockNotificationService, - templatesRepository: mockTemplatesRepository, - templatesService: mockTemplatesService, - inlineRenderingService: mockInlineRenderingService, - smsAdapter: mockSmsAdapter, - requestDetailService: mockRequestDetailService, - }) - - expect(mockSmsQueue.process).toHaveBeenCalled() - }) - - it('should register event listeners on the SMS queue', () => { - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, - notificationService: mockNotificationService, - templatesRepository: mockTemplatesRepository, - templatesService: mockTemplatesService, - inlineRenderingService: mockInlineRenderingService, - smsAdapter: mockSmsAdapter, - requestDetailService: mockRequestDetailService, - }) - - expect(mockSmsQueue.on).toHaveBeenCalledWith('completed', expect.any(Function)) - expect(mockSmsQueue.on).toHaveBeenCalledWith('failed', expect.any(Function)) - }) - }) - describe('process handler', () => { it('records each recipient separately when the transport reports a partial failure', async () => { mockSmsAdapter.send = vi.fn().mockResolvedValue({ @@ -138,8 +139,7 @@ describe('SmsDeliveryWorker', () => { ], }) - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -148,7 +148,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-partial', tenantId: 'tenant-123', @@ -166,7 +166,7 @@ describe('SmsDeliveryWorker', () => { // The job must not fail: throwing here is what made the queue retry and re-send to the // recipient who had already received the message. - await expect(processHandler(job as Bull.Job)).resolves.toMatchObject({ + await expect(processHandler(job as Job)).resolves.toMatchObject({ success: true, }) @@ -196,8 +196,7 @@ describe('SmsDeliveryWorker', () => { .fn() .mockResolvedValue({ messageId: 'SM1', providerResponse: 'sent' }) - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -206,7 +205,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-plain', tenantId: 'tenant-123', @@ -222,7 +221,7 @@ describe('SmsDeliveryWorker', () => { attemptsMade: 0, } - await processHandler(job as Bull.Job) + await processHandler(job as Job) expect(mockRequestDetailService.markSent).toHaveBeenCalled() expect(mockNotificationService.update).toHaveBeenCalledWith( @@ -232,9 +231,8 @@ describe('SmsDeliveryWorker', () => { ) }) - it('skips recipients an earlier run delivered even when Bull did not count it as a retry', async () => { - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + it('skips recipients an earlier run delivered even when BullMQ did not count it as a retry', async () => { + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -245,7 +243,7 @@ describe('SmsDeliveryWorker', () => { mockRequestDetailService.findSentAddresses.mockResolvedValue(new Set(['+16135551234'])) // attemptsMade 0: a stalled job re-run after its pod stopped mid-send. - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-stalled', tenantId: 'tenant-123', @@ -261,7 +259,7 @@ describe('SmsDeliveryWorker', () => { attemptsMade: 0, } - await processHandler(job as Bull.Job) + await processHandler(job as Job) expect(mockRequestDetailService.resetForRetry).not.toHaveBeenCalled() expect(mockSmsAdapter.send).toHaveBeenCalledTimes(1) @@ -271,8 +269,7 @@ describe('SmsDeliveryWorker', () => { }) it('should successfully send SMS and update status', async () => { - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -281,7 +278,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-sms-123', tenantId: 'tenant-123', @@ -303,7 +300,7 @@ describe('SmsDeliveryWorker', () => { attemptsMade: 0, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result).toEqual({ success: true, @@ -329,8 +326,7 @@ describe('SmsDeliveryWorker', () => { }) it('should throw error when SMS payload is missing', async () => { - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -339,7 +335,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', @@ -358,7 +354,7 @@ describe('SmsDeliveryWorker', () => { attemptsMade: 0, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid delivery job: SMS payload is missing or invalid', ) @@ -366,8 +362,7 @@ describe('SmsDeliveryWorker', () => { }) it('should throw error when recipient phone is missing', async () => { - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -376,7 +371,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', @@ -398,14 +393,13 @@ describe('SmsDeliveryWorker', () => { attemptsMade: 0, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid SMS payload: recipient phone number is missing or invalid', ) }) it('should permanently fail a malformed recipient before calling the SMS provider', async () => { - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -414,7 +408,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-invalid-recipient', tenantId: 'tenant-123', @@ -428,18 +422,17 @@ describe('SmsDeliveryWorker', () => { } as DeliveryJobPayload, opts: { attempts: 3 } as any, attemptsMade: 0, - discard: vi.fn(), } const expectedError = 'Invalid SMS payload: recipient phone number at index 1 is not valid E.164' - await expect(processHandler(job as Bull.Job)).rejects.toThrow( - expectedError, - ) + const failure = processHandler(job as Job) + await expect(failure).rejects.toThrow(expectedError) expect(mockSmsAdapter.send).not.toHaveBeenCalled() expect(mockTemplatesRepository.findById).not.toHaveBeenCalled() - expect(job.discard).toHaveBeenCalledTimes(1) + // Failed outright rather than retried. + await expect(failure).rejects.toBeInstanceOf(UnrecoverableError) expect(job.attemptsMade).toBe(0) expect(mockRequestDetailService.resetForRetry).not.toHaveBeenCalled() expect(mockNotificationService.update).toHaveBeenCalledTimes(1) @@ -459,8 +452,7 @@ describe('SmsDeliveryWorker', () => { }) it('should throw error when SMS body is missing', async () => { - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -469,7 +461,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', @@ -490,7 +482,7 @@ describe('SmsDeliveryWorker', () => { attemptsMade: 0, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid SMS payload: body is missing or invalid', ) }) @@ -506,8 +498,7 @@ describe('SmsDeliveryWorker', () => { bodyType: 'markdown', }) - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -516,7 +507,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-sms-template', tenantId: 'tenant-123', @@ -534,7 +525,7 @@ describe('SmsDeliveryWorker', () => { attemptsMade: 0, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result.success).toBe(true) expect(mockTemplatesService.renderTemplateContent).toHaveBeenCalledWith( @@ -561,8 +552,7 @@ describe('SmsDeliveryWorker', () => { bodyType: 'markdown', }) - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -571,7 +561,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-sms-template-params', tenantId: 'tenant-123', @@ -590,7 +580,7 @@ describe('SmsDeliveryWorker', () => { attemptsMade: 0, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result.success).toBe(true) // Channel-level param wins per-key; non-overlapping global param is retained. @@ -605,8 +595,7 @@ describe('SmsDeliveryWorker', () => { body: 'Rendered SMS Body', }) - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -615,7 +604,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-sms-inline-params', tenantId: 'tenant-123', @@ -634,7 +623,7 @@ describe('SmsDeliveryWorker', () => { attemptsMade: 0, } - const result = await processHandler(job as Bull.Job) + const result = await processHandler(job as Job) expect(result.success).toBe(true) expect(mockTemplatesService.renderTemplateContent).not.toHaveBeenCalled() @@ -655,8 +644,7 @@ describe('SmsDeliveryWorker', () => { new BadRequestException('Missing personalisation for template ID template-sms-uuid: code'), ) - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -665,7 +653,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-sms-template-missing', tenantId: 'tenant-123', @@ -681,14 +669,15 @@ describe('SmsDeliveryWorker', () => { } as any, opts: { attempts: 3 } as any, attemptsMade: 0, - discard: vi.fn(), } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + const failure = processHandler(job as Job) + await expect(failure).rejects.toThrow( 'Missing personalisation for template ID template-sms-uuid: code', ) expect(mockSmsAdapter.send).not.toHaveBeenCalled() - expect(job.discard).toHaveBeenCalledTimes(1) + // Failed outright rather than retried. + await expect(failure).rejects.toBeInstanceOf(UnrecoverableError) expect(mockNotificationService.update).toHaveBeenCalledWith( 'notify-sms-template-missing', 'tenant-123', @@ -704,8 +693,7 @@ describe('SmsDeliveryWorker', () => { }) it('should throw error when notifyId is missing', async () => { - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -714,7 +702,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: undefined as any, @@ -737,14 +725,13 @@ describe('SmsDeliveryWorker', () => { attemptsMade: 0, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid delivery job: notifyId is missing or invalid', ) }) it('should throw error when tenantId is missing', async () => { - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -753,7 +740,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', @@ -776,14 +763,13 @@ describe('SmsDeliveryWorker', () => { attemptsMade: 0, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Invalid delivery job: tenantId is missing or invalid', ) }) it('should throw error when attempt is invalid', async () => { - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -792,7 +778,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-123', @@ -816,14 +802,13 @@ describe('SmsDeliveryWorker', () => { } // Note: attempt field in payload is not validated/used by worker - // Worker uses job.attemptsMade from Bull queue metadata instead - const result = await processHandler(job as Bull.Job) + // Worker uses job.attemptsMade from the job's queue metadata instead + const result = await processHandler(job as Job) expect(result.success).toBe(true) }) it('should mark notification as FAILED on final attempt (attempt 2)', async () => { - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -834,7 +819,7 @@ describe('SmsDeliveryWorker', () => { mockNotificationService.update.mockRejectedValueOnce(new Error('DB Error')) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-fail', @@ -857,7 +842,7 @@ describe('SmsDeliveryWorker', () => { attemptsMade: 0, } - await expect(processHandler(job as Bull.Job)).rejects.toThrow('DB Error') + await expect(processHandler(job as Job)).rejects.toThrow('DB Error') // Verify we attempted the first update to SENDING expect(mockNotificationService.update).toHaveBeenCalledWith( @@ -870,8 +855,7 @@ describe('SmsDeliveryWorker', () => { }) it('should NOT mark notification as FAILED on non-final attempts', async () => { - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -880,7 +864,7 @@ describe('SmsDeliveryWorker', () => { requestDetailService: mockRequestDetailService, }) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-retry', @@ -911,7 +895,7 @@ describe('SmsDeliveryWorker', () => { throw new Error('Network error - should retry') }) - await expect(processHandler(job as Bull.Job)).rejects.toThrow( + await expect(processHandler(job as Job)).rejects.toThrow( 'Network error - should retry', ) @@ -926,8 +910,7 @@ describe('SmsDeliveryWorker', () => { }) it('should call failed callback on job error', async () => { - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -938,7 +921,7 @@ describe('SmsDeliveryWorker', () => { mockNotificationService.update.mockRejectedValueOnce(new Error('Send failed')) - const job: Partial> = { + const job: Partial> = { data: { notifyId: 'notify-error', tenantId: 'tenant-error', @@ -962,22 +945,76 @@ describe('SmsDeliveryWorker', () => { const error = new Error('Send failed') try { - await processHandler(job as Bull.Job) + await processHandler(job as Job) } catch { // Expected } - failedCallback(job as Bull.Job, error) + failedCallback(job as Job, error) // The logger.error should be called expect(Logger.prototype.error).toHaveBeenCalled() }) }) + describe('once retries are spent', () => { + const exhausted = (data: Partial = {}) => + ({ + data: { notifyId: 'notify-spent', tenantId: 'tenant-123', ...data }, + opts: { attempts: 3 }, + attemptsMade: 3, + }) as Job + + beforeEach(() => + initialize({ + notificationService: mockNotificationService, + templatesRepository: mockTemplatesRepository, + templatesService: mockTemplatesService, + inlineRenderingService: mockInlineRenderingService, + smsAdapter: mockSmsAdapter, + requestDetailService: mockRequestDetailService, + }), + ) + + // The same safety net as email: a plain send is never left in SENDING. + it('marks a plain send FAILED', async () => { + await failedCallback(exhausted(), new Error('gateway rejected')) + + expect(mockNotificationService.update).toHaveBeenCalledWith('notify-spent', 'tenant-123', { + status: NotificationStatus.FAILED, + updatedBy: 'system', + errorReason: 'gateway rejected', + }) + expect(mockRequestDetailService.markFailed).toHaveBeenCalledWith( + 'notify-spent', + 'gateway rejected', + ) + }) + + it('leaves a transient failure owed, and a merge batch to its own settlement', async () => { + await failedCallback(exhausted(), new TransientDeliveryError('ACS down', 502)) + await failedCallback( + exhausted({ mailMerge: true, batchId: 'notify-spent-SMS-0' }), + new Error('template gone'), + ) + + expect(mockNotificationService.update).not.toHaveBeenCalled() + expect(mockRequestDetailService.markFailed).not.toHaveBeenCalled() + }) + + it('does nothing while retries remain', async () => { + await failedCallback( + { ...exhausted(), attemptsMade: 1 } as Job, + new Error('x'), + ) + + expect(mockRequestDetailService.markFailed).not.toHaveBeenCalled() + }) + }) + describe('provider outages', () => { const init = () => - SmsDeliveryWorker.initialize({ - smsQueue: mockSmsQueue as Bull.Queue, + initialize({ notificationService: mockNotificationService, templatesRepository: mockTemplatesRepository, templatesService: mockTemplatesService, @@ -1008,8 +1045,7 @@ describe('SmsDeliveryWorker', () => { }, opts: { attempts: 3 }, attemptsMade, - discard: vi.fn(), - }) as unknown as Bull.Job + }) as unknown as Job const plainJob = (attemptsMade = 0) => ({ data: { @@ -1025,7 +1061,7 @@ describe('SmsDeliveryWorker', () => { }, opts: { attempts: 3 }, attemptsMade, - }) as unknown as Bull.Job + }) as unknown as Job beforeEach(() => { mockInlineRenderingService.renderSms = vi.fn().mockResolvedValue({ body: 'Hi A' }) @@ -1087,11 +1123,11 @@ describe('SmsDeliveryWorker', () => { mockTemplatesRepository.findById = vi.fn().mockResolvedValue(null) const job = mergeJob(2) ;(job.data.mailMergeData as { content: object }).content = { templateId: 'gone' } - mockRequestDetailService.countByStatus = vi - .fn() - .mockResolvedValueOnce(0) - .mockResolvedValueOnce(3) - .mockResolvedValueOnce(0) + mockRequestDetailService.countByStatuses = vi.fn().mockResolvedValueOnce({ + pending: 0, + failed: 3, + sent: 0, + }) await init() await expect(processHandler(job)).rejects.toThrow('not found') diff --git a/backend/src/queue/workers/sms-delivery.worker.ts b/backend/src/queue/workers/sms-delivery.worker.ts index 0556a9d59..5c93b8f72 100644 --- a/backend/src/queue/workers/sms-delivery.worker.ts +++ b/backend/src/queue/workers/sms-delivery.worker.ts @@ -1,626 +1,301 @@ -import { BadRequestException, HttpException, Logger, NotFoundException } from '@nestjs/common' -import Bull from 'bull' -import { DeliveryJobPayload, MailMergeJobData } from '../queue.types' -import { batchProgressReporter, type BatchProgressReporter } from '../batch-progress' -import { loadBatchRecipients } from './merge-batch-recipients' -import { TransientDeliveryError, isTransientDeliveryError } from '../../adapters/delivery-errors' +import { + BadRequestException, + Inject, + Injectable, + Logger, + NotFoundException, + Optional, +} from '@nestjs/common' +import type { Job } from 'bullmq' +import { DeliveryJobPayload } from '../queue.types' +import { SMS_CHANNEL } from './delivery-channel' +import { DeliveryFailureHandler } from './delivery-failure-handler' +import { MergeBatchRunner, MergeSender } from './merge-batch-runner' +import { DeliveryWorker } from './delivery-worker' +import { TransientDeliveryError } from '../../adapters/delivery-errors' +import { QueueName } from '../../enum/queue-name.enum' import { NotificationService } from '../../api/notification/notification.service' import { NotificationRequestDetailService } from '../../api/notification/notification-request-detail.service' import { TemplatesRepository } from '../../api/templates/templates.repository' import { TemplatesService } from '../../api/templates/templates.service' import { InlineRenderingService } from '../../services/rendering/inline-rendering.service' import { NotificationStatus } from '../../enum/notification-status.enum' -import { ISmsTransport, SmsRecipientResult } from '../../adapters' +import { ISmsTransport, SMS_ADAPTER, SmsRecipientResult } from '../../adapters' import { StructuredLoggerService } from '../../common/logger' import { PhoneNumberService } from '../../api/notify/services/phone-number.service' -export interface SmsDeliveryWorkerDeps { - /** The Bull queue instance for SMS delivery jobs */ - smsQueue: Bull.Queue - /** Service for database updates */ - notificationService: NotificationService - /** Repository for template resolution */ - templatesRepository: TemplatesRepository - /** Service for template rendering */ - templatesService: TemplatesService - /** Service for inline template rendering */ - inlineRenderingService: InlineRenderingService - /** SMS transport adapter for sending SMS messages */ - smsAdapter: ISmsTransport - requestDetailService: NotificationRequestDetailService - /** Number of jobs to process in parallel (default: 2) */ - concurrency?: number - structuredLogger?: StructuredLoggerService +type SmsPayload = DeliveryJobPayload['payload'] + +interface SmsSendResult { + externalId: string + provider: string + providerResponse?: string + /** Per-recipient outcomes, from a transport that reports them. */ + results?: SmsRecipientResult[] +} + +function hasBody(payload: SmsPayload): boolean { + return !!payload.content?.body && typeof payload.content.body === 'string' } /** * SMS Delivery Worker * - * Processes SMS delivery jobs: - * 1. Receives SMS delivery jobs from the SMS_DELIVERY queue - * 2. Updates notification status to SENDING in database - * 3. Gets the appropriate adapter (future: GC Notify, Twilio, etc) - * 4. Sends SMS via adapter - * 5. Updates notification status to COMPLETED on success, FAILED on error - * 6. Implements retry logic with exponential backoff + * A plain send renders its body (a server template or inline content) and sends it to every + * recipient through the SMS adapter; a transport that reports per recipient has each outcome + * recorded on its own row. A merge batch sends each recipient their own rendering + * (MergeBatchRunner). A failure is settled by DeliveryFailureHandler and retried. * - * Idempotency: Job key is notifyId_sms, preventing duplicate delivery - * Tracing: All operations logged with notifyId for end-to-end visibility + * Idempotency: the job key is notifyId_sms, or the batchId for a merge batch. */ -export class SmsDeliveryWorker { - private readonly logger = new Logger(SmsDeliveryWorker.name) - private static readonly phoneNumberService = new PhoneNumberService() - - private static isPermanentValidationError(error: unknown): error is HttpException { - return error instanceof HttpException && error.getStatus() === 400 +@Injectable() +export class SmsDeliveryWorker extends DeliveryWorker { + readonly queueName = QueueName.SMS_DELIVERY + readonly concurrency = { configKey: 'smsDeliveryWorkerConcurrency', default: 2 } + protected readonly channel = SMS_CHANNEL + protected readonly logger = new Logger(SmsDeliveryWorker.name) + + constructor( + private readonly notificationService: NotificationService, + private readonly requestDetailService: NotificationRequestDetailService, + private readonly templatesRepository: TemplatesRepository, + private readonly templatesService: TemplatesService, + private readonly inlineRenderingService: InlineRenderingService, + @Inject(SMS_ADAPTER) private readonly smsAdapter: ISmsTransport, + private readonly phoneNumberService: PhoneNumberService, + mergeBatches: MergeBatchRunner, + failures: DeliveryFailureHandler, + @Optional() structuredLogger?: StructuredLoggerService, + ) { + super(mergeBatches, failures, structuredLogger) } - /** Initialize the SMS delivery worker on a queue */ - static initialize({ - smsQueue, - notificationService, - templatesRepository, - templatesService, - inlineRenderingService, - smsAdapter, - requestDetailService, - concurrency = 2, - structuredLogger, - }: SmsDeliveryWorkerDeps): void { - const logger = new Logger(SmsDeliveryWorker.name) - const workerContext = SmsDeliveryWorker.name - - logger.log(`Registering SMS delivery worker processor (concurrency=${concurrency})`) - - // Register the job processor with configurable concurrency - // Note: Don't await process() - it sets up listeners and never resolves - smsQueue.process(concurrency, async (job: Bull.Job) => { - const { notifyId, tenantId, payload, request } = job.data - const startedAt = Date.now() - - logger.debug(`[${notifyId}] Processing SMS delivery job for tenant=${tenantId}`) - - try { - // Validate DeliveryJobPayload structure - if (!notifyId || typeof notifyId !== 'string') { - throw new Error('Invalid delivery job: notifyId is missing or invalid') - } - if (!tenantId || typeof tenantId !== 'string') { - throw new Error('Invalid delivery job: tenantId is missing or invalid') - } - - // Emit a structured lifecycle "start" event. - structuredLogger?.logNotificationStart(notifyId, tenantId, 'sms', workerContext) - - // A merge batch carries its recipients in mailMergeData, not a single payload: each one - // gets its own rendered body, so they are sent individually below. - if (job.data.mailMerge && job.data.mailMergeData && job.data.batchId) { - return await SmsDeliveryWorker.processMergeBatch( - job.data.batchId, - job.data.mailMergeData, - notifyId, - tenantId, - logger, - templatesRepository, - templatesService, - inlineRenderingService, - smsAdapter, - requestDetailService, - notificationService, - batchProgressReporter(job), - ) - } - - // Validate job data - if (!payload || typeof payload !== 'object') { - throw new Error('Invalid delivery job: SMS payload is missing or invalid') - } - - if ( - !payload.recipients || - !payload.recipients.to || - !Array.isArray(payload.recipients.to) || - payload.recipients.to.length === 0 - ) { - throw new Error('Invalid SMS payload: recipient phone number is missing or invalid') - } - - const invalidRecipientIndex = payload.recipients.to.findIndex( - (recipient) => !SmsDeliveryWorker.phoneNumberService.isValid(recipient), + /** Render each merge recipient's body from their own params and send it. */ + protected mergeSender(job: Job): MergeSender { + const { notifyId } = job.data + return { + channel: SMS_CHANNEL, + hasInlineContent: (content) => !!content?.body, + send: async (recipient, params, { template, inline }) => { + const { body } = template + ? await this.templatesService.renderTemplateContent(template, params) + : await this.inlineRenderingService.renderSms(inline, params) + const result = await this.sendSms( + { recipients: { to: [recipient.address] }, content: { body } }, + notifyId, ) - if (invalidRecipientIndex !== -1) { - throw new BadRequestException( - `Invalid SMS payload: recipient phone number at index ${invalidRecipientIndex} is not valid E.164`, - ) - } - - // A redelivered job (a stalled worker, a rolling deploy) must not send again to anyone a - // previous attempt already reached. resetForRetry keeps their rows; this filters them out - // of the send list, which comes from the payload rather than from those rows. The lookup - // runs on every attempt because Bull re-runs a stalled job without incrementing - // attemptsMade. - if ((job.attemptsMade ?? 0) > 0) { - await requestDetailService.resetForRetry(notifyId) - } - const alreadySent = await requestDetailService.findSentAddresses(notifyId) - - let resolvedPayload = payload - const smsTemplateId = payload.content?.templateId - if ( - !smsTemplateId && - (!payload.content?.body || typeof payload.content.body !== 'string') - ) { - throw new Error('Invalid SMS payload: body is missing or invalid') - } - - if (smsTemplateId) { - logger.debug(`[${notifyId}] Resolving template: ${smsTemplateId}`) - try { - const template = await templatesRepository.findById(tenantId, smsTemplateId) - if (!template) { - throw new NotFoundException( - `Template '${smsTemplateId}' not found for tenant '${tenantId}'`, - ) - } - - // Merge template content into SMS payload if channel matches - if (template.channelCode === 'SMS') { - // Render the template with personalisation data from request.params. - // SMS always renders as plain text, so no bodyType override applies here. - const rendered = await templatesService.renderTemplateContent(template, { - ...request?.params, - ...payload.params, - }) + return result.externalId + }, + } + } - resolvedPayload = { - ...payload, - content: { - ...payload.content, - body: rendered.body, - bodyType: rendered.bodyType, - }, - } - logger.debug(`[${notifyId}] Template resolved and rendered into SMS payload`) - } - } catch (templateError) { - logger.error( - `[${notifyId}] Failed to resolve template: ${(templateError as Error).message}`, - ) - throw templateError - } - } else if (payload.content?.renderer) { - // Handle inline rendering if renderer is specified and no templateId - logger.debug( - `[${notifyId}] Rendering inline content with renderer: ${payload.content.renderer}`, - ) - try { - const rendered = await inlineRenderingService.renderSms(payload.content, { - ...request?.params, - ...payload.params, - }) + /** A plain send: one body to every recipient on the job. */ + protected async deliver(job: Job, startedAt: number): Promise { + const { notifyId, tenantId } = job.data + const payload = this.validatedPayload(job.data.payload) + + // A redelivered job (a stalled worker, a rolling deploy) must not send again to anyone a + // previous attempt already reached. resetForRetry keeps their rows; the send list, which comes + // from the payload rather than those rows, is filtered against them below. The lookup runs on + // every attempt because a stalled job is re-run without incrementing attemptsMade. + if ((job.attemptsMade ?? 0) > 0) { + await this.requestDetailService.resetForRetry(notifyId) + } + const alreadySent = await this.requestDetailService.findSentAddresses(notifyId) - resolvedPayload = { - ...payload, - content: { - ...payload.content, - body: rendered.body, - }, - } - logger.debug(`[${notifyId}] Inline content rendered successfully`) - } catch (renderError) { - logger.error( - `[${notifyId}] Failed to render inline content: ${(renderError as Error).message}`, - ) - throw renderError - } - } + let resolved = await this.render(job.data, payload) - if (!resolvedPayload.content?.body || typeof resolvedPayload.content.body !== 'string') { - throw new Error('Invalid SMS payload: body is missing or invalid') - } + await this.notificationService.update(notifyId, tenantId, { + status: NotificationStatus.SENDING, + updatedBy: 'system', + }) + await this.requestDetailService.updateStatus(notifyId, NotificationStatus.SENDING, { + preserveCompleted: true, + }) + this.logger.debug(`[${notifyId}] Updated notification status to SENDING`) - // Update status to SENDING - await notificationService.update(notifyId, tenantId, { - status: NotificationStatus.SENDING, + if (alreadySent.size > 0) { + const remaining = resolved.recipients.to.filter((recipient) => !alreadySent.has(recipient)) + if (remaining.length === 0) { + this.logger.log( + `[${notifyId}] Every recipient was delivered on an earlier attempt; nothing to re-send`, + ) + await this.notificationService.update(notifyId, tenantId, { + status: NotificationStatus.COMPLETED, updatedBy: 'system', }) - await requestDetailService.updateStatus(notifyId, NotificationStatus.SENDING, { - preserveCompleted: true, - }) - logger.debug(`[${notifyId}] Updated notification status to SENDING`) - - if (alreadySent.size > 0) { - const remaining = resolvedPayload.recipients.to.filter( - (recipient: string) => !alreadySent.has(recipient), - ) - - if (remaining.length === 0) { - logger.log( - `[${notifyId}] Every recipient was delivered on an earlier attempt; nothing to re-send`, - ) - await notificationService.update(notifyId, tenantId, { - status: NotificationStatus.COMPLETED, - updatedBy: 'system', - }) - return { success: true, notifyId } - } - - logger.log( - `[${notifyId}] Skipping ${alreadySent.size} recipient(s) delivered on an earlier attempt`, - ) - resolvedPayload = { - ...resolvedPayload, - recipients: { ...resolvedPayload.recipients, to: remaining }, - } - } + return { success: true, notifyId } + } - // Send SMS using the injected adapter - const result = await SmsDeliveryWorker.sendSmsViaAdapter( - resolvedPayload, - logger, - notifyId, - smsAdapter, - ) + this.logger.log( + `[${notifyId}] Skipping ${alreadySent.size} recipient(s) delivered on an earlier attempt`, + ) + resolved = { ...resolved, recipients: { ...resolved.recipients, to: remaining } } + } - logger.debug(`[${notifyId}] SMS sent: ${result.providerResponse ?? ''}`) + const result = await this.sendSms(resolved, notifyId) + this.logger.debug(`[${notifyId}] SMS sent: ${result.providerResponse ?? ''}`) - // A transport that reports per recipient gets each outcome recorded on its own row, so a - // partial failure is visible and - crucially - the successes are not repeated by a retry. - // A transport that cannot (or a send that wholly succeeded) is still all-or-nothing. - let finalStatus = NotificationStatus.COMPLETED - if (result.results && result.results.length > 0) { - for (const recipient of result.results) { - // The provider could not take it: still owed, not failed. Sent again below. - if (recipient.transient) continue - if (recipient.success) { - await requestDetailService.markRecipientSent( - notifyId, - null, - recipient.to, - recipient.messageId, - ) - } else { - await requestDetailService.markRecipientFailed( - notifyId, - null, - recipient.to, - recipient.error ?? 'Send failed', - ) - } - } + const finalStatus = await this.recordOutcome(notifyId, result) + await this.notificationService.update(notifyId, tenantId, { + status: finalStatus, + updatedBy: 'system', + }) + this.logger.log(`[${notifyId}] Notification marked as ${finalStatus.toUpperCase()}`) + this.structuredLogger?.logNotificationSuccess( + notifyId, + tenantId, + 'sms', + result.externalId, + Date.now() - startedAt, + SMS_CHANNEL.context, + ) - // Who was sent is recorded above, so the retry skips them; the rest are sent again once - // the provider is back. - const owed = result.results.filter((recipient) => recipient.transient) - if (owed.length > 0) { - throw new TransientDeliveryError( - `${owed.length} recipient(s) not sent, provider unavailable: ${owed[0].error ?? ''}`, - 502, - ) - } + return { success: true, externalId: result.externalId, provider: result.provider } + } - const failedCount = result.results.filter((recipient) => !recipient.success).length - if (failedCount > 0) { - finalStatus = NotificationStatus.PARTIALLY_COMPLETED - logger.warn( - `[${notifyId}] SMS partially delivered: ${failedCount} of ${result.results.length} recipient(s) failed`, - ) - } - } else { - // Request has made it to the sms gateway, update request detail records as sent - await requestDetailService.markSent(notifyId, result.externalId) - } + private validatedPayload(payload: SmsPayload): SmsPayload { + if (!payload || typeof payload !== 'object') { + throw new Error('Invalid delivery job: SMS payload is missing or invalid') + } + const to = payload.recipients?.to + if (!Array.isArray(to) || to.length === 0) { + throw new Error('Invalid SMS payload: recipient phone number is missing or invalid') + } + const invalidIndex = to.findIndex((recipient) => !this.phoneNumberService.isValid(recipient)) + if (invalidIndex !== -1) { + throw new BadRequestException( + `Invalid SMS payload: recipient phone number at index ${invalidIndex} is not valid E.164`, + ) + } + // A template supplies the body when rendered, so only inline content needs one now. + if (!payload.content?.templateId && !hasBody(payload)) { + throw new Error('Invalid SMS payload: body is missing or invalid') + } + return payload + } - await notificationService.update(notifyId, tenantId, { - status: finalStatus, - updatedBy: 'system', - }) - logger.log(`[${notifyId}] Notification marked as ${finalStatus.toUpperCase()}`) - structuredLogger?.logNotificationSuccess( - notifyId, - tenantId, - 'sms', - result.externalId, - Date.now() - startedAt, - workerContext, - ) + /** Render the body from a server template or inline content. SMS is always plain text. */ + private async render( + { notifyId, tenantId, request }: DeliveryJobPayload, + payload: SmsPayload, + ): Promise { + const templateId = payload.content?.templateId + const params = { ...request?.params, ...payload.params } + let rendered = payload - return { success: true, externalId: result.externalId, provider: result.provider } + if (templateId) { + rendered = await this.renderTemplate(notifyId, tenantId, templateId, payload, params) + } else if (payload.content?.renderer) { + this.logger.debug( + `[${notifyId}] Rendering inline content with renderer: ${payload.content.renderer}`, + ) + try { + const { body } = await this.inlineRenderingService.renderSms(payload.content, params) + rendered = { ...payload, content: { ...payload.content, body } } + this.logger.debug(`[${notifyId}] Inline content rendered successfully`) } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error) - const attempt = (job.attemptsMade || 0) + 1 - const finalAttempt = (job.attemptsMade || 0) >= (job.opts.attempts || 3) - 1 - logger.error( - `[${notifyId}] Failed to send SMS delivery job (attempt ${attempt}/3): ${errorMessage}`, - error instanceof Error ? error.stack : '', + this.logger.error( + `[${notifyId}] Failed to render inline content: ${(error as Error).message}`, ) - - // The provider could not take it right now. Nothing is marked failed, even on the last - // attempt: the recipients stay owed, and the delivery reconciler re-queues the job once - // Bull's retries are spent. - if (isTransientDeliveryError(error)) throw error - - // A merge batch settles only itself: its sent rows, and every other batch, are not its - // to fail. - const batchId = job.data.mailMerge ? job.data.batchId : undefined - if (batchId) { - if (finalAttempt || SmsDeliveryWorker.isPermanentValidationError(error)) { - await requestDetailService.markBatchUnsentFailed(notifyId, batchId, errorMessage) - await SmsDeliveryWorker.settleMergeRequest( - notifyId, - tenantId, - requestDetailService, - notificationService, - logger, - ) - job.discard() - logger.error(`[${notifyId}] Batch ${batchId} failed; its unsent recipients are FAILED`) - } - throw error - } - - if (SmsDeliveryWorker.isPermanentValidationError(error)) { - await notificationService.update(notifyId, tenantId, { - status: NotificationStatus.FAILED, - updatedBy: 'system', - errorReason: errorMessage, - }) - await requestDetailService.markFailed(notifyId, errorMessage) - job.discard() - logger.error( - `[${notifyId}] Notification marked as FAILED after permanent validation error. Error: ${errorMessage}`, - ) - throw error - } - - // Update status to FAILED only on final attempt (when no retries left) - if (finalAttempt) { - await notificationService.update(notifyId, tenantId, { - status: NotificationStatus.FAILED, - updatedBy: 'system', - errorReason: errorMessage, - }) - await requestDetailService.markFailed(notifyId, errorMessage) - logger.error( - `[${notifyId}] Notification marked as FAILED after 3 attempts. Error: ${errorMessage}`, - ) - structuredLogger?.logNotificationFailure( - notifyId, - tenantId, - 'sms', - error instanceof Error ? error : errorMessage, - Date.now() - startedAt, - workerContext, - ) - } - - // Re-throw to trigger BullMQ retry logic throw error } - }) - - // Event listeners for job lifecycle - smsQueue.on('completed', (job: Bull.Job) => { - const { notifyId } = job.data - logger.debug(`[${notifyId}] SMS delivery job completed`) - structuredLogger?.logQueueOperation('complete', smsQueue.name, job.id?.toString(), notifyId, { - channel: 'sms', - context: workerContext, - }) - }) - - smsQueue.on('failed', (job: Bull.Job, err: Error) => { - const { notifyId } = job.data - logger.error( - `[${notifyId}] SMS delivery job failed (attempt ${job.attemptsMade}/${job.opts.attempts}): error=${err.message}`, - ) - structuredLogger?.logQueueOperation('failed', smsQueue.name, job.id?.toString(), notifyId, { - channel: 'sms', - context: workerContext, - error: err.message, - attemptsMade: job.attemptsMade, - maxAttempts: job.opts.attempts, - }) - }) + } - logger.log('SMS delivery worker initialized') + if (!hasBody(rendered)) { + throw new Error('Invalid SMS payload: body is missing or invalid') + } + return rendered } - /** - * Send SMS via adapter - * @param payload SMS payload - * @param logger Logger instance - * @param notifyId Notification ID for tracing - * @param smsAdapter SMS transport adapter - * @returns Promise with send result - */ - /** - * Send one batch of a merge: render each recipient's body from their own params and send it. - * - * A failed recipient is recorded and skipped rather than throwing, so one bad row cannot strand - * the addresses already delivered in this batch; only systemic errors (a missing template) throw - * to trigger a retry of the whole batch. - */ - private static async processMergeBatch( - batchId: string, - mailMergeData: MailMergeJobData, + private async renderTemplate( notifyId: string, tenantId: string, - logger: Logger, - templatesRepository: TemplatesRepository, - templatesService: TemplatesService, - inlineRenderingService: InlineRenderingService, - smsAdapter: ISmsTransport, - requestDetailService: NotificationRequestDetailService, - notificationService: NotificationService, - reportProgress?: BatchProgressReporter, - ): Promise<{ success: boolean; batchId: string; sent: number; failed: number }> { - const { content, params } = mailMergeData - const { recipients, alreadySent } = await loadBatchRecipients( - requestDetailService, - notifyId, - batchId, - mailMergeData.recipients, - ) - const templateId = content?.templateId - const hasInlineContent = !!content?.body - - // A merge renders from exactly one source: a server template or inline content. - if (templateId && hasInlineContent) { - throw new Error('SMS merge requires either a templateId or inline content, not both') - } - if (!templateId && !hasInlineContent) { - throw new Error('SMS merge requires either a templateId or inline content') - } - - const template = templateId ? await templatesRepository.findById(tenantId, templateId) : null - if (templateId) { + templateId: string, + payload: SmsPayload, + params: Record, + ): Promise { + this.logger.debug(`[${notifyId}] Resolving template: ${templateId}`) + try { + const template = await this.templatesRepository.findById(tenantId, templateId) if (!template) { throw new NotFoundException(`Template '${templateId}' not found for tenant '${tenantId}'`) } - if (template.channelCode !== 'SMS') { - throw new Error(`Template '${templateId}' is not an SMS template`) - } + // A template of another channel leaves the payload's own content to be validated. + if (template.channelCode !== 'SMS') return payload + + const { body, bodyType } = await this.templatesService.renderTemplateContent(template, params) + this.logger.debug(`[${notifyId}] Template resolved and rendered into SMS payload`) + return { ...payload, content: { ...payload.content, body, bodyType } } + } catch (error) { + this.logger.error(`[${notifyId}] Failed to resolve template: ${(error as Error).message}`) + throw error } + } - // Inline content defaults to handlebars so per-recipient variables are substituted. - const inlineContent = content - ? { ...content, renderer: content.renderer ?? ('handlebars' as const) } - : null - - logger.debug( - `[${notifyId}] Processing SMS merge batch ${batchId}: ${recipients.length} recipient(s)`, - ) - - // The merge branch returns before the retry handling above, so a redelivered batch filters - // here instead. Addresses delivered by an earlier attempt keep their rows and are skipped. - const pending = alreadySent.size - ? recipients.filter((recipient) => !alreadySent.has(recipient.address)) - : recipients - - if (alreadySent.size > 0) { - logger.log( - `[${notifyId}] Batch ${batchId}: skipping ${alreadySent.size} recipient(s) delivered on an earlier attempt`, - ) + /** + * Record who the send reached, and return the request's status. A transport that reports per + * recipient has each outcome written to its own row, so a partial failure is visible and the + * successes are not repeated by a retry; one that cannot is all-or-nothing. + */ + private async recordOutcome( + notifyId: string, + result: SmsSendResult, + ): Promise { + if (!result.results?.length) { + await this.requestDetailService.markSent(notifyId, result.externalId) + return NotificationStatus.COMPLETED } - let sent = 0 - let failed = 0 - // Recipients delivered by an earlier attempt count as sent, so a retried batch resumes its bar. - const progress = () => ({ sent: alreadySent.size + sent, failed, total: recipients.length }) - reportProgress?.(progress()) - - for (const recipient of pending) { - try { - // Per-recipient params take precedence over the global ones. - const mergedParams = { ...(params || {}), ...recipient.params } - - const body = template - ? (await templatesService.renderTemplateContent(template, mergedParams)).body - : (await inlineRenderingService.renderSms(inlineContent!, mergedParams)).body - - const result = await SmsDeliveryWorker.sendSmsViaAdapter( - { recipients: { to: [recipient.address] }, content: { body } }, - logger, - notifyId, - smsAdapter, - ) - - await requestDetailService.markRecipientSent( - notifyId, - batchId, - recipient.address, - result.externalId, - ) - sent++ - } catch (recipientError) { - // The provider, not this recipient: stop the batch and leave the rest owed. The job is - // retried, skipping whoever was sent, and the circuit breaker holds it until it is back. - if (isTransientDeliveryError(recipientError)) { - logger.warn( - `[${notifyId}] Batch ${batchId} paused after ${sent} sent: ${recipientError.message}`, + // One row at a time: a plain send's recipient list is unbounded, and this keeps it to one + // pooled connection rather than one per recipient. + for (const recipient of result.results) { + // The provider could not take it: still owed, not failed. Sent again below. + if (recipient.transient) continue + await (recipient.success + ? this.requestDetailService.markRecipientSent( + notifyId, + null, + recipient.to, + recipient.messageId, ) - throw recipientError - } - const errorMessage = - recipientError instanceof Error ? recipientError.message : String(recipientError) - logger.error( - `[${notifyId}] SMS merge recipient send failed (batch=${batchId}): ${errorMessage}`, - ) - await requestDetailService.markRecipientFailed( - notifyId, - batchId, - recipient.address, - errorMessage, - ) - failed++ - } - reportProgress?.(progress()) + : this.requestDetailService.markRecipientFailed( + notifyId, + null, + recipient.to, + recipient.error ?? 'Send failed', + )) } - logger.log(`[${notifyId}] SMS merge batch ${batchId} complete: sent=${sent}, failed=${failed}`) - await SmsDeliveryWorker.settleMergeRequest( - notifyId, - tenantId, - requestDetailService, - notificationService, - logger, - ) - - return { success: failed === 0, batchId, sent, failed } - } - - /** Settle the parent request once no recipients remain pending across all batches. */ - private static async settleMergeRequest( - notifyId: string, - tenantId: string, - requestDetailService: NotificationRequestDetailService, - notificationService: NotificationService, - logger: Logger, - ): Promise { - const pendingRemaining = await requestDetailService.countByStatus(notifyId, 'pending') - if (pendingRemaining === 0) { - const failedRemaining = await requestDetailService.countByStatus(notifyId, 'failed') - const sentRemaining = await requestDetailService.countByStatus(notifyId, 'sent') - let finalStatus: NotificationStatus - if (failedRemaining === 0) { - finalStatus = NotificationStatus.COMPLETED - } else if (sentRemaining === 0) { - finalStatus = NotificationStatus.FAILED - } else { - finalStatus = NotificationStatus.PARTIALLY_COMPLETED - } - await notificationService.update(notifyId, tenantId, { - status: finalStatus, - updatedBy: 'sms-delivery-worker', - }) - logger.log( - `[${notifyId}] All SMS merge batches complete; parent marked ${finalStatus.toUpperCase()}`, + // Who was sent is recorded above, so the retry skips them; the rest are sent again once the + // provider is back. + const owed = result.results.filter((recipient) => recipient.transient) + if (owed.length > 0) { + throw new TransientDeliveryError( + `${owed.length} recipient(s) not sent, provider unavailable: ${owed[0].error ?? ''}`, + 502, ) } + + const failedCount = result.results.filter((recipient) => !recipient.success).length + if (failedCount === 0) return NotificationStatus.COMPLETED + this.logger.warn( + `[${notifyId}] SMS partially delivered: ${failedCount} of ${result.results.length} recipient(s) failed`, + ) + return NotificationStatus.PARTIALLY_COMPLETED } - private static async sendSmsViaAdapter( - payload: any, - logger: Logger, - notifyId: string, - smsAdapter: ISmsTransport, - ): Promise<{ - externalId: string - provider: string - providerResponse?: string - results?: SmsRecipientResult[] - }> { - logger.debug(`[${notifyId}] Sending SMS via ${smsAdapter.name} adapter`) + private async sendSms(payload: any, notifyId: string): Promise { + this.logger.debug(`[${notifyId}] Sending SMS via ${this.smsAdapter.name} adapter`) // Tagged with our id so a provider delivery report can be matched back to this notification. - const result = await smsAdapter.send({ ...(payload as object), tag: notifyId } as any) + const result = await this.smsAdapter.send({ ...(payload as object), tag: notifyId } as any) return { - externalId: result.messageId || result.providerResponse || `${smsAdapter.name}-${Date.now()}`, - provider: smsAdapter.name, + externalId: + result.messageId || result.providerResponse || `${this.smsAdapter.name}-${Date.now()}`, + provider: this.smsAdapter.name, providerResponse: result.providerResponse, - // Passed through so the caller can record who actually received the message. results: result.results, } } diff --git a/backend/src/queue/workers/webhook-delivery.worker.spec.ts b/backend/src/queue/workers/webhook-delivery.worker.spec.ts new file mode 100644 index 000000000..5b0784921 --- /dev/null +++ b/backend/src/queue/workers/webhook-delivery.worker.spec.ts @@ -0,0 +1,76 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { Logger } from '@nestjs/common' +import axios from 'axios' +import type { Job } from 'bullmq' +import { WebhookDeliveryWorker } from './webhook-delivery.worker' +import { WEBHOOK_JOB_OPTIONS } from '../job-options' +import type { WebhookJobPayload } from '../queue.types' +import type { WebhookService } from '../../api/webhook/webhook.service' +import type { WebhookDeliveryLogRepository } from '../../api/webhook/webhook-delivery-log.repository' + +vi.mock('axios', async (importOriginal) => { + const actual = await importOriginal() + return { default: { ...actual.default, post: vi.fn() } } +}) + +describe('WebhookDeliveryWorker', () => { + const webhookService = { + findById: vi.fn(), + } + const deliveryLog = { logAttempt: vi.fn().mockResolvedValue({}) } + const worker = new WebhookDeliveryWorker( + webhookService as unknown as WebhookService, + deliveryLog as unknown as WebhookDeliveryLogRepository, + ) + const job = (attemptsMade: number) => + ({ + id: 'webhook_w1_n1_completed', + data: { + notificationId: 'n1', + tenantId: 't1', + webhookId: 'w1', + event: 'notification.status.changed', + payload: { status: 'completed' }, + }, + opts: WEBHOOK_JOB_OPTIONS, + attemptsMade, + }) as unknown as Job + + beforeEach(() => { + vi.clearAllMocks() + vi.spyOn(Logger.prototype, 'log').mockImplementation(() => {}) + vi.spyOn(Logger.prototype, 'error').mockImplementation(() => {}) + webhookService.findById.mockResolvedValue({ + active: true, + url: 'https://tenant.example/hook', + webhookType: 'generic', + }) + vi.mocked(axios.post).mockRejectedValue(new Error('connect ECONNREFUSED')) + }) + + // The log row says when the next attempt is due; it must match when the queue retries. + it.each([ + [0, 5000], + [1, 15000], + ])( + 'records the next retry on the queue’s schedule after attempt %i fails', + async (made, delay) => { + const before = Date.now() + await expect(worker.process(job(made))).rejects.toThrow('ECONNREFUSED') + + const [attempt] = deliveryLog.logAttempt.mock.calls[0] + expect(attempt).toMatchObject({ status: 'RETRYING', attemptNumber: made + 1 }) + const dueIn = (attempt.nextRetryAt as Date).getTime() - before + expect(dueIn).toBeGreaterThanOrEqual(delay) + expect(dueIn).toBeLessThan(delay + 1000) + }, + ) + + it('records the last attempt as failed, with no retry due', async () => { + await expect(worker.process(job(2))).rejects.toThrow('ECONNREFUSED') + + expect(deliveryLog.logAttempt).toHaveBeenCalledWith( + expect.objectContaining({ status: 'FAILED', attemptNumber: 3, nextRetryAt: undefined }), + ) + }) +}) diff --git a/backend/src/queue/workers/webhook-delivery.worker.ts b/backend/src/queue/workers/webhook-delivery.worker.ts index 449f4117d..ef7cd0f59 100644 --- a/backend/src/queue/workers/webhook-delivery.worker.ts +++ b/backend/src/queue/workers/webhook-delivery.worker.ts @@ -1,11 +1,15 @@ -import { Logger } from '@nestjs/common' +import { Injectable, Logger } from '@nestjs/common' +import { describeError } from '../../common/utils/describe-error' import { createHmac } from 'crypto' -import Bull from 'bull' +import type { Job } from 'bullmq' import axios from 'axios' import { WebhookJobPayload } from '../queue.types' import { WebhookService } from '../../api/webhook/webhook.service' import { WebhookType } from '../../enum/webhook-type.enum' import { WebhookDeliveryLogRepository } from '../../api/webhook/webhook-delivery-log.repository' +import { QueueName } from '../../enum/queue-name.enum' +import { QueueProcessor } from './queue-processor' +import { maxAttempts, retryDelay } from '../job-options' /** * Webhook Delivery Worker @@ -19,172 +23,158 @@ import { WebhookDeliveryLogRepository } from '../../api/webhook/webhook-delivery * * Retry strategy: * - 4xx responses: treated as permanent failures (bad client config), no retry - * - 5xx responses / network errors: retried up to 3 attempts (~5s, 10s, 20s) + * - 5xx responses / network errors: 3 attempts, retried after 5s then 15s (WEBHOOK_JOB_OPTIONS) * * Idempotency: Job key is webhook_{webhookId}_{notificationId}_{status} */ -export class WebhookDeliveryWorker { - /** - * Initialize the webhook delivery worker on a queue - * @param webhookQueue The Bull queue instance for webhook delivery jobs - * @param webhookService Service for fetching webhook configs and delivery logs - * @param webhookDeliveryLogRepository Repository for logging delivery attempts - * @param concurrency Number of jobs to process in parallel (default: 2) - */ - static initialize( - webhookQueue: Bull.Queue, - webhookService: WebhookService, - webhookDeliveryLogRepository: WebhookDeliveryLogRepository, - concurrency: number = 2, - ): void { - const logger = new Logger(WebhookDeliveryWorker.name) - - logger.log(`Registering webhook delivery worker processor (concurrency=${concurrency})`) - - webhookQueue.process(concurrency, async (job: Bull.Job) => { - const { notificationId, tenantId, webhookId, event, payload } = job.data - const attemptNumber = (job.attemptsMade || 0) + 1 - - logger.log( - `[${notificationId}] Processing webhook delivery job for webhook=${webhookId} (attempt ${attemptNumber})`, +@Injectable() +export class WebhookDeliveryWorker implements QueueProcessor { + readonly queueName = QueueName.WEBHOOK_DELIVERY + readonly concurrency = { configKey: 'webhookDeliveryWorkerConcurrency', default: 2 } + private readonly logger = new Logger(WebhookDeliveryWorker.name) + + constructor( + private readonly webhookService: WebhookService, + private readonly webhookDeliveryLogRepository: WebhookDeliveryLogRepository, + ) {} + + async process(job: Job): Promise { + const { notificationId, tenantId, webhookId, payload } = job.data + const attemptNumber = (job.attemptsMade || 0) + 1 + + this.logger.log( + `[${notificationId}] Processing webhook delivery job for webhook=${webhookId} (attempt ${attemptNumber})`, + ) + + // It may have been deleted or deactivated since the job was queued. + const webhookConfig = await this.webhookService.findById(tenantId, webhookId) + if (!webhookConfig?.active) { + this.logger.warn( + `[${notificationId}] Webhook ${webhookId} not found or inactive — skipping delivery`, ) - - // Fetch the webhook config — it may have been deleted or deactivated since queuing - const webhookConfig = await webhookService.findById(tenantId, webhookId) - if (!webhookConfig || !webhookConfig.active) { - logger.warn( - `[${notificationId}] Webhook ${webhookId} not found or inactive — skipping delivery`, - ) - return { success: false, reason: 'webhook_inactive' } - } - - // Build the POST body - const genericPostBody = { - event, - notificationId, - tenantId, - data: payload, - deliveredAt: new Date().toISOString(), - } - - // Teams expects a payload formatted in a certain way - const postBody = - webhookConfig.webhookType === WebhookType.TEAMS - ? buildTeamsPayload(notificationId, payload) - : genericPostBody - const bodyString = JSON.stringify(postBody) - - // Build headers - const requestHeaders: Record = { - 'Content-Type': 'application/json', - ...(webhookConfig.headers ?? {}), - } - - // HMAC-SHA256 sign the raw JSON body if a secret is configured - if (webhookConfig.secret) { - const hmac = createHmac('sha256', webhookConfig.secret).update(bodyString).digest('hex') - requestHeaders['X-Webhook-Signature'] = `sha256=${hmac}` - } - - let httpStatusCode: number | undefined - let responseBody: string | undefined - - try { - const response = await axios.post(webhookConfig.url, postBody, { - headers: requestHeaders, - timeout: 30_000, - }) - - httpStatusCode = response.status - responseBody = - typeof response.data === 'string' ? response.data : JSON.stringify(response.data) - - await webhookDeliveryLogRepository.logAttempt({ - webhookConfigId: webhookId, - notificationId, - tenantId, - eventType: event, - httpStatusCode, - responseBody, - attemptNumber, - status: 'SENT', - }) - - logger.log( - `[${notificationId}] Webhook delivered to ${webhookConfig.url} (status=${httpStatusCode})`, - ) - return { success: true, statusCode: httpStatusCode } - } catch (error) { - const isLastAttempt = (job.attemptsMade || 0) >= (job.opts.attempts || 3) - 1 - - if (axios.isAxiosError(error) && error.response) { - httpStatusCode = error.response.status - responseBody = - typeof error.response.data === 'string' - ? error.response.data - : JSON.stringify(error.response.data) - - // 4xx = client error (bad URL, auth failure, etc.) — do not retry - if (httpStatusCode >= 400 && httpStatusCode < 500) { - await webhookDeliveryLogRepository.logAttempt({ - webhookConfigId: webhookId, - notificationId, - tenantId, - eventType: event, - httpStatusCode, - responseBody, - attemptNumber, - status: 'FAILED', - }) - logger.warn( - `[${notificationId}] Webhook ${webhookId} returned ${httpStatusCode} (4xx) — not retrying`, - ) - return { success: false, statusCode: httpStatusCode } + return { success: false, reason: 'webhook_inactive' } + } + + // Teams expects a payload formatted in a certain way + const postBody = + webhookConfig.webhookType === WebhookType.TEAMS + ? buildTeamsPayload(notificationId, payload) + : { + event: job.data.event, + notificationId, + tenantId, + data: payload, + deliveredAt: new Date().toISOString(), } - } - - const errorMessage = error instanceof Error ? error.message : String(error) - const deliveryStatus = isLastAttempt ? 'FAILED' : 'RETRYING' - - await webhookDeliveryLogRepository.logAttempt({ - webhookConfigId: webhookId, - notificationId, - tenantId, - eventType: event, - httpStatusCode, - responseBody: responseBody ?? errorMessage, - attemptNumber, - status: deliveryStatus, - nextRetryAt: isLastAttempt - ? undefined - : new Date(Date.now() + 5000 * Math.pow(2, attemptNumber)), - }) - - logger.error( - `[${notificationId}] Webhook delivery to ${webhookConfig.url} failed (attempt ${attemptNumber}): ${errorMessage}`, - ) - - // Re-throw to trigger Bull retry for 5xx / network errors - throw error - } - }) + const bodyString = JSON.stringify(postBody) + + const requestHeaders: Record = { + 'Content-Type': 'application/json', + ...(webhookConfig.headers ?? {}), + } + // HMAC-SHA256 sign the raw JSON body if a secret is configured + if (webhookConfig.secret) { + const hmac = createHmac('sha256', webhookConfig.secret).update(bodyString).digest('hex') + requestHeaders['X-Webhook-Signature'] = `sha256=${hmac}` + } + + try { + const response = await axios.post(webhookConfig.url, postBody, { + headers: requestHeaders, + timeout: 30_000, + }) + const httpStatusCode = response.status + await this.logAttempt(job, { + httpStatusCode, + responseBody: responseText(response.data), + status: 'SENT', + }) + this.logger.log( + `[${notificationId}] Webhook delivered to ${webhookConfig.url} (status=${httpStatusCode})`, + ) + return { success: true, statusCode: httpStatusCode } + } catch (error) { + return this.handleDeliveryError(job, webhookConfig.url, error) + } + } - webhookQueue.on('completed', (job: Bull.Job) => { - const { notificationId, webhookId } = job.data - logger.debug(`[${notificationId}] Webhook delivery job completed for webhook=${webhookId}`) - }) + onCompleted(job: Job): void { + const { notificationId, webhookId } = job.data + this.logger.debug(`[${notificationId}] Webhook delivery job completed for webhook=${webhookId}`) + } - webhookQueue.on('failed', (job: Bull.Job, err: Error) => { - const { notificationId, webhookId } = job.data - logger.error( - `[${notificationId}] Webhook delivery job failed for webhook=${webhookId} (attempt ${job.attemptsMade}/${job.opts.attempts}): ${err.message}`, + onFailed(job: Job, error: Error): void { + const { notificationId, webhookId } = job.data + this.logger.error( + `[${notificationId}] Webhook delivery job failed for webhook=${webhookId} (attempt ${job.attemptsMade}/${job.opts.attempts}): ${error.message}`, + ) + } + + /** + * A 4xx is the tenant's configuration (a bad URL, a rejected credential): logged and not + * retried. Anything else - a 5xx, a network error - is rethrown for the queue to retry. + */ + private async handleDeliveryError( + job: Job, + url: string, + error: unknown, + ): Promise<{ success: false; statusCode: number }> { + const { notificationId, webhookId } = job.data + const attemptNumber = (job.attemptsMade ?? 0) + 1 + const isLastAttempt = attemptNumber >= maxAttempts(job) + const response = axios.isAxiosError(error) ? error.response : undefined + const httpStatusCode = response?.status + const responseBody = response ? responseText(response.data) : undefined + + if (httpStatusCode && httpStatusCode >= 400 && httpStatusCode < 500) { + await this.logAttempt(job, { httpStatusCode, responseBody, status: 'FAILED' }) + this.logger.warn( + `[${notificationId}] Webhook ${webhookId} returned ${httpStatusCode} (4xx) — not retrying`, ) + return { success: false, statusCode: httpStatusCode } + } + + const errorMessage = describeError(error) + await this.logAttempt(job, { + httpStatusCode, + responseBody: responseBody ?? errorMessage, + status: isLastAttempt ? 'FAILED' : 'RETRYING', + // The same schedule the queue retries on. + nextRetryAt: isLastAttempt + ? undefined + : new Date(Date.now() + retryDelay(job, attemptNumber)), }) + this.logger.error( + `[${notificationId}] Webhook delivery to ${url} failed (attempt ${attemptNumber}): ${errorMessage}`, + ) + throw error + } - logger.log('Webhook delivery worker initialized') + private async logAttempt( + job: Job, + attempt: { + httpStatusCode?: number + responseBody?: string + status: 'SENT' | 'FAILED' | 'RETRYING' + nextRetryAt?: Date + }, + ): Promise { + const { notificationId, tenantId, webhookId, event } = job.data + await this.webhookDeliveryLogRepository.logAttempt({ + webhookConfigId: webhookId, + notificationId, + tenantId, + eventType: event, + attemptNumber: (job.attemptsMade || 0) + 1, + ...attempt, + }) } } +function responseText(data: unknown): string { + return typeof data === 'string' ? data : JSON.stringify(data) +} + function buildTeamsPayload(notificationId: string, payload: any): any { const status = String(payload.status) const channel = String(payload.channel) diff --git a/backend/test/app.e2e-spec.ts b/backend/test/app.e2e-spec.ts index a6572061b..cad5f37af 100644 --- a/backend/test/app.e2e-spec.ts +++ b/backend/test/app.e2e-spec.ts @@ -68,7 +68,7 @@ describe('AppController (e2e)', () => { it('creates each module once', () => { // A static module registered twice runs its lifecycle hooks twice. QueueModule did: every - // pod started two sets of Bull workers and two heartbeats under one pod name, which made + // pod started two sets of queue workers and two heartbeats under one pod name, which made // the monitoring page report running jobs as stalled. forwardRef() inside a global dynamic // module (GcNotifyModule.forRoot) produced the orphaned copies. // TypeOrmModule repeats legitimately: each forFeature() call is its own dynamic module. diff --git a/tests/load/drain-test.js b/tests/load/drain-test.js index 00dc3f96b..00fff5806 100644 --- a/tests/load/drain-test.js +++ b/tests/load/drain-test.js @@ -8,7 +8,7 @@ import { Rate } from 'k6/metrics' * Fires a fixed BURST of notifysimple requests as fast as possible to pile jobs * onto the ingestion queue, then exits. This script only PRODUCES the burst and * confirms it was accepted (202); the actual DRAIN RATE and peak queue depth are - * measured by the workflow's monitor step (oc + redis-cli against the Bull queues), + * measured by the workflow's monitor step (oc + redis-cli against the BullMQ queues), * because the app exposes no queue-depth metric or API-key-readable status. * * This is the test that exposes INGESTION_WORKER_CONCURRENCY=1 as the drain