From 9eeb513401e8cf0481f308c85012641ae691bdfe Mon Sep 17 00:00:00 2001 From: Nikolay Karadzhov Date: Thu, 14 Aug 2025 14:11:15 +0300 Subject: [PATCH 01/14] feat(errors): Add specialized timeout error types for maintenance scenarios - Added `SocketTimeoutDuringMaintananceError`, a subclass of `TimeoutError`, to handle socket timeouts during maintenance. - Added `CommandTimeoutDuringMaintananceError`, another subclass of `TimeoutError`, to address command write timeouts during maintenance. --- packages/client/lib/errors.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/client/lib/errors.ts b/packages/client/lib/errors.ts index 5cb9166df0..ae4d598abd 100644 --- a/packages/client/lib/errors.ts +++ b/packages/client/lib/errors.ts @@ -71,6 +71,18 @@ export class BlobError extends ErrorReply {} export class TimeoutError extends Error {} +export class SocketTimeoutDuringMaintananceError extends TimeoutError { + constructor(timeout: number) { + super(`Socket timeout during maintenance. Expecting data, but didn't receive any in ${timeout}ms.`); + } +} + +export class CommandTimeoutDuringMaintananceError extends TimeoutError { + constructor(timeout: number) { + super(`Command timeout during maintenance. Waited to write command for more than ${timeout}ms.`); + } +} + export class MultiErrorReply extends ErrorReply { replies: Array; errorIndexes: Array; From 99f24ad2fd50ecbe418201e72f0e92eefabdec55 Mon Sep 17 00:00:00 2001 From: Nikolay Karadzhov Date: Thu, 14 Aug 2025 14:12:35 +0300 Subject: [PATCH 02/14] feat(linked-list): Add EmptyAwareSinglyLinkedList and enhance DoublyLinkedList functionality - Introduced `EmptyAwareSinglyLinkedList`, a subclass of `SinglyLinkedList` that emits an `empty` event when the list becomes empty due to `reset`, `shift`, or `remove` operations. - Added `nodes()` iterator method to `DoublyLinkedList` for iterating over nodes directly. - Enhanced unit tests for `DoublyLinkedList` and `SinglyLinkedList` to cover edge cases and new functionality. - Added comprehensive tests for `EmptyAwareSinglyLinkedList` to validate `empty` event emission under various scenarios. - Improved code formatting and consistency. --- .../client/lib/client/linked-list.spec.ts | 111 ++++++++++++++---- packages/client/lib/client/linked-list.ts | 41 ++++++- 2 files changed, 124 insertions(+), 28 deletions(-) diff --git a/packages/client/lib/client/linked-list.spec.ts b/packages/client/lib/client/linked-list.spec.ts index 9547fb81c7..c791d21900 100644 --- a/packages/client/lib/client/linked-list.spec.ts +++ b/packages/client/lib/client/linked-list.spec.ts @@ -1,138 +1,197 @@ -import { SinglyLinkedList, DoublyLinkedList } from './linked-list'; -import { equal, deepEqual } from 'assert/strict'; - -describe('DoublyLinkedList', () => { +import { + SinglyLinkedList, + DoublyLinkedList, + EmptyAwareSinglyLinkedList, +} from "./linked-list"; +import { equal, deepEqual } from "assert/strict"; + +describe("DoublyLinkedList", () => { const list = new DoublyLinkedList(); - it('should start empty', () => { + it("should start empty", () => { equal(list.length, 0); equal(list.head, undefined); equal(list.tail, undefined); deepEqual(Array.from(list), []); }); - it('shift empty', () => { + it("shift empty", () => { equal(list.shift(), undefined); equal(list.length, 0); deepEqual(Array.from(list), []); }); - it('push 1', () => { + it("push 1", () => { list.push(1); equal(list.length, 1); deepEqual(Array.from(list), [1]); }); - it('push 2', () => { + it("push 2", () => { list.push(2); equal(list.length, 2); deepEqual(Array.from(list), [1, 2]); }); - it('unshift 0', () => { + it("unshift 0", () => { list.unshift(0); equal(list.length, 3); deepEqual(Array.from(list), [0, 1, 2]); }); - it('remove middle node', () => { + it("remove middle node", () => { list.remove(list.head!.next!); equal(list.length, 2); deepEqual(Array.from(list), [0, 2]); }); - it('remove head', () => { + it("remove head", () => { list.remove(list.head!); equal(list.length, 1); deepEqual(Array.from(list), [2]); }); - it('remove tail', () => { + it("remove tail", () => { list.remove(list.tail!); equal(list.length, 0); deepEqual(Array.from(list), []); }); - it('unshift empty queue', () => { + it("unshift empty queue", () => { list.unshift(0); equal(list.length, 1); deepEqual(Array.from(list), [0]); }); - it('push 1', () => { + it("push 1", () => { list.push(1); equal(list.length, 2); deepEqual(Array.from(list), [0, 1]); }); - it('shift', () => { + it("shift", () => { equal(list.shift(), 0); equal(list.length, 1); deepEqual(Array.from(list), [1]); }); - it('shift last element', () => { + it("shift last element", () => { equal(list.shift(), 1); equal(list.length, 0); deepEqual(Array.from(list), []); }); + + it("provide forEach for nodes", () => { + list.reset(); + list.push(1); + list.push(2); + list.push(3); + let count = 0; + for(const _ of list.nodes()) { + count++; + } + equal(count, 3); + for(const _ of list.nodes()) { + count++; + } + equal(count, 6); + }); }); -describe('SinglyLinkedList', () => { +describe("SinglyLinkedList", () => { const list = new SinglyLinkedList(); - it('should start empty', () => { + it("should start empty", () => { equal(list.length, 0); equal(list.head, undefined); equal(list.tail, undefined); deepEqual(Array.from(list), []); }); - it('shift empty', () => { + it("shift empty", () => { equal(list.shift(), undefined); equal(list.length, 0); deepEqual(Array.from(list), []); }); - it('push 1', () => { + it("push 1", () => { list.push(1); equal(list.length, 1); deepEqual(Array.from(list), [1]); }); - it('push 2', () => { + it("push 2", () => { list.push(2); equal(list.length, 2); deepEqual(Array.from(list), [1, 2]); }); - it('push 3', () => { + it("push 3", () => { list.push(3); equal(list.length, 3); deepEqual(Array.from(list), [1, 2, 3]); }); - it('shift 1', () => { + it("shift 1", () => { equal(list.shift(), 1); equal(list.length, 2); deepEqual(Array.from(list), [2, 3]); }); - it('shift 2', () => { + it("shift 2", () => { equal(list.shift(), 2); equal(list.length, 1); deepEqual(Array.from(list), [3]); }); - it('shift 3', () => { + it("shift 3", () => { equal(list.shift(), 3); equal(list.length, 0); deepEqual(Array.from(list), []); }); - it('should be empty', () => { + it("should be empty", () => { equal(list.length, 0); equal(list.head, undefined); equal(list.tail, undefined); }); }); + +describe("EmptyAwareSinglyLinkedList", () => { + it("should emit 'empty' event when reset", () => { + const list = new EmptyAwareSinglyLinkedList(); + let count = 0; + list.events.on("empty", () => count++); + list.push(1); + list.reset(); + equal(count, 1); + list.reset(); + equal(count, 1); + }); + + it("should emit 'empty' event when shift makes the list empty", () => { + const list = new EmptyAwareSinglyLinkedList(); + let count = 0; + list.events.on("empty", () => count++); + list.push(1); + list.push(2); + list.shift(); + equal(count, 0); + list.shift(); + equal(count, 1); + list.shift(); + equal(count, 1); + }); + + it("should emit 'empty' event when remove makes the list empty", () => { + const list = new EmptyAwareSinglyLinkedList(); + let count = 0; + list.events.on("empty", () => count++); + const node1 = list.push(1); + const node2 = list.push(2); + list.remove(node1, undefined); + equal(count, 0); + list.remove(node2, undefined); + equal(count, 1); + }); +}); diff --git a/packages/client/lib/client/linked-list.ts b/packages/client/lib/client/linked-list.ts index 29678f027b..461f1d4082 100644 --- a/packages/client/lib/client/linked-list.ts +++ b/packages/client/lib/client/linked-list.ts @@ -1,3 +1,5 @@ +import EventEmitter from "events"; + export interface DoublyLinkedNode { value: T; previous: DoublyLinkedNode | undefined; @@ -32,7 +34,7 @@ export class DoublyLinkedList { next: undefined, value }; - } + } return this.#tail = this.#tail.next = { previous: this.#tail, @@ -93,7 +95,7 @@ export class DoublyLinkedList { node.previous!.next = node.next; node.previous = undefined; } - + node.next = undefined; } @@ -109,6 +111,14 @@ export class DoublyLinkedList { node = node.next; } } + + *nodes() { + let node = this.#head; + while(node) { + yield node; + node = node.next; + } + } } export interface SinglyLinkedNode { @@ -201,3 +211,30 @@ export class SinglyLinkedList { } } } + +export class EmptyAwareSinglyLinkedList extends SinglyLinkedList { + readonly events = new EventEmitter(); + reset() { + const old = this.length; + super.reset(); + if(old !== this.length && this.length === 0) { + this.events.emit('empty'); + } + } + shift(): T | undefined { + const old = this.length; + const ret = super.shift(); + if(old !== this.length && this.length === 0) { + this.events.emit('empty'); + } + return ret; + } + remove(node: SinglyLinkedNode, parent: SinglyLinkedNode | undefined) { + const old = this.length; + super.remove(node, parent); + if(old !== this.length && this.length === 0) { + this.events.emit('empty'); + } + } + +} From 2f09cc1ddbe51e3f74e432f772fe01045ff28829 Mon Sep 17 00:00:00 2001 From: Nikolay Karadzhov Date: Thu, 14 Aug 2025 14:21:52 +0300 Subject: [PATCH 03/14] refactor(commands-queue): Improve push notification handling - Replaced `setInvalidateCallback` with a more flexible `addPushHandler` method, allowing multiple handlers for push notifications. - Introduced the `PushHandler` type to standardize push notification processing. - Refactored `RedisCommandsQueue` to use a `#pushHandlers` array, enabling dynamic and modular handling of push notifications. - Updated `RedisClient` to leverage the new handler mechanism for `invalidate` push notifications, simplifying and decoupling logic. --- packages/client/lib/client/commands-queue.ts | 35 ++++++++------------ packages/client/lib/client/index.ts | 14 +++++++- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/packages/client/lib/client/commands-queue.ts b/packages/client/lib/client/commands-queue.ts index 52a07a7e3b..dd8f9ebe3a 100644 --- a/packages/client/lib/client/commands-queue.ts +++ b/packages/client/lib/client/commands-queue.ts @@ -50,6 +50,13 @@ const RESP2_PUSH_TYPE_MAPPING = { [RESP_TYPES.SIMPLE_STRING]: Buffer }; +// Try to handle a push notification. Return whether you +// successfully consumed the notification or not. This is +// important in order for the queue to be able to pass the +// notification to another handler if the current one did not +// succeed. +type PushHandler = (pushItems: Array) => boolean; + export default class RedisCommandsQueue { readonly #respVersion; readonly #maxLength; @@ -60,12 +67,11 @@ export default class RedisCommandsQueue { readonly decoder; readonly #pubSub = new PubSub(); + #pushHandlers: PushHandler[] = [this.#onPush.bind(this)]; get isPubSubActive() { return this.#pubSub.isActive; } - #invalidateCallback?: (key: RedisArgument | null) => unknown; - constructor( respVersion: RespVersions, maxLength: number | null | undefined, @@ -107,6 +113,7 @@ export default class RedisCommandsQueue { } return true; } + return false } #getTypeMapping() { @@ -119,30 +126,16 @@ export default class RedisCommandsQueue { onErrorReply: err => this.#onErrorReply(err), //TODO: we can shave off a few cycles by not adding onPush handler at all if CSC is not used onPush: push => { - if (!this.#onPush(push)) { - // currently only supporting "invalidate" over RESP3 push messages - switch (push[0].toString()) { - case "invalidate": { - if (this.#invalidateCallback) { - if (push[1] !== null) { - for (const key of push[1]) { - this.#invalidateCallback(key); - } - } else { - this.#invalidateCallback(null); - } - } - break; - } - } + for(const pushHandler of this.#pushHandlers) { + if(pushHandler(push)) return } }, getTypeMapping: () => this.#getTypeMapping() }); } - setInvalidateCallback(callback?: (key: RedisArgument | null) => unknown) { - this.#invalidateCallback = callback; + addPushHandler(handler: PushHandler): void { + this.#pushHandlers.push(handler); } addCommand( @@ -432,7 +425,7 @@ export default class RedisCommandsQueue { } static #removeTimeoutListener(command: CommandToWrite) { - command.timeout!.signal.removeEventListener('abort', command.timeout!.listener); + command.timeout?.signal.removeEventListener('abort', command.timeout!.listener); } static #flushToWrite(toBeSent: CommandToWrite, err: Error) { diff --git a/packages/client/lib/client/index.ts b/packages/client/lib/client/index.ts index 1a27ea8898..56757bcc00 100644 --- a/packages/client/lib/client/index.ts +++ b/packages/client/lib/client/index.ts @@ -464,7 +464,19 @@ export default class RedisClient< const cscConfig = options.clientSideCache; this.#clientSideCache = new BasicClientSideCache(cscConfig); } - this.#queue.setInvalidateCallback(this.#clientSideCache.invalidate.bind(this.#clientSideCache)); + this.#queue.addPushHandler((push: Array): boolean => { + if (push[0].toString() !== 'invalidate') return false; + + if (push[1] !== null) { + for (const key of push[1]) { + this.#clientSideCache?.invalidate(key) + } + } else { + this.#clientSideCache?.invalidate(null) + } + + return true + }); } } From c6c7b9d65eb6a022cf90f74c0f35f09280cbebc8 Mon Sep 17 00:00:00 2001 From: Nikolay Karadzhov Date: Thu, 14 Aug 2025 14:25:52 +0300 Subject: [PATCH 04/14] feat(commands-queue): Add method to wait for in-flight commands to complete - Introduced `waitForInflightCommandsToComplete` method to asynchronously wait for all in-flight commands to finish processing. - Utilized the `empty` event from `#waitingForReply` to signal when all commands have been completed. --- packages/client/lib/client/commands-queue.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/client/lib/client/commands-queue.ts b/packages/client/lib/client/commands-queue.ts index dd8f9ebe3a..a2016623e6 100644 --- a/packages/client/lib/client/commands-queue.ts +++ b/packages/client/lib/client/commands-queue.ts @@ -138,6 +138,17 @@ export default class RedisCommandsQueue { this.#pushHandlers.push(handler); } + async waitForInflightCommandsToComplete(): Promise { + // In-flight commands already completed + if(this.#waitingForReply.length === 0) { + return + }; + // Otherwise wait for in-flight commands to fire `empty` event + return new Promise(resolve => { + this.#waitingForReply.events.on('empty', resolve) + }); + } + addCommand( args: ReadonlyArray, options?: CommandOptions From 0adb776a875d728cd51cc2d4cbe07e5ddc2fb5e0 Mon Sep 17 00:00:00 2001 From: Nikolay Karadzhov Date: Thu, 14 Aug 2025 14:28:55 +0300 Subject: [PATCH 05/14] feat(commands-queue): Introduce maintenance mode support for commands-queue - Added `#inMaintenance` property and `set inMaintenance` setter to track maintenance mode state.d `#maintenanceCommandTimeout` and `setMaintenanceCommandTimeout` method to dynamically adjust command timeouts during maintenance.mmandTimeout` over individual command timeouts.DuringMaintananceError` is used when in maintenance mode. --- packages/client/lib/client/commands-queue.ts | 62 ++++++++++++++++++-- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/packages/client/lib/client/commands-queue.ts b/packages/client/lib/client/commands-queue.ts index a2016623e6..0f9341b484 100644 --- a/packages/client/lib/client/commands-queue.ts +++ b/packages/client/lib/client/commands-queue.ts @@ -1,9 +1,9 @@ -import { SinglyLinkedList, DoublyLinkedNode, DoublyLinkedList } from './linked-list'; +import { DoublyLinkedNode, DoublyLinkedList, EmptyAwareSinglyLinkedList } from './linked-list'; import encodeCommand from '../RESP/encoder'; import { Decoder, PUSH_TYPE_MAPPING, RESP_TYPES } from '../RESP/decoder'; import { TypeMapping, ReplyUnion, RespVersions, RedisArgument } from '../RESP/types'; import { ChannelListeners, PubSub, PubSubCommand, PubSubListener, PubSubType, PubSubTypeListeners } from './pub-sub'; -import { AbortError, ErrorReply, TimeoutError } from '../errors'; +import { AbortError, ErrorReply, CommandTimeoutDuringMaintananceError, TimeoutError } from '../errors'; import { MonitorCallback } from '.'; export interface CommandOptions { @@ -30,6 +30,7 @@ export interface CommandToWrite extends CommandWaitingForReply { timeout: { signal: AbortSignal; listener: () => unknown; + originalTimeout: number | undefined; } | undefined; } @@ -61,13 +62,58 @@ export default class RedisCommandsQueue { readonly #respVersion; readonly #maxLength; readonly #toWrite = new DoublyLinkedList(); - readonly #waitingForReply = new SinglyLinkedList(); + readonly #waitingForReply = new EmptyAwareSinglyLinkedList(); readonly #onShardedChannelMoved; #chainInExecution: symbol | undefined; readonly decoder; readonly #pubSub = new PubSub(); #pushHandlers: PushHandler[] = [this.#onPush.bind(this)]; + + #inMaintenance = false; + + set inMaintenance(value: boolean) { + this.#inMaintenance = value; + } + + #maintenanceCommandTimeout: number | undefined + + setMaintenanceCommandTimeout(ms: number | undefined) { + // Prevent possible api misuse + if (this.#maintenanceCommandTimeout === ms) return; + + this.#maintenanceCommandTimeout = ms; + + let counter = 0; + + // Overwrite timeouts of all eligible toWrite commands + for(const node of this.#toWrite.nodes()) { + const command = node.value; + + // Remove timeout listener if it exists + RedisCommandsQueue.#removeTimeoutListener(command) + + // Determine newTimeout + const newTimeout = this.#maintenanceCommandTimeout ?? command.timeout?.originalTimeout; + // if no timeout is given and the command didnt have any timeout before, skip + if (!newTimeout) return; + + counter++; + + // Overwrite the command's timeout + const signal = AbortSignal.timeout(newTimeout); + command.timeout = { + signal, + listener: () => { + this.#toWrite.remove(node); + command.reject(this.#inMaintenance ? new CommandTimeoutDuringMaintananceError(newTimeout) : new TimeoutError()); + }, + originalTimeout: command.timeout?.originalTimeout + }; + signal.addEventListener('abort', command.timeout.listener, { once: true }); + }; + } + get isPubSubActive() { return this.#pubSub.isActive; } @@ -172,15 +218,19 @@ export default class RedisCommandsQueue { typeMapping: options?.typeMapping }; - const timeout = options?.timeout; + // If #maintenanceCommandTimeout was explicitly set, we should + // use it instead of the timeout provided by the command + const timeout = this.#maintenanceCommandTimeout || options?.timeout if (timeout) { + const signal = AbortSignal.timeout(timeout); value.timeout = { signal, listener: () => { this.#toWrite.remove(node); - value.reject(new TimeoutError()); - } + value.reject(this.#inMaintenance ? new CommandTimeoutDuringMaintananceError(timeout) : new TimeoutError()); + }, + originalTimeout: options?.timeout }; signal.addEventListener('abort', value.timeout.listener, { once: true }); } From dcb76d1ba298f73bc11beb2405b21b5d4164d7e3 Mon Sep 17 00:00:00 2001 From: Nikolay Karadzhov Date: Thu, 14 Aug 2025 14:33:26 +0300 Subject: [PATCH 06/14] refator(client): Extract socket event listener setup into helper method --- packages/client/lib/client/index.ts | 59 ++++++++++++++++------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/packages/client/lib/client/index.ts b/packages/client/lib/client/index.ts index 56757bcc00..e5cfc50a8c 100644 --- a/packages/client/lib/client/index.ts +++ b/packages/client/lib/client/index.ts @@ -694,6 +694,35 @@ export default class RedisClient< return commands; } + #attachListeners(socket: RedisSocket) { + socket.on('data', chunk => { + try { + this.#queue.decoder.write(chunk); + } catch (err) { + this.#queue.resetDecoder(); + this.emit('error', err); + } + }) + .on('error', err => { + this.emit('error', err); + this.#clientSideCache?.onError(); + if (this.#socket.isOpen && !this.#options?.disableOfflineQueue) { + this.#queue.flushWaitingForReply(err); + } else { + this.#queue.flushAll(err); + } + }) + .on('connect', () => this.emit('connect')) + .on('ready', () => { + this.emit('ready'); + this.#setPingTimer(); + this.#maybeScheduleWrite(); + }) + .on('reconnecting', () => this.emit('reconnecting')) + .on('drain', () => this.#maybeScheduleWrite()) + .on('end', () => this.emit('end')); + } + #initiateSocket(): RedisSocket { const socketInitiator = async () => { const promises = [], @@ -725,33 +754,9 @@ export default class RedisClient< } }; - return new RedisSocket(socketInitiator, this.#options?.socket) - .on('data', chunk => { - try { - this.#queue.decoder.write(chunk); - } catch (err) { - this.#queue.resetDecoder(); - this.emit('error', err); - } - }) - .on('error', err => { - this.emit('error', err); - this.#clientSideCache?.onError(); - if (this.#socket.isOpen && !this.#options?.disableOfflineQueue) { - this.#queue.flushWaitingForReply(err); - } else { - this.#queue.flushAll(err); - } - }) - .on('connect', () => this.emit('connect')) - .on('ready', () => { - this.emit('ready'); - this.#setPingTimer(); - this.#maybeScheduleWrite(); - }) - .on('reconnecting', () => this.emit('reconnecting')) - .on('drain', () => this.#maybeScheduleWrite()) - .on('end', () => this.emit('end')); + const socket = new RedisSocket(socketInitiator, this.#options?.socket); + this.#attachListeners(socket); + return socket; } #pingTimer?: NodeJS.Timeout; From 85ef46159e48d265c07f5fadb5cbb919dd69114d Mon Sep 17 00:00:00 2001 From: Nikolay Karadzhov Date: Thu, 14 Aug 2025 14:55:12 +0300 Subject: [PATCH 07/14] refactor(socket): Add maintenance mode support and dynamic timeout handling - Introduced `#inMaintenance` property and setter to track maintenance mode state in `RedisSocket`. - Added `#maintenanceTimeout` and `setMaintenanceTimeout` method to dynamically adjust socket timeouts during maintenance. - Enhanced timeout error handling to differentiate between regular timeouts (`SocketTimeoutError`) and maintenance-specific timeouts (`SocketTimeoutDuringMaintananceError`). --- packages/client/lib/client/socket.ts | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/packages/client/lib/client/socket.ts b/packages/client/lib/client/socket.ts index 5f0bcc4492..5193e9d8a3 100644 --- a/packages/client/lib/client/socket.ts +++ b/packages/client/lib/client/socket.ts @@ -1,7 +1,7 @@ import { EventEmitter, once } from 'node:events'; import net from 'node:net'; import tls from 'node:tls'; -import { ConnectionTimeoutError, ClientClosedError, SocketClosedUnexpectedlyError, ReconnectStrategyError, SocketTimeoutError } from '../errors'; +import { ConnectionTimeoutError, ClientClosedError, SocketClosedUnexpectedlyError, ReconnectStrategyError, SocketTimeoutError, SocketTimeoutDuringMaintananceError } from '../errors'; import { setTimeout } from 'node:timers/promises'; import { RedisArgument } from '../RESP/types'; @@ -60,6 +60,8 @@ export default class RedisSocket extends EventEmitter { readonly #socketFactory; readonly #socketTimeout; + #maintenanceTimeout: number | undefined; + #socket?: net.Socket | tls.TLSSocket; #isOpen = false; @@ -82,6 +84,12 @@ export default class RedisSocket extends EventEmitter { return this.#socketEpoch; } + #inMaintenance = false; + + set inMaintenance(value: boolean) { + this.#inMaintenance = value; + } + constructor(initiator: RedisSocketInitiator, options?: RedisSocketOptions) { super(); @@ -238,6 +246,18 @@ export default class RedisSocket extends EventEmitter { } while (this.#isOpen && !this.#isReady); } + setMaintenanceTimeout(ms?: number) { + if (this.#maintenanceTimeout === ms) return; + + this.#maintenanceTimeout = ms; + + if(ms !== undefined) { + this.#socket?.setTimeout(ms); + } else { + this.#socket?.setTimeout(this.#socketTimeout ?? 0); + } + } + async #createSocket(): Promise { const socket = this.#socketFactory.create(); @@ -260,7 +280,10 @@ export default class RedisSocket extends EventEmitter { if (this.#socketTimeout) { socket.once('timeout', () => { - socket.destroy(new SocketTimeoutError(this.#socketTimeout!)); + const error = this.#inMaintenance + ? new SocketTimeoutDuringMaintananceError(this.#socketTimeout!) + : new SocketTimeoutError(this.#socketTimeout!) + socket.destroy(error); }); socket.setTimeout(this.#socketTimeout); } From 8cf7c936289baf8bbbc2970dec0a0773f3468c1b Mon Sep 17 00:00:00 2001 From: Nikolay Karadzhov Date: Thu, 14 Aug 2025 14:58:33 +0300 Subject: [PATCH 08/14] feat(client): Add Redis Enterprise maintenance configuration options - Introduced `maintPushNotifications` option to control how the client handles Redis Enterprise maintenance push notifications (`disabled`, `enabled`, `au to`). - Added `maintMovingEndpointType` option to specify the endpoint type for reconnecting during a MOVING notification (`auto`, `internal-ip`, `external-ip`, etc.). - Added `maintRelaxedCommandTimeout` option to define a relaxed timeout for commands during maintenance. - Added `maintRelaxedSocketTimeout` option to define a relaxed timeout for the socket during maintenance. - Enforced RESP3 requirement for maintenance-related features (`maintPushNotifications`). --- packages/client/lib/client/index.ts | 46 ++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/packages/client/lib/client/index.ts b/packages/client/lib/client/index.ts index e5cfc50a8c..d3f6d6ecb0 100644 --- a/packages/client/lib/client/index.ts +++ b/packages/client/lib/client/index.ts @@ -144,7 +144,46 @@ export interface RedisClientOptions< * Tag to append to library name that is sent to the Redis server */ clientInfoTag?: string; -} + /** + * Controls how the client handles Redis Enterprise maintenance push notifications. + * + * - `disabled`: The feature is not used by the client. + * - `enabled`: The client attempts to enable the feature on the server. If the server responds with an error, the connection is interrupted. + * - `auto`: The client attempts to enable the feature on the server. If the server returns an error, the client disables the feature and continues. + * + * The default is `auto`. + */ + maintPushNotifications?: 'disabled' | 'enabled' | 'auto'; + /** + * Controls how the client requests the endpoint to reconnect to during a MOVING notification in Redis Enterprise maintenance. + * + * - `auto`: If the connection is opened to a name or IP address that is from/resolves to a reserved private IP range, request an internal endpoint (e.g., internal-ip), otherwise an external one. If TLS is enabled, then request a FQDN. + * - `internal-ip`: Enforce requesting the internal IP. + * - `internal-fqdn`: Enforce requesting the internal FQDN. + * - `external-ip`: Enforce requesting the external IP address. + * - `external-fqdn`: Enforce requesting the external FQDN. + * - `none`: Used to request a null endpoint, which tells the client to reconnect based on its current config + + * The default is `auto`. + */ + maintMovingEndpointType?: MovingEndpointType; + /** + * Specifies a more relaxed timeout (in milliseconds) for commands during a maintenance window. + * This helps minimize command timeouts during maintenance. If not provided, the `commandOptions.timeout` + * will be used instead. Timeouts during maintenance period result in a `CommandTimeoutDuringMaintanance` error. + * + * The default is 10000 + */ + maintRelaxedCommandTimeout?: number; + /** + * Specifies a more relaxed timeout (in milliseconds) for the socket during a maintenance window. + * This helps minimize socket timeouts during maintenance. If not provided, the `socket.timeout` + * will be used instead. Timeouts during maintenance period result in a `SocketTimeoutDuringMaintanance` error. + * + * The default is 10000 + */ + maintRelaxedSocketTimeout?: number; +}; type WithCommands< RESP extends RespVersions, @@ -485,7 +524,12 @@ export default class RedisClient< throw new Error('Client Side Caching is only supported with RESP3'); } + if (options?.maintPushNotifications && options?.maintPushNotifications !== 'disabled' && options?.RESP !== 3) { + throw new Error('Graceful Maintenance is only supported with RESP3'); + } + } + #initiateOptions(options?: RedisClientOptions): RedisClientOptions | undefined { // Convert username/password to credentialsProvider if no credentialsProvider is already in place From f652d1e59c5f4a05110ecd7ffa5291a9f909d05e Mon Sep 17 00:00:00 2001 From: Nikolay Karadzhov Date: Thu, 14 Aug 2025 15:09:15 +0300 Subject: [PATCH 09/14] feat(client): Add socket helpers and pause mechanism - Introduced `#paused` flag with corresponding `_pause` and `_unpause` methods to temporarily halt writing commands to the socket during maintenance windows. - Updated `#write` method to respect the `#paused` flag, preventing new commands from being written during maintenance. - Added `_ejectSocket` method to safely detach from and return the current socket - Added `_insertSocket` method to receive and start using a new socket --- packages/client/lib/client/index.ts | 48 +++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/packages/client/lib/client/index.ts b/packages/client/lib/client/index.ts index d3f6d6ecb0..321cf6e7e6 100644 --- a/packages/client/lib/client/index.ts +++ b/packages/client/lib/client/index.ts @@ -429,7 +429,7 @@ export default class RedisClient< } readonly #options?: RedisClientOptions; - readonly #socket: RedisSocket; + #socket: RedisSocket; readonly #queue: RedisCommandsQueue; #selectedDB = 0; #monitorCallback?: MonitorCallback; @@ -442,11 +442,16 @@ export default class RedisClient< #watchEpoch?: number; #clientSideCache?: ClientSideCacheProvider; #credentialsSubscription: Disposable | null = null; + // Flag used to pause writing to the socket during maintenance windows. + // When true, prevents new commands from being written while waiting for: + // 1. New socket to be ready after maintenance redirect + // 2. In-flight commands on the old socket to complete + #paused = false; + get clientSideCache() { return this._self.#clientSideCache; } - get options(): RedisClientOptions | undefined { return this._self.#options; } @@ -912,6 +917,42 @@ export default class RedisClient< return this as unknown as RedisClientType; } + /** + * @internal + */ + _ejectSocket(): RedisSocket { + const socket = this._self.#socket; + // @ts-ignore + this.#socket = null; + socket.removeAllListeners(); + return socket; + } + + /** + * @intenal + */ + _insertSocket(socket: RedisSocket) { + if(this._self.#socket) { + this._self._ejectSocket().destroy(); + } + this._self.#socket = socket; + this._self.#attachListeners(this._self.#socket); + } + + /** + * @internal + */ + _pause() { + this._self.#paused = true; + } + + /** + * @internal + */ + _unpause() { + this._self.#paused = false; + } + /** * @internal */ @@ -1141,6 +1182,9 @@ export default class RedisClient< } #write() { + if(this.#paused) { + return + } this.#socket.write(this.#queue.commandsToWrite()); } From 064d78d4839b0a5d042ddb1b31bf326a40f0e70a Mon Sep 17 00:00:00 2001 From: Nikolay Karadzhov Date: Thu, 14 Aug 2025 15:12:34 +0300 Subject: [PATCH 10/14] feat(client): Add Redis Enterprise maintenance handling capabilities - Introduced `EnterpriseMaintenanceManager` to manage Redis Enterprise maintenance events and push notifications. - Integrated `EnterpriseMaintenanceManager` into `RedisClient` to handle maintenance push notifications and manage socket transitions. - Implemented graceful handling of MOVING, MIGRATING, and FAILOVER push notifications, including socket replacement and timeout adjustments. --- packages/client/lib/client/commands-queue.ts | 3 + .../client/enterprise-maintenance-manager.ts | 309 ++++++++++++++++++ packages/client/lib/client/index.ts | 30 +- packages/client/lib/client/socket.ts | 2 + 4 files changed, 341 insertions(+), 3 deletions(-) create mode 100644 packages/client/lib/client/enterprise-maintenance-manager.ts diff --git a/packages/client/lib/client/commands-queue.ts b/packages/client/lib/client/commands-queue.ts index 0f9341b484..ef0db6ad57 100644 --- a/packages/client/lib/client/commands-queue.ts +++ b/packages/client/lib/client/commands-queue.ts @@ -5,6 +5,7 @@ import { TypeMapping, ReplyUnion, RespVersions, RedisArgument } from '../RESP/ty import { ChannelListeners, PubSub, PubSubCommand, PubSubListener, PubSubType, PubSubTypeListeners } from './pub-sub'; import { AbortError, ErrorReply, CommandTimeoutDuringMaintananceError, TimeoutError } from '../errors'; import { MonitorCallback } from '.'; +import { dbgMaintenance } from './enterprise-maintenance-manager'; export interface CommandOptions { chainId?: symbol; @@ -79,6 +80,7 @@ export default class RedisCommandsQueue { #maintenanceCommandTimeout: number | undefined setMaintenanceCommandTimeout(ms: number | undefined) { + dbgMaintenance(`Setting maintenance command timeout to ${ms}`); // Prevent possible api misuse if (this.#maintenanceCommandTimeout === ms) return; @@ -112,6 +114,7 @@ export default class RedisCommandsQueue { }; signal.addEventListener('abort', command.timeout.listener, { once: true }); }; + dbgMaintenance(`Total of ${counter} timeouts reset to ${ms}`); } get isPubSubActive() { diff --git a/packages/client/lib/client/enterprise-maintenance-manager.ts b/packages/client/lib/client/enterprise-maintenance-manager.ts new file mode 100644 index 0000000000..60eaea5057 --- /dev/null +++ b/packages/client/lib/client/enterprise-maintenance-manager.ts @@ -0,0 +1,309 @@ +import { RedisClientOptions } from "."; +import RedisCommandsQueue from "./commands-queue"; +import { RedisArgument } from "../.."; +import { isIP } from "net"; +import { lookup } from "dns/promises"; +import assert from "node:assert"; +import { setTimeout } from "node:timers/promises"; +import RedisSocket from "./socket"; + +export const MAINTENANCE_EVENTS = { + PAUSE_WRITING: "pause-writing", + RESUME_WRITING: "resume-writing", + TIMEOUTS_UPDATE: "timeouts-update", +} as const; + +const PN = { + MOVING: "MOVING", + MIGRATING: "MIGRATING", + MIGRATED: "MIGRATED", + FAILING_OVER: "FAILING_OVER", + FAILED_OVER: "FAILED_OVER", +}; + +export const dbgMaintenance = (...args: any[]) => { + if (!process.env.DEBUG_MAINTENANCE) return; + return console.log("[MNT]", ...args); +}; + +export interface MaintenanceUpdate { + inMaintenance: boolean; + relaxedCommandTimeout?: number; + relaxedSocketTimeout?: number; +} + +interface Client { + _ejectSocket: () => RedisSocket; + _insertSocket: (socket: RedisSocket) => void; + _pause: () => void; + _unpause: () => void; + _maintenanceUpdate: (update: MaintenanceUpdate) => void; + duplicate: (options: RedisClientOptions) => Client; + connect: () => Promise; + destroy: () => void; +} + +export default class EnterpriseMaintenanceManager { + #commandsQueue: RedisCommandsQueue; + #options: RedisClientOptions; + #isMaintenance = 0; + #client: Client; + + static setupDefaultMaintOptions(options: RedisClientOptions) { + if (options.maintPushNotifications === undefined) { + options.maintPushNotifications = + options?.RESP === 3 ? "auto" : "disabled"; + } + if (options.maintMovingEndpointType === undefined) { + options.maintMovingEndpointType = "auto"; + } + if (options.maintRelaxedSocketTimeout === undefined) { + options.maintRelaxedSocketTimeout = 10000; + } + if (options.maintRelaxedCommandTimeout === undefined) { + options.maintRelaxedCommandTimeout = 10000; + } + } + + static async getHandshakeCommand( + tls: boolean, + host: string, + options: RedisClientOptions, + ): Promise< + | { cmd: Array; errorHandler: (error: Error) => void } + | undefined + > { + if (options.maintPushNotifications === "disabled") return; + + const movingEndpointType = await determineEndpoint(tls, host, options); + return { + cmd: [ + "CLIENT", + "MAINT_NOTIFICATIONS", + "ON", + "moving-endpoint-type", + movingEndpointType, + ], + errorHandler: (error: Error) => { + dbgMaintenance("handshake failed:", error); + if (options.maintPushNotifications === "enabled") { + throw error; + } + }, + }; + } + + constructor( + commandsQueue: RedisCommandsQueue, + client: Client, + options: RedisClientOptions, + ) { + this.#commandsQueue = commandsQueue; + this.#options = options; + this.#client = client; + + this.#commandsQueue.addPushHandler(this.#onPush); + } + + #onPush = (push: Array): boolean => { + dbgMaintenance("ONPUSH:", push.map(String)); + switch (push[0].toString()) { + case PN.MOVING: { + // [ 'MOVING', '17', '15', '54.78.247.156:12075' ] + // ^seq ^after ^new ip + const afterSeconds = push[2]; + const url: string | null = push[3] ? String(push[3]) : null; + dbgMaintenance("Received MOVING:", afterSeconds, url); + this.#onMoving(afterSeconds, url); + return true; + } + case PN.MIGRATING: + case PN.FAILING_OVER: { + dbgMaintenance("Received MIGRATING|FAILING_OVER"); + this.#onMigrating(); + return true; + } + case PN.MIGRATED: + case PN.FAILED_OVER: { + dbgMaintenance("Received MIGRATED|FAILED_OVER"); + this.#onMigrated(); + return true; + } + } + return false; + }; + + // Queue: + // toWrite [ C D E ] + // waitingForReply [ A B ] - aka In-flight commands + // + // time: ---1-2---3-4-5-6--------------------------- + // + // 1. [EVENT] MOVING PN received + // 2. [ACTION] Pause writing ( we need to wait for new socket to connect and for all in-flight commands to complete ) + // 3. [EVENT] New socket connected + // 4. [EVENT] In-flight commands completed + // 5. [ACTION] Destroy old socket + // 6. [ACTION] Resume writing -> we are going to write to the new socket from now on + #onMoving = async ( + afterSeconds: number, + url: string | null, + ): Promise => { + // 1 [EVENT] MOVING PN received + this.#onMigrating(); + + let host: string; + let port: number; + + // The special value `none` indicates that the `MOVING` message doesn’t need + // to contain an endpoint. Instead it contains the value `null` then. In + // such a corner case, the client is expected to schedule a graceful + // reconnect to its currently configured endpoint after half of the grace + // period that was communicated by the server is over. + if (url === null) { + assert(this.#options.maintMovingEndpointType === "none"); + assert(this.#options.socket !== undefined); + assert("host" in this.#options.socket); + assert(typeof this.#options.socket.host === "string"); + host = this.#options.socket.host; + assert(typeof this.#options.socket.port === "number"); + port = this.#options.socket.port; + const waitTime = (afterSeconds * 1000) / 2; + dbgMaintenance(`Wait for ${waitTime}ms`); + await setTimeout(waitTime); + } else { + const split = url.split(":"); + host = split[0]; + port = Number(split[1]); + } + + // 2 [ACTION] Pause writing + dbgMaintenance("Pausing writing of new commands to old socket"); + this.#client._pause(); + + const tmpClient = this.#client.duplicate({ + maintPushNotifications: "disabled", + socket: { + ...this.#options.socket, + host, + port, + }, + }); + + dbgMaintenance(`Connecting tmp client: ${host}:${port}`); + await tmpClient.connect(); + dbgMaintenance(`Connected to tmp client`); + // 3 [EVENT] New socket connected + + //TODO + // dbgMaintenance( + // `Set timeout for new socket to ${this.#options.maintRelaxedSocketTimeout}`, + // ); + // newSocket.setMaintenanceTimeout(this.#options.maintRelaxedSocketTimeout); + + dbgMaintenance(`Wait for all in-flight commands to complete`); + await this.#commandsQueue.waitForInflightCommandsToComplete(); + dbgMaintenance(`In-flight commands completed`); + // 4 [EVENT] In-flight commands completed + + dbgMaintenance("Swap client sockets..."); + const oldSocket = this.#client._ejectSocket(); + const newSocket = tmpClient._ejectSocket(); + this.#client._insertSocket(newSocket); + tmpClient._insertSocket(oldSocket); + tmpClient.destroy(); + dbgMaintenance("Swap client sockets done."); + // 5 + 6 + dbgMaintenance("Resume writing"); + this.#client._unpause(); + this.#onMigrated(); + }; + + #onMigrating = async () => { + this.#isMaintenance++; + if (this.#isMaintenance > 1) { + dbgMaintenance(`Timeout relaxation already done`); + return; + } + + const update: MaintenanceUpdate = { + inMaintenance: true, + relaxedCommandTimeout: this.#options.maintRelaxedCommandTimeout, + relaxedSocketTimeout: this.#options.maintRelaxedSocketTimeout, + }; + + this.#client._maintenanceUpdate(update); + }; + + #onMigrated = async () => { + this.#isMaintenance--; + assert(this.#isMaintenance >= 0); + if (this.#isMaintenance > 0) { + dbgMaintenance(`Not ready to unrelax timeouts yet`); + return; + } + + const update: MaintenanceUpdate = { + inMaintenance : false + }; + + this.#client._maintenanceUpdate(update); + }; +} + +export type MovingEndpointType = + | "auto" + | "internal-ip" + | "internal-fqdn" + | "external-ip" + | "external-fqdn" + | "none"; + +function isPrivateIP(ip: string): boolean { + const version = isIP(ip); + if (version === 4) { + const octets = ip.split(".").map(Number); + return ( + octets[0] === 10 || + (octets[0] === 172 && octets[1] >= 16 && octets[1] <= 31) || + (octets[0] === 192 && octets[1] === 168) + ); + } + if (version === 6) { + return ( + ip.startsWith("fc") || // Unique local + ip.startsWith("fd") || // Unique local + ip === "::1" || // Loopback + ip.startsWith("fe80") // Link-local unicast + ); + } + return false; +} + +async function determineEndpoint( + tlsEnabled: boolean, + host: string, + options: RedisClientOptions, +): Promise { + assert(options.maintMovingEndpointType !== undefined); + if (options.maintMovingEndpointType !== "auto") { + dbgMaintenance( + `Determine endpoint type: ${options.maintMovingEndpointType}`, + ); + return options.maintMovingEndpointType; + } + + const ip = isIP(host) ? host : (await lookup(host, { family: 0 })).address; + + const isPrivate = isPrivateIP(ip); + + let result: MovingEndpointType; + if (tlsEnabled) { + result = isPrivate ? "internal-fqdn" : "external-fqdn"; + } else { + result = isPrivate ? "internal-ip" : "external-ip"; + } + + dbgMaintenance(`Determine endpoint type: ${result}`); + return result; +} diff --git a/packages/client/lib/client/index.ts b/packages/client/lib/client/index.ts index 321cf6e7e6..8316c68da7 100644 --- a/packages/client/lib/client/index.ts +++ b/packages/client/lib/client/index.ts @@ -1,5 +1,5 @@ import COMMANDS from '../commands'; -import RedisSocket, { RedisSocketOptions } from './socket'; +import RedisSocket, { RedisSocketOptions, RedisTcpSocketOptions } from './socket'; import { BasicAuth, CredentialsError, CredentialsProvider, StreamingCredentialsProvider, UnableToObtainNewCredentialsError, Disposable } from '../authx'; import RedisCommandsQueue, { CommandOptions } from './commands-queue'; import { EventEmitter } from 'node:events'; @@ -20,6 +20,7 @@ import { BasicClientSideCache, ClientSideCacheConfig, ClientSideCacheProvider } import { BasicCommandParser, CommandParser } from './parser'; import SingleEntryCache from '../single-entry-cache'; import { version } from '../../package.json' +import EnterpriseMaintenanceManager, { MaintenanceUpdate, MovingEndpointType } from './enterprise-maintenance-manager'; export interface RedisClientOptions< M extends RedisModules = RedisModules, @@ -501,6 +502,11 @@ export default class RedisClient< this.#queue = this.#initiateQueue(); this.#socket = this.#initiateSocket(); + + if(options?.maintPushNotifications !== 'disabled') { + new EnterpriseMaintenanceManager(this.#queue, this, this.#options!); + }; + if (options?.clientSideCache) { if (options.clientSideCache instanceof ClientSideCacheProvider) { this.#clientSideCache = options.clientSideCache; @@ -557,13 +563,15 @@ export default class RedisClient< this._commandOptions = options.commandOptions; } + if(options?.maintPushNotifications !== 'disabled') { + EnterpriseMaintenanceManager.setupDefaultMaintOptions(options!); + } + if (options?.url) { const parsedOptions = RedisClient.parseOptions(options); - if (parsedOptions?.database) { this._self.#selectedDB = parsedOptions.database; } - return parsedOptions; } @@ -740,6 +748,12 @@ export default class RedisClient< commands.push({cmd: this.#clientSideCache.trackingOn()}); } + const { tls, host } = this.#options!.socket as RedisTcpSocketOptions; + const maintenanceHandshakeCmd = await EnterpriseMaintenanceManager.getHandshakeCommand(!!tls, host!, this.#options!); + if(maintenanceHandshakeCmd) { + commands.push(maintenanceHandshakeCmd); + }; + return commands; } @@ -939,6 +953,16 @@ export default class RedisClient< this._self.#attachListeners(this._self.#socket); } + /** + * @internal + */ + _maintenanceUpdate(update: MaintenanceUpdate) { + this.#socket.inMaintenance = update.inMaintenance; + this.#socket.setMaintenanceTimeout(update.relaxedSocketTimeout); + this.#queue.inMaintenance = update.inMaintenance; + this.#queue.setMaintenanceCommandTimeout(update.relaxedCommandTimeout); + } + /** * @internal */ diff --git a/packages/client/lib/client/socket.ts b/packages/client/lib/client/socket.ts index 5193e9d8a3..1235b9c00f 100644 --- a/packages/client/lib/client/socket.ts +++ b/packages/client/lib/client/socket.ts @@ -4,6 +4,7 @@ import tls from 'node:tls'; import { ConnectionTimeoutError, ClientClosedError, SocketClosedUnexpectedlyError, ReconnectStrategyError, SocketTimeoutError, SocketTimeoutDuringMaintananceError } from '../errors'; import { setTimeout } from 'node:timers/promises'; import { RedisArgument } from '../RESP/types'; +import { dbgMaintenance } from './enterprise-maintenance-manager'; type NetOptions = { tls?: false; @@ -247,6 +248,7 @@ export default class RedisSocket extends EventEmitter { } setMaintenanceTimeout(ms?: number) { + dbgMaintenance(`Set socket timeout to ${ms}`); if (this.#maintenanceTimeout === ms) return; this.#maintenanceTimeout = ms; From c3d8fe8ebf94a24ba96cbcdf409f3922eb33c996 Mon Sep 17 00:00:00 2001 From: Nikolay Karadzhov Date: Fri, 15 Aug 2025 12:19:25 +0300 Subject: [PATCH 11/14] add _self --- packages/client/lib/client/index.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/client/lib/client/index.ts b/packages/client/lib/client/index.ts index 8316c68da7..e090a6c903 100644 --- a/packages/client/lib/client/index.ts +++ b/packages/client/lib/client/index.ts @@ -937,7 +937,7 @@ export default class RedisClient< _ejectSocket(): RedisSocket { const socket = this._self.#socket; // @ts-ignore - this.#socket = null; + this._self.#socket = null; socket.removeAllListeners(); return socket; } @@ -957,10 +957,10 @@ export default class RedisClient< * @internal */ _maintenanceUpdate(update: MaintenanceUpdate) { - this.#socket.inMaintenance = update.inMaintenance; - this.#socket.setMaintenanceTimeout(update.relaxedSocketTimeout); - this.#queue.inMaintenance = update.inMaintenance; - this.#queue.setMaintenanceCommandTimeout(update.relaxedCommandTimeout); + this._self.#socket.inMaintenance = update.inMaintenance; + this._self.#socket.setMaintenanceTimeout(update.relaxedSocketTimeout); + this._self.#queue.inMaintenance = update.inMaintenance; + this._self.#queue.setMaintenanceCommandTimeout(update.relaxedCommandTimeout); } /** From d0418eb55887b2ab56e65de9b83116a3e0969750 Mon Sep 17 00:00:00 2001 From: Nikolay Karadzhov Date: Fri, 15 Aug 2025 12:58:50 +0300 Subject: [PATCH 12/14] chore(maint): measure tmp client creation --- packages/client/lib/client/enterprise-maintenance-manager.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/client/lib/client/enterprise-maintenance-manager.ts b/packages/client/lib/client/enterprise-maintenance-manager.ts index 60eaea5057..84b6cd4eb4 100644 --- a/packages/client/lib/client/enterprise-maintenance-manager.ts +++ b/packages/client/lib/client/enterprise-maintenance-manager.ts @@ -181,6 +181,8 @@ export default class EnterpriseMaintenanceManager { dbgMaintenance("Pausing writing of new commands to old socket"); this.#client._pause(); + dbgMaintenance("Creating new tmp client"); + const start = performance.now(); const tmpClient = this.#client.duplicate({ maintPushNotifications: "disabled", socket: { @@ -189,7 +191,7 @@ export default class EnterpriseMaintenanceManager { port, }, }); - + dbgMaintenance(`Tmp client created in ${( performance.now() - start ).toFixed(2)}ms`); dbgMaintenance(`Connecting tmp client: ${host}:${port}`); await tmpClient.connect(); dbgMaintenance(`Connected to tmp client`); From 4851fcfddf4e905358f22a2c4094e74b70171117 Mon Sep 17 00:00:00 2001 From: Nikolay Karadzhov Date: Fri, 15 Aug 2025 14:43:25 +0300 Subject: [PATCH 13/14] --wip-- [skip ci] --- packages/client/lib/client/commands-queue.ts | 3 ++- .../client/lib/client/enterprise-maintenance-manager.ts | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/client/lib/client/commands-queue.ts b/packages/client/lib/client/commands-queue.ts index ef0db6ad57..6c7ef38a54 100644 --- a/packages/client/lib/client/commands-queue.ts +++ b/packages/client/lib/client/commands-queue.ts @@ -87,6 +87,7 @@ export default class RedisCommandsQueue { this.#maintenanceCommandTimeout = ms; let counter = 0; + const total = this.#toWrite.length; // Overwrite timeouts of all eligible toWrite commands for(const node of this.#toWrite.nodes()) { @@ -114,7 +115,7 @@ export default class RedisCommandsQueue { }; signal.addEventListener('abort', command.timeout.listener, { once: true }); }; - dbgMaintenance(`Total of ${counter} timeouts reset to ${ms}`); + dbgMaintenance(`Total of ${counter} of ${total} timeouts reset to ${ms}`); } get isPubSubActive() { diff --git a/packages/client/lib/client/enterprise-maintenance-manager.ts b/packages/client/lib/client/enterprise-maintenance-manager.ts index 84b6cd4eb4..3939b8d6f5 100644 --- a/packages/client/lib/client/enterprise-maintenance-manager.ts +++ b/packages/client/lib/client/enterprise-maintenance-manager.ts @@ -182,9 +182,8 @@ export default class EnterpriseMaintenanceManager { this.#client._pause(); dbgMaintenance("Creating new tmp client"); - const start = performance.now(); + let start = performance.now(); const tmpClient = this.#client.duplicate({ - maintPushNotifications: "disabled", socket: { ...this.#options.socket, host, @@ -193,8 +192,9 @@ export default class EnterpriseMaintenanceManager { }); dbgMaintenance(`Tmp client created in ${( performance.now() - start ).toFixed(2)}ms`); dbgMaintenance(`Connecting tmp client: ${host}:${port}`); + start = performance.now(); await tmpClient.connect(); - dbgMaintenance(`Connected to tmp client`); + dbgMaintenance(`Connected to tmp client in ${(performance.now() - start).toFixed(2)}ms`); // 3 [EVENT] New socket connected //TODO From 860c5cf77ed77f69613a2aa3c6d48b08a98444b1 Mon Sep 17 00:00:00 2001 From: Nikolay Karadzhov Date: Fri, 15 Aug 2025 15:07:15 +0300 Subject: [PATCH 14/14] --wip-- [skip ci] --- packages/client/lib/client/commands-queue.ts | 31 +++++++++---------- .../client/enterprise-maintenance-manager.ts | 5 ++- packages/client/lib/client/index.ts | 2 -- packages/client/lib/client/socket.ts | 15 ++++----- 4 files changed, 23 insertions(+), 30 deletions(-) diff --git a/packages/client/lib/client/commands-queue.ts b/packages/client/lib/client/commands-queue.ts index 6c7ef38a54..6893a04f14 100644 --- a/packages/client/lib/client/commands-queue.ts +++ b/packages/client/lib/client/commands-queue.ts @@ -71,21 +71,23 @@ export default class RedisCommandsQueue { #pushHandlers: PushHandler[] = [this.#onPush.bind(this)]; - #inMaintenance = false; - - set inMaintenance(value: boolean) { - this.#inMaintenance = value; - } - #maintenanceCommandTimeout: number | undefined setMaintenanceCommandTimeout(ms: number | undefined) { - dbgMaintenance(`Setting maintenance command timeout to ${ms}`); // Prevent possible api misuse - if (this.#maintenanceCommandTimeout === ms) return; + if (this.#maintenanceCommandTimeout === ms) { + dbgMaintenance(`Queue already set maintenanceCommandTimeout to ${ms}, skipping`); + return; + }; + dbgMaintenance(`Setting maintenance command timeout to ${ms}`); this.#maintenanceCommandTimeout = ms; + if(this.#maintenanceCommandTimeout === undefined) { + dbgMaintenance(`Queue will keep maintenanceCommandTimeout for exisitng commands, just to be on the safe side. New commands will receive normal timeouts`); + return; + } + let counter = 0; const total = this.#toWrite.length; @@ -96,12 +98,8 @@ export default class RedisCommandsQueue { // Remove timeout listener if it exists RedisCommandsQueue.#removeTimeoutListener(command) - // Determine newTimeout - const newTimeout = this.#maintenanceCommandTimeout ?? command.timeout?.originalTimeout; - // if no timeout is given and the command didnt have any timeout before, skip - if (!newTimeout) return; - counter++; + const newTimeout = this.#maintenanceCommandTimeout; // Overwrite the command's timeout const signal = AbortSignal.timeout(newTimeout); @@ -109,7 +107,7 @@ export default class RedisCommandsQueue { signal, listener: () => { this.#toWrite.remove(node); - command.reject(this.#inMaintenance ? new CommandTimeoutDuringMaintananceError(newTimeout) : new TimeoutError()); + command.reject(new CommandTimeoutDuringMaintananceError(newTimeout)); }, originalTimeout: command.timeout?.originalTimeout }; @@ -224,7 +222,8 @@ export default class RedisCommandsQueue { // If #maintenanceCommandTimeout was explicitly set, we should // use it instead of the timeout provided by the command - const timeout = this.#maintenanceCommandTimeout || options?.timeout + const timeout = this.#maintenanceCommandTimeout ?? options?.timeout; + const wasInMaintenance = this.#maintenanceCommandTimeout !== undefined; if (timeout) { const signal = AbortSignal.timeout(timeout); @@ -232,7 +231,7 @@ export default class RedisCommandsQueue { signal, listener: () => { this.#toWrite.remove(node); - value.reject(this.#inMaintenance ? new CommandTimeoutDuringMaintananceError(timeout) : new TimeoutError()); + value.reject(wasInMaintenance ? new CommandTimeoutDuringMaintananceError(timeout) : new TimeoutError()); }, originalTimeout: options?.timeout }; diff --git a/packages/client/lib/client/enterprise-maintenance-manager.ts b/packages/client/lib/client/enterprise-maintenance-manager.ts index 3939b8d6f5..69cf7c7ce2 100644 --- a/packages/client/lib/client/enterprise-maintenance-manager.ts +++ b/packages/client/lib/client/enterprise-maintenance-manager.ts @@ -27,7 +27,6 @@ export const dbgMaintenance = (...args: any[]) => { }; export interface MaintenanceUpdate { - inMaintenance: boolean; relaxedCommandTimeout?: number; relaxedSocketTimeout?: number; } @@ -229,7 +228,6 @@ export default class EnterpriseMaintenanceManager { } const update: MaintenanceUpdate = { - inMaintenance: true, relaxedCommandTimeout: this.#options.maintRelaxedCommandTimeout, relaxedSocketTimeout: this.#options.maintRelaxedSocketTimeout, }; @@ -246,7 +244,8 @@ export default class EnterpriseMaintenanceManager { } const update: MaintenanceUpdate = { - inMaintenance : false + relaxedCommandTimeout: undefined, + relaxedSocketTimeout: undefined }; this.#client._maintenanceUpdate(update); diff --git a/packages/client/lib/client/index.ts b/packages/client/lib/client/index.ts index e090a6c903..511f34f3d3 100644 --- a/packages/client/lib/client/index.ts +++ b/packages/client/lib/client/index.ts @@ -957,9 +957,7 @@ export default class RedisClient< * @internal */ _maintenanceUpdate(update: MaintenanceUpdate) { - this._self.#socket.inMaintenance = update.inMaintenance; this._self.#socket.setMaintenanceTimeout(update.relaxedSocketTimeout); - this._self.#queue.inMaintenance = update.inMaintenance; this._self.#queue.setMaintenanceCommandTimeout(update.relaxedCommandTimeout); } diff --git a/packages/client/lib/client/socket.ts b/packages/client/lib/client/socket.ts index 1235b9c00f..9d8ebdae07 100644 --- a/packages/client/lib/client/socket.ts +++ b/packages/client/lib/client/socket.ts @@ -85,12 +85,6 @@ export default class RedisSocket extends EventEmitter { return this.#socketEpoch; } - #inMaintenance = false; - - set inMaintenance(value: boolean) { - this.#inMaintenance = value; - } - constructor(initiator: RedisSocketInitiator, options?: RedisSocketOptions) { super(); @@ -249,7 +243,10 @@ export default class RedisSocket extends EventEmitter { setMaintenanceTimeout(ms?: number) { dbgMaintenance(`Set socket timeout to ${ms}`); - if (this.#maintenanceTimeout === ms) return; + if (this.#maintenanceTimeout === ms) { + dbgMaintenance(`Socket already set maintenanceCommandTimeout to ${ms}, skipping`); + return; + }; this.#maintenanceTimeout = ms; @@ -282,8 +279,8 @@ export default class RedisSocket extends EventEmitter { if (this.#socketTimeout) { socket.once('timeout', () => { - const error = this.#inMaintenance - ? new SocketTimeoutDuringMaintananceError(this.#socketTimeout!) + const error = this.#maintenanceTimeout + ? new SocketTimeoutDuringMaintananceError(this.#maintenanceTimeout) : new SocketTimeoutError(this.#socketTimeout!) socket.destroy(error); });