diff --git a/packages/client/lib/client/commands-queue.ts b/packages/client/lib/client/commands-queue.ts index 52a07a7e3b..ca17dcfe9b 100644 --- a/packages/client/lib/client/commands-queue.ts +++ b/packages/client/lib/client/commands-queue.ts @@ -1,10 +1,11 @@ -import { SinglyLinkedList, DoublyLinkedNode, DoublyLinkedList } from './linked-list'; +import { DoublyLinkedNode, DoublyLinkedList, EmptyAwareSinglyLinkedList } from './linked-list'; import encodeCommand from '../RESP/encoder'; import { Decoder, PUSH_TYPE_MAPPING, RESP_TYPES } from '../RESP/decoder'; import { TypeMapping, ReplyUnion, RespVersions, RedisArgument } from '../RESP/types'; import { ChannelListeners, PubSub, PubSubCommand, PubSubListener, PubSubType, PubSubTypeListeners } from './pub-sub'; import { AbortError, ErrorReply, TimeoutError } from '../errors'; import { MonitorCallback } from '.'; +import EventEmitter from 'events'; export interface CommandOptions { chainId?: symbol; @@ -54,11 +55,12 @@ export default class RedisCommandsQueue { readonly #respVersion; readonly #maxLength; readonly #toWrite = new DoublyLinkedList(); - readonly #waitingForReply = new SinglyLinkedList(); + readonly #waitingForReply = new EmptyAwareSinglyLinkedList(); readonly #onShardedChannelMoved; #chainInExecution: symbol | undefined; readonly decoder; readonly #pubSub = new PubSub(); + readonly events = new EventEmitter(); get isPubSubActive() { return this.#pubSub.isActive; @@ -134,6 +136,12 @@ export default class RedisCommandsQueue { } break; } + case 'MOVING': { + const [_, afterMs, url] = push; + const [host, port] = url.toString().split(':'); + this.events.emit('moving', afterMs, host, Number(port)) + break; + } } } }, @@ -145,6 +153,17 @@ export default class RedisCommandsQueue { this.#invalidateCallback = callback; } + async waitForInflightCommandsToComplete(): Promise { + // In-flight commands already completed + if(this.#waitingForReply.length === 0) { + return + }; + // Otherwise wait for in-flight commands to fire `empty` event + return new Promise(resolve => { + this.#waitingForReply.events.on('empty', resolve) + }); + } + addCommand( args: ReadonlyArray, options?: CommandOptions diff --git a/packages/client/lib/client/enterprise-maintenance-manager.ts b/packages/client/lib/client/enterprise-maintenance-manager.ts new file mode 100644 index 0000000000..a69f41a13e --- /dev/null +++ b/packages/client/lib/client/enterprise-maintenance-manager.ts @@ -0,0 +1,56 @@ +import EventEmitter from "events"; +import { RedisClientOptions } from "."; +import RedisCommandsQueue from "./commands-queue"; +import RedisSocket from "./socket"; + +export default class EnterpriseMaintenanceManager extends EventEmitter { + commandsQueue: RedisCommandsQueue; + options: RedisClientOptions; + constructor( + commandsQueue: RedisCommandsQueue, + options: RedisClientOptions, + ) { + super(); + this.commandsQueue = commandsQueue; + this.options = options; + + this.commandsQueue.events.on("moving", this.#onMoving); + } + + // Queue: + // toWrite [ C D E ] + // waitingForReply [ A B ] - aka In-flight commands + // + // time: ---1-2---3-4-5-6--------------------------- + // + // 1. [EVENT] MOVING PN received + // 2. [ACTION] Pause writing ( we need to wait for new socket to connect and for all in-flight commands to complete ) + // 3. [EVENT] New socket connected + // 4. [EVENT] In-flight commands completed + // 5. [ACTION] Destroy old socket + // 6. [ACTION] Resume writing -> we are going to write to the new socket from now on + #onMoving = async ( + _afterMs: number, + host: string, + port: number, + ): Promise => { + // 1 [EVENT] MOVING PN received + // 2 [ACTION] Pause writing + this.emit('pause') + + const newSocket = new RedisSocket({ + ...this.options.socket, + host, + port, + }); + await newSocket.connect(); + // 3 [EVENT] New socket connected + + await this.commandsQueue.waitForInflightCommandsToComplete(); + // 4 [EVENT] In-flight commands completed + + // 5 + 6 + this.emit('resume', newSocket); + + }; +} diff --git a/packages/client/lib/client/index.ts b/packages/client/lib/client/index.ts index 128dc59967..f7dda94f30 100644 --- a/packages/client/lib/client/index.ts +++ b/packages/client/lib/client/index.ts @@ -20,6 +20,7 @@ import { BasicClientSideCache, ClientSideCacheConfig, ClientSideCacheProvider } import { BasicCommandParser, CommandParser } from './parser'; import SingleEntryCache from '../single-entry-cache'; import { version } from '../../package.json' +import EnterpriseMaintenanceManager from './enterprise-maintenance-manager'; export interface RedisClientOptions< M extends RedisModules = RedisModules, @@ -144,6 +145,44 @@ export interface RedisClientOptions< * Tag to append to library name that is sent to the Redis server */ clientInfoTag?: string; + + /** + * Configuration for handling Redis Enterprise graceful maintenance scenarios. + * + * When Redis Enterprise performs maintenance operations, nodes will be replaced, resulting in disconnects. + * This configuration allows the client to handle these scenarios gracefully by automatically + * reconnecting and managing command execution during maintenance windows. + * + * @example Basic graceful maintenance configuration + * ``` + * const client = createClient({ + * gracefulMaintenance: { + * handleFailedCommands: 'retry', + * handleTimeouts: 'exception', + * } + * }); + * ``` + * + * @example Graceful maintenance with timeout smoothing + * ``` + * const client = createClient({ + * gracefulMaintenance: { + * handleFailedCommands: 'retry', + * handleTimeouts: 5000, // Extend timeouts to 5 seconds during maintenance + * } + * }); + * ``` + */ + gracefulMaintenance?: { + /** + * Designates how failed commands should be handled. A failed command is when the time isn’t sufficient to deal with the responses on the old connection before the server shuts it down + */ + handleFailedCommands: 'exception' | 'retry', + /** + * Specify whether we should throw a MaintenanceTimeout exception or provide more relaxed timeout, in order to minimize command timeouts during maintenance. + */ + handleTimeouts: 'exception' | number, + } } type WithCommands< @@ -366,7 +405,7 @@ export default class RedisClient< } readonly #options?: RedisClientOptions; - readonly #socket: RedisSocket; + #socket: RedisSocket; readonly #queue: RedisCommandsQueue; #selectedDB = 0; #monitorCallback?: MonitorCallback; @@ -379,11 +418,16 @@ export default class RedisClient< #watchEpoch?: number; #clientSideCache?: ClientSideCacheProvider; #credentialsSubscription: Disposable | null = null; + // Flag used to pause writing to the socket during maintenance windows. + // When true, prevents new commands from being written while waiting for: + // 1. New socket to be ready after maintenance redirect + // 2. In-flight commands on the old socket to complete + #paused = false; + get clientSideCache() { return this._self.#clientSideCache; } - get options(): RedisClientOptions | undefined { return this._self.#options; } @@ -417,6 +461,19 @@ export default class RedisClient< return this._self.#dirtyWatch !== undefined } + #pauseForMaintenance() { + this._self.#paused = true; + } + + #resumeFromMaintenance(newSocket: RedisSocket) { + this._self.#socket.removeAllListeners(); + this._self.#socket.destroy(); + this._self.#socket = newSocket; + this._self.#paused = false; + this._self.#initiateSocket(); + this._self.#maybeScheduleWrite(); + } + /** * Marks the client's WATCH command as invalidated due to a topology change. * This will cause any subsequent EXEC in a transaction to fail with a WatchError. @@ -431,7 +488,13 @@ export default class RedisClient< this.#validateOptions(options) this.#options = this.#initiateOptions(options); this.#queue = this.#initiateQueue(); - this.#socket = this.#initiateSocket(); + this.#socket = this.#createSocket(this.#options); + + if(options?.gracefulMaintenance) { + new EnterpriseMaintenanceManager(this.#queue, this.#options!) + .on('pause', this.#pauseForMaintenance.bind(this)) + .on('resume', this.#resumeFromMaintenance.bind(this)) + } if (options?.clientSideCache) { if (options.clientSideCache instanceof ClientSideCacheProvider) { @@ -449,7 +512,12 @@ export default class RedisClient< throw new Error('Client Side Caching is only supported with RESP3'); } + if (options?.gracefulMaintenance && options?.RESP !== 3) { + throw new Error('Graceful Maintenance is only supported with RESP3'); + } + } + #initiateOptions(options?: RedisClientOptions): RedisClientOptions | undefined { // Convert username/password to credentialsProvider if no credentialsProvider is already in place @@ -657,38 +725,10 @@ export default class RedisClient< return commands; } - #initiateSocket(): RedisSocket { - const socketInitiator = async () => { - const promises = [], - chainId = Symbol('Socket Initiator'); - - const resubscribePromise = this.#queue.resubscribe(chainId); - if (resubscribePromise) { - promises.push(resubscribePromise); - } + async #initiateSocket(): Promise { + await this.#socket.waitForReady(); - if (this.#monitorCallback) { - promises.push( - this.#queue.monitor( - this.#monitorCallback, - { - typeMapping: this._commandOptions?.typeMapping, - chainId, - asap: true - } - ) - ); - } - - promises.push(...(await this.#handshake(chainId, true))); - - if (promises.length) { - this.#write(); - return Promise.all(promises); - } - }; - - return new RedisSocket(socketInitiator, this.#options?.socket) + this.#socket .on('data', chunk => { try { this.#queue.decoder.write(chunk); @@ -706,15 +746,47 @@ export default class RedisClient< this.#queue.flushAll(err); } }) - .on('connect', () => this.emit('connect')) - .on('ready', () => { - this.emit('ready'); - this.#setPingTimer(); - this.#maybeScheduleWrite(); - }) .on('reconnecting', () => this.emit('reconnecting')) .on('drain', () => this.#maybeScheduleWrite()) .on('end', () => this.emit('end')); + + const promises = []; + const chainId = Symbol('Socket Initiator'); + + const resubscribePromise = this.#queue.resubscribe(chainId); + if (resubscribePromise) { + promises.push(resubscribePromise); + } + + if (this.#monitorCallback) { + promises.push( + this.#queue.monitor( + this.#monitorCallback, + { + typeMapping: this._commandOptions?.typeMapping, + chainId, + asap: true + } + ) + ); + } + + promises.push(...(await this.#handshake(chainId, true))); + + this.#setPingTimer(); + + if (promises.length) { + this.#write(); + await Promise.all(promises); + } + } + + #createSocket(options?: RedisClientOptions): RedisSocket { + return new RedisSocket(options?.socket) + .on('connect', () => this.emit('connect')) + .on('ready', () => { + this.emit('ready'); + }); } #pingTimer?: NodeJS.Timeout; @@ -823,6 +895,7 @@ export default class RedisClient< async connect() { await this._self.#socket.connect(); + await this._self.#initiateSocket(); return this as unknown as RedisClientType; } @@ -1055,6 +1128,9 @@ export default class RedisClient< } #write() { + if(this.#paused) { + return + } this.#socket.write(this.#queue.commandsToWrite()); } diff --git a/packages/client/lib/client/linked-list.ts b/packages/client/lib/client/linked-list.ts index 29678f027b..6db71fe529 100644 --- a/packages/client/lib/client/linked-list.ts +++ b/packages/client/lib/client/linked-list.ts @@ -1,3 +1,5 @@ +import EventEmitter from "events"; + export interface DoublyLinkedNode { value: T; previous: DoublyLinkedNode | undefined; @@ -32,7 +34,7 @@ export class DoublyLinkedList { next: undefined, value }; - } + } return this.#tail = this.#tail.next = { previous: this.#tail, @@ -93,7 +95,7 @@ export class DoublyLinkedList { node.previous!.next = node.next; node.previous = undefined; } - + node.next = undefined; } @@ -201,3 +203,26 @@ export class SinglyLinkedList { } } } + +export class EmptyAwareSinglyLinkedList extends SinglyLinkedList { + readonly events = new EventEmitter(); + reset() { + super.reset(); + this.events.emit('empty'); + } + shift(): T | undefined { + const old = this.length; + const ret = super.shift(); + if(old !== this.length && this.length === 0) { + this.events.emit('empty'); + } + return ret; + } + remove(node: SinglyLinkedNode, parent: SinglyLinkedNode | undefined) { + const old = this.length; + super.remove(node, parent); + if(old !== this.length && this.length === 0) { + this.events.emit('empty'); + } + } +} diff --git a/packages/client/lib/client/socket.ts b/packages/client/lib/client/socket.ts index 5f0bcc4492..5d28f59ace 100644 --- a/packages/client/lib/client/socket.ts +++ b/packages/client/lib/client/socket.ts @@ -51,10 +51,7 @@ export type RedisTcpSocketOptions = RedisTcpOptions | RedisTlsOptions; export type RedisSocketOptions = RedisTcpSocketOptions | RedisIpcOptions; -export type RedisSocketInitiator = () => void | Promise; - export default class RedisSocket extends EventEmitter { - readonly #initiator; readonly #connectTimeout; readonly #reconnectStrategy; readonly #socketFactory; @@ -82,16 +79,23 @@ export default class RedisSocket extends EventEmitter { return this.#socketEpoch; } - constructor(initiator: RedisSocketInitiator, options?: RedisSocketOptions) { + constructor(options?: RedisSocketOptions) { super(); - this.#initiator = initiator; this.#connectTimeout = options?.connectTimeout ?? 5000; this.#reconnectStrategy = this.#createReconnectStrategy(options); this.#socketFactory = this.#createSocketFactory(options); this.#socketTimeout = options?.socketTimeout; } + async waitForReady(): Promise { + if (this.#isReady) return + return new Promise((resolve, reject) => { + this.once('ready', resolve); + this.once('error', reject); + }); + } + #createReconnectStrategy(options?: RedisSocketOptions): ReconnectStrategyFunction { const strategy = options?.reconnectStrategy; if (strategy === false || typeof strategy === 'number') { @@ -214,14 +218,6 @@ export default class RedisSocket extends EventEmitter { try { this.#socket = await this.#createSocket(); this.emit('connect'); - - try { - await this.#initiator(); - } catch (err) { - this.#socket.destroy(); - this.#socket = undefined; - throw err; - } this.#isReady = true; this.#socketEpoch++; this.emit('ready');