From 16e8c6eed0ae1ec10dde35ac316b8ec6af2ffa5d Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Fri, 21 Feb 2025 14:54:52 -0800 Subject: [PATCH 01/22] Update rate limit with redis store --- app.js | 47 +--- bin/www | 98 +++++-- lib/rateLimit.js | 127 +++++++++ package-lock.json | 595 +++++++++++++++++++++++++++++------------- package.json | 7 +- test/app.js | 13 +- test/lib/rateLimit.js | 196 ++++++++++++++ 7 files changed, 832 insertions(+), 251 deletions(-) mode change 100755 => 100644 bin/www create mode 100644 lib/rateLimit.js create mode 100644 test/lib/rateLimit.js diff --git a/app.js b/app.js index 51ff40142..8db4a9021 100644 --- a/app.js +++ b/app.js @@ -6,23 +6,18 @@ const morgan = require('morgan') const bodyParser = require('body-parser') const cookieParser = require('cookie-parser') const cors = require('cors') -const rateLimit = require('express-rate-limit') -const rateLimitRedisStore = require('rate-limit-redis') -const redis = require('redis') const helmet = require('helmet') const serializeError = require('serialize-error') const requestId = require('request-id/express') const passport = require('passport') const swaggerUi = require('swagger-ui-express') -const loggerFactory = require('./providers/logging/logger') const routesVersioning = require('express-routes-versioning')() const v1 = '1.0.0' -function createApp(config) { +function createApp(config, { logger, rateLimiter, batchRateLimiter }) { const initializers = [] - const logger = loggerFactory(config.logging.logger()) process.on('unhandledRejection', exception => logger.error('unhandledRejection', exception)) config.auth.service.permissionsSetup() @@ -133,30 +128,7 @@ function createApp(config) { app.set('trust-proxy', true) - // If Redis is configured for caching, connect to it - const client = config.caching.caching_redis_service - ? redis.createClient(6380, config.caching.caching_redis_service, { - auth_pass: config.caching.caching_redis_api_key, - tls: { servername: config.caching_redis_service } - }) - : undefined - - // rate-limit the remaining routes - const apiLimiter = config.caching.caching_redis_service - ? rateLimit({ - store: new rateLimitRedisStore({ - client: client, - prefix: 'api' - }), - windowMs: config.limits.windowSeconds * 1000, - max: config.limits.max - }) - : rateLimit({ - windowMs: config.limits.windowSeconds * 1000, - max: config.limits.max - }) - - app.use(apiLimiter) + app.use(rateLimiter.middleware) // Use a (potentially lower) different API limit // for batch API request @@ -164,20 +136,7 @@ function createApp(config) { // * POST /definitions // * POST /curations // * POST /notices - const batchApiLimiter = config.caching.caching_redis_service - ? rateLimit({ - store: new rateLimitRedisStore({ - client: client, - prefix: 'batch-api' - }), - windowMs: config.limits.batchWindowSeconds * 1000, - max: config.limits.batchMax - }) - : rateLimit({ - windowMs: config.limits.batchWindowSeconds * 1000, - max: config.limits.batchMax - }) - + const batchApiLimiter = batchRateLimiter.middleware app.post('/definitions', batchApiLimiter) app.post('/curations', batchApiLimiter) app.post('/notices', batchApiLimiter) diff --git a/bin/www b/bin/www old mode 100755 new mode 100644 index fee9cba45..e1188ad57 --- a/bin/www +++ b/bin/www @@ -3,34 +3,46 @@ // SPDX-License-Identifier: MIT const config = require('./config') -const app = require('../app')(config) +const loggerFactory = require('../providers/logging/logger') +const { createApiLimiter, createBatchApiLimiter } = require('../lib/rateLimit') +const createApp = require('../app') const debug = require('debug')('service:server') const http = require('http') const init = require('express-init') -/** - * Get port from environment and store in Express. - */ -const port = normalizePort(process.env.PORT || '4000') -app.set('port', port) +connect(config).then(resources => { + const app = createApp(config, resources) -/** - * Create HTTP server. - */ -const server = http.createServer(app) + /** + * Get port from environment and store in Express. + */ + const port = normalizePort(process.env.PORT || '4000') + app.set('port', port) -/** - * Initialize the apps (if they have async init functions) and start listening - */ -init(app, error => { - if (error) { - console.log('Error initializing the Express app: ' + error) - throw new Error(error) - } - server.listen(port) - server.on('error', onError) - server.on('listening', onListening) - console.log(`Service listening on port: ${port}`) + /** + * Create HTTP server. + */ + const server = http.createServer(app) + + /** + * Initialize the apps (if they have async init functions) and start listening + */ + init(app, error => { + if (error) { + console.log('Error initializing the Express app: ' + error) + throw new Error(error) + } + server.listen(port) + server.on('error', onError) + server.on('listening', () => onListening(server)) + console.log(`Service listening on port: ${port}`) + }) + + /** + * Handles graceful shutdown + */ + process.on('SIGINT', () => onShutdown(server, resources)) + process.on('SIGTERM', () => onShutdown(server, resources)) }) /** @@ -69,8 +81,48 @@ function onError(error) { /** * Event listener for HTTP server "listening" event. */ -function onListening() { +function onListening(server) { const addr = server.address() const bind = typeof addr === 'string' ? 'pipe ' + addr : 'port ' + addr.port debug('Listening on ' + bind) } + +/** + * + * Handles gradelful shutdown + * + * @param server - The server instance to be shut down. + * @param {Object} resources - The resources to be cleaned up during shutdown. + */ +const onShutdown = (server, { rateLimiter, batchRateLimiter, logger }) => { + logger.info('Shutdown started') + server.close(() => { + logger.info('Server closed') + Promise.allSettled([rateLimiter.done(), batchRateLimiter.done()]).then(results => { + const errorResults = results.filter(result => result.status === 'rejected') + if (errorResults.length === 0) { + logger.info('Shutdown complete') + process.exit(0) + } + errorResults.forEach(errorResult => logger.error(errorResult.reason)) + process.exit(1) + }) + }) +} + +/** + * Connect to the required services and return the resources + * + * @param {Object} config - The configuration object + * @returns {Promise} - The resources object + */ +async function connect(config) { + const logger = loggerFactory(config.logging.logger()) + const rateLimiter = createApiLimiter(config, logger) + const batchRateLimiter = createBatchApiLimiter(config, logger) + await Promise.all([rateLimiter.initialize(), batchRateLimiter.initialize()]).catch(error => { + logger.error('Error initializing rate limiters', error) + process.exit(1) + }) + return { rateLimiter, batchRateLimiter, logger } +} \ No newline at end of file diff --git a/lib/rateLimit.js b/lib/rateLimit.js new file mode 100644 index 000000000..5c8edfae5 --- /dev/null +++ b/lib/rateLimit.js @@ -0,0 +1,127 @@ +// (c) Copyright 2025, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const { createClient } = require('redis') +const { RedisStore } = require('rate-limit-redis') +const { rateLimit } = require('express-rate-limit') + +class RateLimiter { + constructor(opts) { + this.options = opts + this.logger = opts.logger + } + + async initialize(store) { + if (!this._limiter) { + this.logger.debug('Creating rate limiter: %o', this.options.limit) + this._limiter = RateLimiter.build(this.options.limit, store) + this.logger.info('Rate limiter initialized') + } + } + + get middleware() { + return this._limiter + } + + async done() { + //do nothing + this.logger.info('Rate limiter done') + } + + static build({ windowMs, max }, store) { + //TODO: use standardHeaders? + const opts = { + windowMs, + max, + standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers + legacyHeaders: false // Disable the `X-RateLimit-*` headers + } + if (store) { + opts.store = store + } + return rateLimit(opts) + } +} + +class RedisBackedRateLimiter extends RateLimiter { + async initialize() { + if (!this._limiter) { + this._client = await this._initializeClient() + const store = RedisBackedRateLimiter.buildRedisStore(this._client, this.options.redis) + await super.initialize(store) + } + } + + async done() { + return this._client?.disconnect().then(() => super.done()) + } + + async _initializeClient() { + if (!this.options.redis) throw new Error('Redis configuration is missing') + const client = RedisBackedRateLimiter.buildRedisClient(this.options.redis) + client.on('error', error => { + this.logger.error(`Redis client error: ${error}`) + }) + await client.connect().then(() => this.logger.info('Done connecting to redis: %s', this.options.redis.service)) + return client + } + + static buildRedisStore(client, { prefix }) { + return new RedisStore({ + prefix, + sendCommand: (...args) => client.sendCommand(args) + }) + } + + static buildRedisClient({ apiKey, service }) { + return createClient({ + username: 'default', + password: apiKey, + socket: { + host: service, + port: 6380, + tls: true + }, + pingInterval: 5 * 60 * 1000 // https://learn.microsoft.com/en-us/azure/azure-cache-for-redis/cache-best-practices-connection#idle-timeout + }) + } +} + +function createRateLimiter(config) { + return config.redis ? new RedisBackedRateLimiter(config) : new RateLimiter(config) +} + +function buildOpts({ windowSeconds, max }, logger, { caching_redis_service, caching_redis_api_key } = {}, prefix) { + const limit = { + windowMs: windowSeconds * 1000, + max + } + let redis + if (caching_redis_service) { + redis = { + service: caching_redis_service, + apiKey: caching_redis_api_key, + prefix + } + } + return { limit, redis, logger } +} + +function buildBatchOpts({ batchWindowSeconds, batchMax }, ...args) { + return buildOpts({ windowSeconds: batchWindowSeconds, max: batchMax }, ...args) +} + +function createApiLimiter(config, logger) { + return createRateLimiter(buildOpts(config.limits, logger, config.caching, 'api')) +} + +function createBatchApiLimiter(config, logger) { + return createRateLimiter(buildBatchOpts(config.limits, logger, config.caching, 'batch-api')) +} + +module.exports = { + createApiLimiter, + createBatchApiLimiter, + RedisBackedRateLimiter, + RateLimiter +} diff --git a/package-lock.json b/package-lock.json index fc8d3aa2d..5681150fe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,7 +27,7 @@ "debug": "~2.6.9", "express": "^4.19.2", "express-init": "^1.1.1", - "express-rate-limit": "^6.0.4", + "express-rate-limit": "^7.5.0", "express-routes-versioning": "^1.0.1", "extend": "^3.0.2", "geit": "0.0.7", @@ -47,10 +47,10 @@ "passport-github": "^1.1.0", "patch-package": "^6.5.1", "promise-retry": "^1.1.1", - "rate-limit-redis": "^2.1.0", + "rate-limit-redis": "^4.2.0", "readdirp": "^2.1.0", "recursive-readdir": "^2.2.1", - "redis": "^3.1.1", + "redis": "^4.7.0", "request-id": "^0.11.1", "semver": "7.6.0", "serialize-error": "^2.1.0", @@ -82,6 +82,7 @@ "prettier": "3.2.5", "proxyquire": "^2.0.1", "sinon": "^5.0.0", + "supertest": "7.0.0", "typescript": "5.0.4" } }, @@ -2379,6 +2380,59 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.0.tgz", + "integrity": "sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz", + "integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz", + "integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz", + "integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/@sindresorhus/is": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", @@ -2878,6 +2932,12 @@ "integrity": "sha1-z+nYwmYoudxa7MYqn12PHzUsEZU=", "dev": true }, + "node_modules/asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", + "dev": true + }, "node_modules/asn1": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", @@ -3473,16 +3533,25 @@ "node": ">=8" } }, - "node_modules/call-bind": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", - "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", "dependencies": { - "es-define-property": "^1.0.0", "es-errors": "^1.3.0", - "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.4", - "set-function-length": "^1.2.1" + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.3.tgz", + "integrity": "sha512-YTd+6wGlNlPxSuri7Y6X8tY2dmm12UMH66RpKMhiX6rsk5wXXnYgbUcOt8kiS31/AjfoTOvCsE+w8nZQLQnzHA==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "get-intrinsic": "^1.2.6" }, "engines": { "node": ">= 0.4" @@ -3717,14 +3786,6 @@ "node": ">=6" } }, - "node_modules/clone": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", - "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=", - "engines": { - "node": ">=0.8" - } - }, "node_modules/clone-response": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", @@ -3754,6 +3815,14 @@ "semver": "bin/semver" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/color-convert": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.1.tgz", @@ -3803,9 +3872,12 @@ "dev": true }, "node_modules/component-emitter": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", - "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=" + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.1.tgz", + "integrity": "sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } }, "node_modules/concat-map": { "version": "0.0.1", @@ -4175,14 +4247,6 @@ "node": ">=8" } }, - "node_modules/defaults": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz", - "integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=", - "dependencies": { - "clone": "^1.0.2" - } - }, "node_modules/defer-to-connect": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz", @@ -4197,22 +4261,6 @@ "abstract-leveldown": "~2.6.0" } }, - "node_modules/define-data-property": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", - "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "dependencies": { - "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "gopd": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/degenerator": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", @@ -4235,14 +4283,6 @@ "node": ">=0.4.0" } }, - "node_modules/denque": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", - "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==", - "engines": { - "node": ">=0.10" - } - }, "node_modules/depd": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz", @@ -4272,6 +4312,16 @@ "node": ">=12.0.0" } }, + "node_modules/dezalgo": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", + "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", + "dev": true, + "dependencies": { + "asap": "^2.0.0", + "wrappy": "1" + } + }, "node_modules/diagnostic-channel": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/diagnostic-channel/-/diagnostic-channel-0.2.0.tgz", @@ -4345,6 +4395,19 @@ "node": ">=8" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/duplexer3": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", @@ -4424,12 +4487,9 @@ } }, "node_modules/es-define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", - "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", - "dependencies": { - "get-intrinsic": "^1.2.4" - }, + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", "engines": { "node": ">= 0.4" } @@ -4442,6 +4502,32 @@ "node": ">= 0.4" } }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dev": true, + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/es6-error": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", @@ -4965,14 +5051,17 @@ } }, "node_modules/express-rate-limit": { - "version": "6.0.4", - "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-6.0.4.tgz", - "integrity": "sha512-TratTfxxTAFb6ZUAxPIigqhcS0e7ql9XDTorjD+SihV5ua5h6agoKyr45iKM6m5OzTppesh9o/RCuvf5eTiwCw==", + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-7.5.0.tgz", + "integrity": "sha512-eB5zbQh5h+VenMPM3fh+nw1YExi5nMr6HUCR62ELSP11huvxm/Uir1H1QEyTkk5QX6A58pX6NmaTMceKZ0Eodg==", "engines": { - "node": ">= 14.5.0" + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" }, "peerDependencies": { - "express": "^4" + "express": "^4.11 || 5 || ^5.0.0-beta.1" } }, "node_modules/express-routes-versioning": { @@ -5172,6 +5261,12 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", + "dev": true + }, "node_modules/fast-xml-parser": { "version": "4.0.11", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.11.tgz", @@ -5551,6 +5646,14 @@ "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", "integrity": "sha1-8/dSL073gjSNqBYbrZ7P1Rv4OnU=" }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "engines": { + "node": ">= 4" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -5579,15 +5682,20 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", - "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.7.tgz", + "integrity": "sha512-VW6Pxhsrk0KAOqs3WEd0klDiF/+V7gQOpAvY1jVU/LHmaD/kQO4523aiJuikX/QAKYiW6x8Jh+RJej1almdtCA==", "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-define-property": "^1.0.1", "es-errors": "^1.3.0", + "es-object-atoms": "^1.0.0", "function-bind": "^1.1.2", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3", - "hasown": "^2.0.0" + "get-proto": "^1.0.0", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -5617,6 +5725,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-stream": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", @@ -5742,11 +5862,11 @@ } }, "node_modules/gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "dependencies": { - "get-intrinsic": "^1.1.3" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -5900,21 +6020,10 @@ "node": ">=4" } }, - "node_modules/has-property-descriptors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "dependencies": { - "es-define-property": "^1.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-proto": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", - "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", "engines": { "node": ">= 0.4" }, @@ -5922,10 +6031,14 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.3" + }, "engines": { "node": ">= 0.4" }, @@ -6053,6 +6166,15 @@ "node": ">= 0.8" } }, + "node_modules/hexoid": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-2.0.0.tgz", + "integrity": "sha512-qlspKUK7IlSQv2o+5I7yhUd7TxlOG2Vr5LTa3ve2XSNVKAL/n/u/7KLvKmFNimomDIKvZFXWHv0T12mv7rT8Aw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/hide-powered-by": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/hide-powered-by/-/hide-powered-by-1.1.0.tgz", @@ -7181,6 +7303,14 @@ "semver": "bin/semver.js" } }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/md5-file": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/md5-file/-/md5-file-5.0.0.tgz", @@ -8444,9 +8574,12 @@ } }, "node_modules/object-inspect": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", - "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==", + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -9245,30 +9378,14 @@ } }, "node_modules/rate-limit-redis": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/rate-limit-redis/-/rate-limit-redis-2.1.0.tgz", - "integrity": "sha512-6SAsTCzY0v6UCIKLOLLYqR2XzFmgdtF7jWXlSPq2FrNIZk8tZ7xwBvyGW7GFMCe5I4S9lYNdrSJ9E84rz3/CpA==", - "dependencies": { - "defaults": "^1.0.3", - "redis": "^3.0.2" - } - }, - "node_modules/rate-limit-redis/node_modules/redis": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/redis/-/redis-3.1.2.tgz", - "integrity": "sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw==", - "dependencies": { - "denque": "^1.5.0", - "redis-commands": "^1.7.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0" - }, + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/rate-limit-redis/-/rate-limit-redis-4.2.0.tgz", + "integrity": "sha512-wV450NQyKC24NmPosJb2131RoczLdfIJdKCReNwtVpm5998U8SgKrAZrIHaN/NfQgqOHaan8Uq++B4sa5REwjA==", "engines": { - "node": ">=10" + "node": ">= 16" }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-redis" + "peerDependencies": { + "express-rate-limit": ">= 6" } }, "node_modules/raw-body": { @@ -9362,45 +9479,16 @@ } }, "node_modules/redis": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/redis/-/redis-3.1.1.tgz", - "integrity": "sha512-QhkKhOuzhogR1NDJfBD34TQJz2ZJwDhhIC6ZmvpftlmfYShHHQXjjNspAJ+Z2HH5NwSBVYBVganbiZ8bgFMHjg==", + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.7.0.tgz", + "integrity": "sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==", "dependencies": { - "denque": "^1.5.0", - "redis-commands": "^1.7.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-redis" - } - }, - "node_modules/redis-commands": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", - "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" - }, - "node_modules/redis-errors": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", - "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=", - "engines": { - "node": ">=4" - } - }, - "node_modules/redis-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", - "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", - "dependencies": { - "redis-errors": "^1.0.0" - }, - "engines": { - "node": ">=4" + "@redis/bloom": "1.2.0", + "@redis/client": "1.6.0", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.7", + "@redis/search": "1.2.0", + "@redis/time-series": "1.1.0" } }, "node_modules/referrer-policy": { @@ -9838,22 +9926,6 @@ "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "dev": true }, - "node_modules/set-function-length": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", - "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", - "dependencies": { - "define-data-property": "^1.1.4", - "es-errors": "^1.3.0", - "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.4", - "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/set-immediate-shim": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", @@ -9892,14 +9964,65 @@ "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" }, "node_modules/side-channel": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", - "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", "dependencies": { - "call-bind": "^1.0.7", + "call-bound": "^1.0.2", "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.4", - "object-inspect": "^1.13.1" + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" }, "engines": { "node": ">= 0.4" @@ -10356,6 +10479,118 @@ "safe-buffer": "~5.2.0" } }, + "node_modules/supertest": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/supertest/-/supertest-7.0.0.tgz", + "integrity": "sha512-qlsr7fIC0lSddmA3tzojvzubYxvlGtzumcdHgPwbFWMISQwL22MhM2Y3LNt+6w9Yyx7559VW5ab70dgphm8qQA==", + "dev": true, + "dependencies": { + "methods": "^1.1.2", + "superagent": "^9.0.1" + }, + "engines": { + "node": ">=14.18.0" + } + }, + "node_modules/supertest/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "dev": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/supertest/node_modules/form-data": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz", + "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/supertest/node_modules/formidable": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.2.tgz", + "integrity": "sha512-Jqc1btCy3QzRbJaICGwKcBfGWuLADRerLzDqi2NwSt/UkXLsHJw2TVResiaoBufHVHy9aSgClOHCeJsSsFLTbg==", + "dev": true, + "dependencies": { + "dezalgo": "^1.0.4", + "hexoid": "^2.0.0", + "once": "^1.4.0" + }, + "funding": { + "url": "https://ko-fi.com/tunnckoCore/commissions" + } + }, + "node_modules/supertest/node_modules/mime": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.6.0.tgz", + "integrity": "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==", + "dev": true, + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/supertest/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/supertest/node_modules/qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "dev": true, + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/supertest/node_modules/superagent": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/superagent/-/superagent-9.0.2.tgz", + "integrity": "sha512-xuW7dzkUpcJq7QnhOsnNUgtYp3xRwpt2F7abdRYIpCsAt0hhUqia0EdxyXZQQpNmGtsCzYHryaKSV3q3GJnq7w==", + "dev": true, + "dependencies": { + "component-emitter": "^1.3.0", + "cookiejar": "^2.1.4", + "debug": "^4.3.4", + "fast-safe-stringify": "^2.1.1", + "form-data": "^4.0.0", + "formidable": "^3.5.1", + "methods": "^1.1.2", + "mime": "2.6.0", + "qs": "^6.11.0" + }, + "engines": { + "node": ">=14.18.0" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", diff --git a/package.json b/package.json index 33de7486d..236305297 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "debug": "~2.6.9", "express": "^4.19.2", "express-init": "^1.1.1", - "express-rate-limit": "^6.0.4", + "express-rate-limit": "^7.5.0", "express-routes-versioning": "^1.0.1", "extend": "^3.0.2", "geit": "0.0.7", @@ -58,10 +58,10 @@ "passport-github": "^1.1.0", "patch-package": "^6.5.1", "promise-retry": "^1.1.1", - "rate-limit-redis": "^2.1.0", + "rate-limit-redis": "^4.2.0", "readdirp": "^2.1.0", "recursive-readdir": "^2.2.1", - "redis": "^3.1.1", + "redis": "^4.7.0", "request-id": "^0.11.1", "semver": "7.6.0", "serialize-error": "^2.1.0", @@ -93,6 +93,7 @@ "prettier": "3.2.5", "proxyquire": "^2.0.1", "sinon": "^5.0.0", + "supertest": "7.0.0", "typescript": "5.0.4" } } diff --git a/test/app.js b/test/app.js index 1dcff89c1..55a3aa821 100644 --- a/test/app.js +++ b/test/app.js @@ -20,8 +20,19 @@ const config = proxyquire('../bin/config', { }) describe('Application', () => { + const middleware = async (req, res, next) => {} + let resources = {} + + beforeEach(async () => { + resources = { + rateLimiter: { middleware }, + batchRateLimiter: { middleware }, + logger: console + } + }) + it('should initialize', done => { - const app = Application(config) + const app = Application(config, resources) init(app, error => { if (error) { done(error) diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js new file mode 100644 index 000000000..e3a92897d --- /dev/null +++ b/test/lib/rateLimit.js @@ -0,0 +1,196 @@ +// (c) Copyright 2025, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const assert = require('assert') +const { + RateLimiter, + RedisBackedRateLimiter, + createApiLimiter, + createBatchApiLimiter +} = require('../../lib/rateLimit.js') +const supertest = require('supertest') +const express = require('express') + +const logger = { + info: () => {}, + error: () => {}, + debug: () => {} +} + +const limit = { windowMs: 1000, max: 1 } + +describe('Rate Limiter', () => { + describe('Rate Limiter Tests', () => { + let client, rateLimiter + + beforeEach(async () => { + rateLimiter = new RateLimiter({ limit, logger }) + const app = await buildApp(rateLimiter) + client = supertest(app) + }) + + it('allows requests under the limit', async () => { + await client.get('/').expect(200).expect('Hello World!').expect('RateLimit-Limit', '1') + }) + + it('blocks requests over the limit', async () => { + const counter = await tryBeyondLimit(limit.max, client) + assert.ok(counter === limit.max, `Counter is ${counter}`) + }) + }) + + describe.skip('Redis Based Tests', () => { + const apiKey = process.env['CACHING_REDIS_API_KEY'] + const service = process.env['CACHING_REDIS_SERVICE'] + const redis = { apiKey, service } + + describe('Redis Client Test', () => { + let redisClient + + beforeEach(async () => { + redisClient = RedisBackedRateLimiter.buildRedisClient(redis) + await redisClient.connect() + }) + + afterEach(async () => { + await redisClient.disconnect() + }) + + it('should be empty initially', async () => { + const value = await redisClient.get('foo') + assert.ok(value === null) + }) + + it('sets, gets and removes a value', async () => { + await redisClient.set('foo', 'bar') + let value = await redisClient.get('foo') + assert.ok(value === 'bar') + //clear the value + await redisClient.del('foo') + value = await redisClient.get('foo') + assert.ok(value === null) + }) + }) + + describe('Redis Based Rate Limiter Tests', () => { + let client, rateLimiter + + beforeEach(async () => { + rateLimiter = new RedisBackedRateLimiter({ limit, redis, logger }) + const app = await buildApp(rateLimiter) + client = supertest(app) + }) + + afterEach(async () => { + await rateLimiter.done() + await new Promise(resolve => setTimeout(resolve, limit.windowMs)) + }) + + it('allows requests under the limit', async () => { + await client.get('/').expect(200).expect('Hello World!').expect('RateLimit-Limit', '1') + }) + + it('blocks requests over the limit', async () => { + const counter = await tryBeyondLimit(limit.max, client) + assert.ok(counter === limit.max, `Counter is ${counter}`) + }) + }) + }) + + describe('Create Rate Limiter', () => { + const limits = { + windowSeconds: 1, + max: 0, + batchWindowSeconds: 10, + batchMax: 10 + } + const caching = { + caching_redis_service: 'host', + caching_redis_api_key: 'key' + } + + it('builds a rate limiter', () => { + const rateLimiter = createApiLimiter({ limits }) + assert.ok(rateLimiter instanceof RateLimiter) + const expected = { + limit: { + windowMs: 1000, + max: 0 + }, + redis: undefined, + logger: undefined + } + assert.deepStrictEqual(rateLimiter.options, expected) + }) + + it('builds a batch rate limiter', () => { + const batchRateLimiter = createBatchApiLimiter({ limits }) + assert.ok(batchRateLimiter instanceof RateLimiter) + const expected = { + limit: { + windowMs: 10000, + max: 10 + }, + redis: undefined, + logger: undefined + } + assert.deepStrictEqual(batchRateLimiter.options, expected) + }) + + it('builds a redis based rate limiter', () => { + const rateLimiter = createApiLimiter({ limits, caching }) + assert.ok(rateLimiter instanceof RedisBackedRateLimiter) + const expected = { + limit: { + windowMs: 1000, + max: 0 + }, + redis: { + service: 'host', + apiKey: 'key', + prefix: 'api' + }, + logger: undefined + } + assert.deepStrictEqual(rateLimiter.options, expected) + }) + + it('builds a redis based batch rate limiter', () => { + const batchRateLimiter = createBatchApiLimiter({ limits, caching }) + assert.ok(batchRateLimiter instanceof RedisBackedRateLimiter) + const expected = { + limit: { + windowMs: 10000, + max: 10 + }, + redis: { + service: 'host', + apiKey: 'key', + prefix: 'batch-api' + }, + logger: undefined + } + assert.deepStrictEqual(batchRateLimiter.options, expected) + }) + }) +}) + +async function tryBeyondLimit(max, client) { + let counter = 0 + while (counter < max + 10) { + const response = await client.get('/') + if (!response.ok) { + break + } + counter++ + } + return counter +} + +async function buildApp(rateLimiter) { + const app = express() + await rateLimiter.initialize() + app.use(rateLimiter.middleware) + app.get('/', (req, res) => res.send('Hello World!')) + return app +} From 31f4f69ea5d1c7f433f7b1d2ebd469f924ce722a Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Fri, 21 Feb 2025 18:54:43 -0800 Subject: [PATCH 02/22] Update Redis Cache --- lib/rateLimit.js | 21 +--- providers/caching/redis.js | 49 +++++++--- test/lib/rateLimit.js | 64 ++++-------- test/providers/caching/redis.js | 166 +++++++++++++++++++++++++------- 4 files changed, 186 insertions(+), 114 deletions(-) diff --git a/lib/rateLimit.js b/lib/rateLimit.js index 5c8edfae5..9e5753fe0 100644 --- a/lib/rateLimit.js +++ b/lib/rateLimit.js @@ -4,6 +4,7 @@ const { createClient } = require('redis') const { RedisStore } = require('rate-limit-redis') const { rateLimit } = require('express-rate-limit') +const { RedisCache } = require('../providers/caching/redis') class RateLimiter { constructor(opts) { @@ -58,12 +59,7 @@ class RedisBackedRateLimiter extends RateLimiter { async _initializeClient() { if (!this.options.redis) throw new Error('Redis configuration is missing') - const client = RedisBackedRateLimiter.buildRedisClient(this.options.redis) - client.on('error', error => { - this.logger.error(`Redis client error: ${error}`) - }) - await client.connect().then(() => this.logger.info('Done connecting to redis: %s', this.options.redis.service)) - return client + return RedisCache.initializeClient(this.options.redis, this.logger) } static buildRedisStore(client, { prefix }) { @@ -72,19 +68,6 @@ class RedisBackedRateLimiter extends RateLimiter { sendCommand: (...args) => client.sendCommand(args) }) } - - static buildRedisClient({ apiKey, service }) { - return createClient({ - username: 'default', - password: apiKey, - socket: { - host: service, - port: 6380, - tls: true - }, - pingInterval: 5 * 60 * 1000 // https://learn.microsoft.com/en-us/azure/azure-cache-for-redis/cache-best-practices-connection#idle-timeout - }) - } } function createRateLimiter(config) { diff --git a/providers/caching/redis.js b/providers/caching/redis.js index 574f0d0dc..1fca4ec43 100644 --- a/providers/caching/redis.js +++ b/providers/caching/redis.js @@ -1,8 +1,7 @@ // Copyright (c) Amazon.com, Inc. or its affiliates and others. Licensed under the MIT license. // SPDX-License-Identifier: MIT -const redis = require('redis') -const util = require('util') +const { createClient } = require('redis') const pako = require('pako') const objectPrefix = '*!~%' @@ -10,20 +9,19 @@ const objectPrefix = '*!~%' class RedisCache { constructor(options) { this.options = options + this.logger = options.logger } - initialize() { - this.redis = redis.createClient(6380, this.options.service, { - auth_pass: this.options.apiKey, - tls: { servername: this.options.service } - }) - this._redisGet = util.promisify(this.redis.get).bind(this.redis) - this._redisSet = util.promisify(this.redis.set).bind(this.redis) - this._redisDel = util.promisify(this.redis.del).bind(this.redis) + async initialize() { + this._client = await RedisCache.initializeClient(this.options, this.logger) + } + + async done() { + return this._client?.disconnect() } async get(item) { - const cacheItem = await this._redisGet(item) + const cacheItem = await this._client.get(item) if (!cacheItem) return null const result = pako.inflate(cacheItem, { to: 'string' }) if (!result.startsWith(objectPrefix)) return result @@ -37,13 +35,36 @@ class RedisCache { async set(item, value, ttlSeconds) { if (typeof value !== 'string') value = objectPrefix + JSON.stringify(value) const data = pako.deflate(value, { to: 'string' }) - if (ttlSeconds) await this._redisSet(item, data, 'EX', ttlSeconds) - else await this._redisSet(item, data) + if (ttlSeconds) await this._client.set(item, data, { EX: ttlSeconds }) + else await this._client.set(item, data) } async delete(item) { - await this._redisDel(item) + await this._client.del(item) + } + + static buildRedisClient({ apiKey, service }) { + return createClient({ + username: 'default', + password: apiKey, + socket: { + host: service, + port: 6380, + tls: true + }, + pingInterval: 5 * 60 * 1000 // https://learn.microsoft.com/en-us/azure/azure-cache-for-redis/cache-best-practices-connection#idle-timeout + }) + } + + static async initializeClient(options, logger) { + const client = this.buildRedisClient(options) + client.on('error', error => { + logger.error(`Redis client error: ${error}`) + }) + await client.connect().then(() => logger.info('Done connecting to redis: %s', options.service)) + return client } } module.exports = options => new RedisCache(options) +module.exports.RedisCache = RedisCache diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js index e3a92897d..efc6c16a2 100644 --- a/test/lib/rateLimit.js +++ b/test/lib/rateLimit.js @@ -39,61 +39,31 @@ describe('Rate Limiter', () => { }) }) - describe.skip('Redis Based Tests', () => { + describe.skip('Redis Based Rate Limiter Integration Tests', () => { const apiKey = process.env['CACHING_REDIS_API_KEY'] const service = process.env['CACHING_REDIS_SERVICE'] const redis = { apiKey, service } - describe('Redis Client Test', () => { - let redisClient - - beforeEach(async () => { - redisClient = RedisBackedRateLimiter.buildRedisClient(redis) - await redisClient.connect() - }) - - afterEach(async () => { - await redisClient.disconnect() - }) - - it('should be empty initially', async () => { - const value = await redisClient.get('foo') - assert.ok(value === null) - }) - - it('sets, gets and removes a value', async () => { - await redisClient.set('foo', 'bar') - let value = await redisClient.get('foo') - assert.ok(value === 'bar') - //clear the value - await redisClient.del('foo') - value = await redisClient.get('foo') - assert.ok(value === null) - }) - }) - - describe('Redis Based Rate Limiter Tests', () => { - let client, rateLimiter + let client, rateLimiter - beforeEach(async () => { - rateLimiter = new RedisBackedRateLimiter({ limit, redis, logger }) - const app = await buildApp(rateLimiter) - client = supertest(app) - }) + beforeEach(async () => { + rateLimiter = new RedisBackedRateLimiter({ limit, redis, logger }) + const app = await buildApp(rateLimiter) + client = supertest(app) + }) - afterEach(async () => { - await rateLimiter.done() - await new Promise(resolve => setTimeout(resolve, limit.windowMs)) - }) + afterEach(async () => { + await rateLimiter.done() + await new Promise(resolve => setTimeout(resolve, limit.windowMs)) + }) - it('allows requests under the limit', async () => { - await client.get('/').expect(200).expect('Hello World!').expect('RateLimit-Limit', '1') - }) + it('allows requests under the limit', async () => { + await client.get('/').expect(200).expect('Hello World!').expect('RateLimit-Limit', '1') + }) - it('blocks requests over the limit', async () => { - const counter = await tryBeyondLimit(limit.max, client) - assert.ok(counter === limit.max, `Counter is ${counter}`) - }) + it('blocks requests over the limit', async () => { + const counter = await tryBeyondLimit(limit.max, client) + assert.ok(counter === limit.max, `Counter is ${counter}`) }) }) diff --git a/test/providers/caching/redis.js b/test/providers/caching/redis.js index ab23c1e63..ddef38cbe 100644 --- a/test/providers/caching/redis.js +++ b/test/providers/caching/redis.js @@ -3,51 +3,149 @@ const sinon = require('sinon') const sandbox = sinon.createSandbox() -const redis = require('redis') const assert = require('assert') const redisCache = require('../../../providers/caching/redis') +const { RedisCache } = require('../../../providers/caching/redis') -describe('get a tool result', () => { - const store = {} - beforeEach(function () { - sandbox.stub(redis, 'createClient').callsFake(() => { - return { - get: (key, callback) => callback(null, store[key]), - set: (key, value, arg, expire, callback) => { +const logger = { + info: () => {}, + error: () => {}, + debug: () => {} +} + +describe('Redis Cache', () => { + describe('get a tool result', () => { + const store = {} + beforeEach(function () { + sandbox.stub(RedisCache, 'initializeClient').resolves({ + get: async key => Promise.resolve(store[key]), + set: async (key, value) => { store[key] = value - callback ? callback(null) : arg(null) }, - del: (key, callback) => { + del: async key => { store[key] = null - callback(null) } - } + }) }) - }) - afterEach(function () { - sandbox.restore() - }) + afterEach(function () { + sandbox.restore() + }) - it('works well for a specific tool version', async () => { - const cache = redisCache({}) - await cache.initialize() - await cache.set('foo', 'bar') - const result = await cache.get('foo') - assert.equal(result, 'bar') - }) - it('works well for an object', async () => { - const cache = redisCache({}) - await cache.initialize() - await cache.set('foo', { temp: 3 }) - const result = await cache.get('foo') - assert.equal(result.temp, 3) + it('works well for a specific tool version', async () => { + const cache = redisCache({ logger }) + await cache.initialize() + await cache.set('foo', 'bar') + const result = await cache.get('foo') + assert.equal(result, 'bar') + }) + + it('works well for an object', async () => { + const cache = redisCache({ logger }) + await cache.initialize() + await cache.set('foo', { temp: 3 }) + const result = await cache.get('foo') + assert.equal(result.temp, 3) + }) + + it('returns null for missing entry', async () => { + const cache = redisCache({ logger }) + await cache.initialize() + const result = await cache.get('bar') + assert.equal(result, null) + }) + + it('deletes a key', async () => { + const cache = redisCache({ logger }) + await cache.initialize() + await cache.set('foo', 'bar') + await cache.delete('foo') + const result = await cache.get('foo') + assert.ok(result === null) + }) }) - it('returns null for missing entry', async () => { - const cache = redisCache({}) - await cache.initialize() - const result = await cache.get('bar') - assert.equal(result, null) + describe('Integration Test', () => { + const apiKey = process.env['CACHING_REDIS_API_KEY'] + const service = process.env['CACHING_REDIS_SERVICE'] + + describe('Redis Client Test', () => { + let client + beforeEach(async () => { + client = await RedisCache.initializeClient({ apiKey, service }, logger) + }) + + afterEach(async () => { + await client.disconnect() + }) + + it('retrieves empty initially', async () => { + const value = await client.get('boo') + assert.ok(value === null) + }) + + it('sets, gets and removes a value', async () => { + await client.set('foo', 'bar') + let value = await client.get('foo') + assert.ok(value === 'bar') + //clear the value + await client.del('foo') + value = await client.get('foo') + assert.ok(value === null) + }) + + it('sets value and exipres', async () => { + let value = await client.get('tee') + assert.ok(value === null) + + await client.set('tee', 'value', { EX: 1 }) + value = await client.get('tee') + assert.ok(value === 'value') + + await new Promise(resolve => setTimeout(resolve, 1200)) + value = await client.get('tee') + assert.ok(value === null) + }).timeout(3000) + }) + + describe('Redis Cache Test', () => { + let cache + beforeEach(async () => { + cache = redisCache({ apiKey, service, logger }) + await cache.initialize() + }) + + afterEach(async () => { + await cache.done() + }) + + it('should be empty initially', async () => { + const value = await cache.get('boo') + assert.ok(value === null) + }) + + it('sets, gets and removes a value', async () => { + await cache.set('foo', 'bar') + let value = await cache.get('foo') + assert.ok(value === 'bar') + //clear the value + await cache.delete('foo') + value = await cache.get('foo') + assert.ok(value === null) + }) + + it('sets value and exipres', async () => { + let value = await cache.get('wee') + assert.ok(value === null) + + await cache.set('wee', 'value', 1) + value = await cache.get('wee') + assert.ok(value === 'value') + + await new Promise(resolve => setTimeout(resolve, 1200)) + value = await cache.get('wee') + assert.ok(value === null) + }).timeout(3000) + }) }) }) From 55fa892a24debdb92690cc563dd6475dc8cc71b3 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Mon, 24 Feb 2025 10:32:31 -0800 Subject: [PATCH 03/22] Use quit() instead of disconnect() and throw if connecting to redis fails --- lib/rateLimit.js | 22 ++++----- providers/caching/redis.js | 19 +++++--- test/lib/rateLimit.js | 86 ++++++++++++++++++++++++--------- test/providers/caching/redis.js | 24 +++++++-- 4 files changed, 103 insertions(+), 48 deletions(-) diff --git a/lib/rateLimit.js b/lib/rateLimit.js index 9e5753fe0..e91ae97ae 100644 --- a/lib/rateLimit.js +++ b/lib/rateLimit.js @@ -21,6 +21,7 @@ class RateLimiter { } get middleware() { + if (!this._limiter) throw new Error('Rate limiter not initialized') return this._limiter } @@ -33,28 +34,26 @@ class RateLimiter { //TODO: use standardHeaders? const opts = { windowMs, - max, + limit: max, //limit is preferred over max standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers legacyHeaders: false // Disable the `X-RateLimit-*` headers } - if (store) { - opts.store = store - } + if (store) opts.store = store return rateLimit(opts) } } -class RedisBackedRateLimiter extends RateLimiter { +class RedisBasedRateLimiter extends RateLimiter { async initialize() { if (!this._limiter) { this._client = await this._initializeClient() - const store = RedisBackedRateLimiter.buildRedisStore(this._client, this.options.redis) + const store = RedisBasedRateLimiter.buildRedisStore(this._client, this.options.redis) await super.initialize(store) } } async done() { - return this._client?.disconnect().then(() => super.done()) + return this._client?.quit().then(() => super.done()) } async _initializeClient() { @@ -71,14 +70,11 @@ class RedisBackedRateLimiter extends RateLimiter { } function createRateLimiter(config) { - return config.redis ? new RedisBackedRateLimiter(config) : new RateLimiter(config) + return config.redis ? new RedisBasedRateLimiter(config) : new RateLimiter(config) } function buildOpts({ windowSeconds, max }, logger, { caching_redis_service, caching_redis_api_key } = {}, prefix) { - const limit = { - windowMs: windowSeconds * 1000, - max - } + const limit = { windowMs: windowSeconds * 1000, max } let redis if (caching_redis_service) { redis = { @@ -105,6 +101,6 @@ function createBatchApiLimiter(config, logger) { module.exports = { createApiLimiter, createBatchApiLimiter, - RedisBackedRateLimiter, + RedisBasedRateLimiter, RateLimiter } diff --git a/providers/caching/redis.js b/providers/caching/redis.js index 1fca4ec43..e10c7de7f 100644 --- a/providers/caching/redis.js +++ b/providers/caching/redis.js @@ -17,7 +17,7 @@ class RedisCache { } async done() { - return this._client?.disconnect() + return this._client?.quit() } async get(item) { @@ -28,6 +28,7 @@ class RedisCache { try { return JSON.parse(result.substring(4)) } catch (error) { + this.logger.error('Error parsing cached item: %s', error) return null } } @@ -58,11 +59,17 @@ class RedisCache { static async initializeClient(options, logger) { const client = this.buildRedisClient(options) - client.on('error', error => { - logger.error(`Redis client error: ${error}`) - }) - await client.connect().then(() => logger.info('Done connecting to redis: %s', options.service)) - return client + try { + await client.connect() + logger.info('Done connecting to redis: %s', options.service) + client.on('error', error => { + logger.error(`Redis client error: ${error}`) + }) + return client + } catch (error) { + logger.error('Error connecting to redis: %s', error) + throw error + } } } diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js index efc6c16a2..53157a394 100644 --- a/test/lib/rateLimit.js +++ b/test/lib/rateLimit.js @@ -4,12 +4,14 @@ const assert = require('assert') const { RateLimiter, - RedisBackedRateLimiter, + RedisBasedRateLimiter, createApiLimiter, createBatchApiLimiter } = require('../../lib/rateLimit.js') const supertest = require('supertest') const express = require('express') +const sandbox = require('sinon').createSandbox() +const { RedisCache } = require('../../providers/caching/redis') const logger = { info: () => {}, @@ -30,40 +32,76 @@ describe('Rate Limiter', () => { }) it('allows requests under the limit', async () => { - await client.get('/').expect(200).expect('Hello World!').expect('RateLimit-Limit', '1') + await client.get('/').expect(200).expect('Hello World!') + .expect('RateLimit-Limit', '1') + .expect('RateLimit-Remaining', '0') }) it('blocks requests over the limit', async () => { const counter = await tryBeyondLimit(limit.max, client) - assert.ok(counter === limit.max, `Counter is ${counter}`) + assert.strictEqual(counter, limit.max, `Counter is ${counter}`) }) }) - describe.skip('Redis Based Rate Limiter Integration Tests', () => { + describe('Redis Based Rate Limiter', () => { const apiKey = process.env['CACHING_REDIS_API_KEY'] const service = process.env['CACHING_REDIS_SERVICE'] const redis = { apiKey, service } - let client, rateLimiter - - beforeEach(async () => { - rateLimiter = new RedisBackedRateLimiter({ limit, redis, logger }) - const app = await buildApp(rateLimiter) - client = supertest(app) - }) - - afterEach(async () => { - await rateLimiter.done() - await new Promise(resolve => setTimeout(resolve, limit.windowMs)) - }) - - it('allows requests under the limit', async () => { - await client.get('/').expect(200).expect('Hello World!').expect('RateLimit-Limit', '1') + describe('Handling errors', () => { + let rateLimiter + + afterEach(async () => { + await rateLimiter.done() + sandbox.restore() + }) + + it('throws error if redis configuration is missing', async () => { + try { + rateLimiter = new RedisBasedRateLimiter({ limit, logger }) + await rateLimiter.initialize() + } catch (error) { + assert.ok(error.message === 'Redis configuration is missing') + } + }) + + it('throws error if connecting to redis fails', async () => { + sandbox.stub(RedisCache, 'buildRedisClient').returns({ + connect: () => Promise.reject(new Error('Connection failed')) + }) + try { + rateLimiter = new RedisBasedRateLimiter({ limit, redis, logger }) + await rateLimiter.initialize() + } catch (error) { + assert.equal(error.message, 'Connection failed') + } + }) }) - it('blocks requests over the limit', async () => { - const counter = await tryBeyondLimit(limit.max, client) - assert.ok(counter === limit.max, `Counter is ${counter}`) + describe('Rate Limit Integration Tests', () => { + let client, rateLimiter + + beforeEach(async () => { + rateLimiter = new RedisBasedRateLimiter({ limit, redis, logger }) + const app = await buildApp(rateLimiter) + client = supertest(app) + }) + + afterEach(async () => { + await rateLimiter.done() + await new Promise(resolve => setTimeout(resolve, limit.windowMs)) + }) + + it('allows requests under the limit', async () => { + await client.get('/').expect(200).expect('Hello World!') + .expect('RateLimit-Limit', '1') + .expect('RateLimit-Remaining', '0') + }) + + it('blocks requests over the limit', async () => { + const counter = await tryBeyondLimit(limit.max, client) + assert.strictEqual(counter, limit.max, `Counter is ${counter}`) + }) }) }) @@ -109,7 +147,7 @@ describe('Rate Limiter', () => { it('builds a redis based rate limiter', () => { const rateLimiter = createApiLimiter({ limits, caching }) - assert.ok(rateLimiter instanceof RedisBackedRateLimiter) + assert.ok(rateLimiter instanceof RedisBasedRateLimiter) const expected = { limit: { windowMs: 1000, @@ -127,7 +165,7 @@ describe('Rate Limiter', () => { it('builds a redis based batch rate limiter', () => { const batchRateLimiter = createBatchApiLimiter({ limits, caching }) - assert.ok(batchRateLimiter instanceof RedisBackedRateLimiter) + assert.ok(batchRateLimiter instanceof RedisBasedRateLimiter) const expected = { limit: { windowMs: 10000, diff --git a/test/providers/caching/redis.js b/test/providers/caching/redis.js index ddef38cbe..3f2a5bb6c 100644 --- a/test/providers/caching/redis.js +++ b/test/providers/caching/redis.js @@ -16,16 +16,20 @@ const logger = { describe('Redis Cache', () => { describe('get a tool result', () => { const store = {} + let mockClient beforeEach(function () { - sandbox.stub(RedisCache, 'initializeClient').resolves({ + mockClient = { get: async key => Promise.resolve(store[key]), set: async (key, value) => { store[key] = value }, del: async key => { store[key] = null - } - }) + }, + connect: async () => Promise.resolve(mockClient), + on: () => {} + } + sandbox.stub(RedisCache, 'buildRedisClient').returns(mockClient) }) afterEach(function () { @@ -63,6 +67,16 @@ describe('Redis Cache', () => { const result = await cache.get('foo') assert.ok(result === null) }) + + it('throws error if redis connection fails', async () => { + mockClient.connect = () => Promise.reject(new Error('Connection failed')) + const cache = redisCache({ logger }) + try { + await cache.initialize() + } catch (error) { + assert.equal(error.message, 'Connection failed') + } + }) }) describe('Integration Test', () => { @@ -76,7 +90,7 @@ describe('Redis Cache', () => { }) afterEach(async () => { - await client.disconnect() + await client.quit() }) it('retrieves empty initially', async () => { @@ -119,7 +133,7 @@ describe('Redis Cache', () => { await cache.done() }) - it('should be empty initially', async () => { + it('retrieves empty initially', async () => { const value = await cache.get('boo') assert.ok(value === null) }) From a46fbf957d5dc76bbec9defe8f35602f1ad529ab Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Mon, 24 Feb 2025 11:59:41 -0800 Subject: [PATCH 04/22] Handle breaking change in express-rate-limit 7.0.0 --- lib/rateLimit.js | 15 ++++++++++----- test/lib/rateLimit.js | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/lib/rateLimit.js b/lib/rateLimit.js index e91ae97ae..ac102d0e4 100644 --- a/lib/rateLimit.js +++ b/lib/rateLimit.js @@ -15,7 +15,8 @@ class RateLimiter { async initialize(store) { if (!this._limiter) { this.logger.debug('Creating rate limiter: %o', this.options.limit) - this._limiter = RateLimiter.build(this.options.limit, store) + const options = RateLimiter.buildOptions(this.options.limit, store) + this._limiter = rateLimit(options) this.logger.info('Rate limiter initialized') } } @@ -30,16 +31,20 @@ class RateLimiter { this.logger.info('Rate limiter done') } - static build({ windowMs, max }, store) { + static buildOptions({ windowMs, max }, store) { //TODO: use standardHeaders? const opts = { - windowMs, - limit: max, //limit is preferred over max standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers legacyHeaders: false // Disable the `X-RateLimit-*` headers } + if (max === 0) { + opts.skip = () => true //See breaking changes in express-rate-limit v7.0.0 https://github.com/express-rate-limit/express-rate-limit/releases/tag/v7.0.0 + } else { + opts.limit = max //limit is preferred over max + opts.windowMs = windowMs + } if (store) opts.store = store - return rateLimit(opts) + return opts } } diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js index 53157a394..64d679c26 100644 --- a/test/lib/rateLimit.js +++ b/test/lib/rateLimit.js @@ -32,7 +32,10 @@ describe('Rate Limiter', () => { }) it('allows requests under the limit', async () => { - await client.get('/').expect(200).expect('Hello World!') + await client + .get('/') + .expect(200) + .expect('Hello World!') .expect('RateLimit-Limit', '1') .expect('RateLimit-Remaining', '0') }) @@ -41,6 +44,35 @@ describe('Rate Limiter', () => { const counter = await tryBeyondLimit(limit.max, client) assert.strictEqual(counter, limit.max, `Counter is ${counter}`) }) + + it('builds rate limit options', () => { + const options = RateLimiter.buildOptions(limit) + assert.deepStrictEqual(options, { + standardHeaders: true, + legacyHeaders: false, + limit: 1, + windowMs: 1000 + }) + }) + + it('builds rate limit options with store', async () => { + const store = {} + const options = RateLimiter.buildOptions(limit, store) + assert.deepStrictEqual(options, { + standardHeaders: true, + legacyHeaders: false, + limit: 1, + windowMs: 1000, + store: {} + }) + }) + + it('builds rate limit options when max === 0', async () => { + const options = RateLimiter.buildOptions({ windowSeconds: 1, max: 0 }) + assert.equal(options.standardHeaders, true) + assert.equal(options.legacyHeaders, false) + assert.equal(options.skip(), true) + }) }) describe('Redis Based Rate Limiter', () => { @@ -93,7 +125,10 @@ describe('Rate Limiter', () => { }) it('allows requests under the limit', async () => { - await client.get('/').expect(200).expect('Hello World!') + await client + .get('/') + .expect(200) + .expect('Hello World!') .expect('RateLimit-Limit', '1') .expect('RateLimit-Remaining', '0') }) From 69be41ff44f1eec7c9d0da87c7fe9682063c2373 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Mon, 24 Feb 2025 15:32:51 -0800 Subject: [PATCH 05/22] Use TestContainers to run redis based integration tests --- package-lock.json | 1535 ++++++++++++++++++++++++++++++- package.json | 3 +- providers/caching/redis.js | 6 +- test/lib/rateLimit.js | 26 +- test/providers/caching/redis.js | 30 +- 5 files changed, 1544 insertions(+), 56 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5681150fe..29835b0ad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -82,7 +82,8 @@ "prettier": "3.2.5", "proxyquire": "^2.0.1", "sinon": "^5.0.0", - "supertest": "7.0.0", + "supertest": "^7.0.0", + "testcontainers": "^10.18.0", "typescript": "5.0.4" } }, @@ -1622,6 +1623,12 @@ "node": ">=6.9.0" } }, + "node_modules/@balena/dockerignore": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==", + "dev": true + }, "node_modules/@clearlydefined/spdx": { "version": "0.1.9", "resolved": "git+ssh://git@github.com/clearlydefined/spdx.git#93916093259bc8593400948b679b0dc32a5a12dd", @@ -1781,6 +1788,15 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "dev": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@gitbeaker/core": { "version": "29.2.4", "resolved": "https://registry.npmjs.org/@gitbeaker/core/-/core-29.2.4.tgz", @@ -2233,6 +2249,102 @@ "integrity": "sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw==", "dev": true }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -2380,6 +2492,16 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@redis/bloom": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", @@ -2540,6 +2662,27 @@ "@types/node": "*" } }, + "node_modules/@types/docker-modem": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/docker-modem/-/docker-modem-3.0.6.tgz", + "integrity": "sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/ssh2": "*" + } + }, + "node_modules/@types/dockerode": { + "version": "3.3.35", + "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-3.3.35.tgz", + "integrity": "sha512-P+DCMASlsH+QaKkDpekKrP5pLls767PPs+/LrlVbKnEnY5tMpEUa2C6U4gRsdFZengOqxdCIqy16R22Q3pLB6Q==", + "dev": true, + "dependencies": { + "@types/docker-modem": "*", + "@types/node": "*", + "@types/ssh2": "*" + } + }, "node_modules/@types/express": { "version": "4.17.17", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.17.tgz", @@ -2623,6 +2766,24 @@ "@types/node": "*" } }, + "node_modules/@types/ssh2": { + "version": "1.15.4", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.4.tgz", + "integrity": "sha512-9JTQgVBWSgq6mAen6PVnrAmty1lqgCMvpfN+1Ck5WRUsyMYPa6qd50/vMJ0y1zkGpOEgLzm8m8Dx/Y5vRouLaA==", + "dev": true, + "dependencies": { + "@types/node": "^18.11.18" + } + }, + "node_modules/@types/ssh2-streams": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/@types/ssh2-streams/-/ssh2-streams-0.1.12.tgz", + "integrity": "sha512-Sy8tpEmCce4Tq0oSOYdfqaBpA3hDM8SoxoFh5vzFsu2oL+znzGz8oVWW7xb4K920yYMUY+PIG31qZnFMfPWNCg==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/tmp": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@types/tmp/-/tmp-0.2.3.tgz", @@ -2677,6 +2838,18 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dev": true, + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/abstract-leveldown": { "version": "2.6.3", "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-2.6.3.tgz", @@ -2899,6 +3072,284 @@ "diagnostic-channel-publishers": "^0.3.3" } }, + "node_modules/archiver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-7.0.1.tgz", + "integrity": "sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==", + "dev": true, + "dependencies": { + "archiver-utils": "^5.0.2", + "async": "^3.2.4", + "buffer-crc32": "^1.0.0", + "readable-stream": "^4.0.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^3.0.0", + "zip-stream": "^6.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/archiver-utils": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-5.0.2.tgz", + "integrity": "sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==", + "dev": true, + "dependencies": { + "glob": "^10.0.0", + "graceful-fs": "^4.2.0", + "is-stream": "^2.0.1", + "lazystream": "^1.0.0", + "lodash": "^4.17.15", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/archiver-utils/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/archiver-utils/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/archiver-utils/node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/archiver-utils/node_modules/glob": { + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "dev": true, + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/archiver-utils/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/archiver-utils/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/archiver-utils/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dev": true, + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/archiver-utils/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/archiver-utils/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/archiver-utils/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/archiver/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/archiver/node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "dev": true, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/archiver/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dev": true, + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/archiver/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/archiver/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/archiver/node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "dev": true, + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/archy": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz", @@ -2939,9 +3390,9 @@ "dev": true }, "node_modules/asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", "dependencies": { "safer-buffer": "~2.1.0" } @@ -2976,9 +3427,9 @@ } }, "node_modules/async": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/async/-/async-3.1.0.tgz", - "integrity": "sha512-4vx/aaY6j/j3Lw3fbCHNWP0pPaTCew3F6F3hYyl/tHs/ndmV1q7NW9T5yuJ2XAGwdQrP+6Wu20x06U4APo/iQQ==" + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==" }, "node_modules/async-hook-jl": { "version": "1.7.6", @@ -3011,6 +3462,12 @@ "semver": "bin/semver" } }, + "node_modules/async-lock": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.4.1.tgz", + "integrity": "sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==", + "dev": true + }, "node_modules/async-mutex": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.3.2.tgz", @@ -3091,6 +3548,12 @@ "node": ">= 0.8.26" } }, + "node_modules/b4a": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", + "dev": true + }, "node_modules/backo2": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz", @@ -3102,6 +3565,70 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, + "node_modules/bare-events": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.4.tgz", + "integrity": "sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==", + "dev": true, + "optional": true + }, + "node_modules/bare-fs": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.0.1.tgz", + "integrity": "sha512-ilQs4fm/l9eMfWY2dY0WCIUplSUp7U0CT1vrqMg1MUdeZl4fypu5UP0XcDBK5WBQPJAKP1b7XEodISmekH/CEg==", + "dev": true, + "optional": true, + "dependencies": { + "bare-events": "^2.0.0", + "bare-path": "^3.0.0", + "bare-stream": "^2.0.0" + }, + "engines": { + "bare": ">=1.7.0" + } + }, + "node_modules/bare-os": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.4.0.tgz", + "integrity": "sha512-9Ous7UlnKbe3fMi7Y+qh0DwAup6A1JkYgPnjvMDNOlmnxNRQvQ/7Nst+OnUQKzk0iAT0m9BisbDVp9gCv8+ETA==", + "dev": true, + "optional": true, + "engines": { + "bare": ">=1.6.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "dev": true, + "optional": true, + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.6.5", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.6.5.tgz", + "integrity": "sha512-jSmxKJNJmHySi6hC42zlZnq00rga4jjxcgNZjY9N5WlOe/iOoGRtdwGsHzQv2RlH2KOYMwGUXhf2zXd32BA9RA==", + "dev": true, + "optional": true, + "dependencies": { + "streamx": "^2.21.0" + }, + "peerDependencies": { + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, "node_modules/base-64": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", @@ -3463,6 +3990,25 @@ "node": "*" } }, + "node_modules/buildcheck": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", + "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "dev": true, + "optional": true, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/byline": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz", + "integrity": "sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/bytes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", @@ -3698,6 +4244,12 @@ "node": ">=8.10.0" } }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true + }, "node_modules/ci-info": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", @@ -3879,6 +4431,103 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/compress-commons": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-6.0.2.tgz", + "integrity": "sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==", + "dev": true, + "dependencies": { + "crc-32": "^1.2.0", + "crc32-stream": "^6.0.0", + "is-stream": "^2.0.1", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/compress-commons/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/compress-commons/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/compress-commons/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dev": true, + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/compress-commons/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/compress-commons/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -4033,24 +4682,133 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, - "node_modules/cors": { - "version": "2.8.4", - "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.4.tgz", - "integrity": "sha1-K9OB8usgECAQXNUOpZ2mMJBpRoY=", + "node_modules/cors": { + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.4.tgz", + "integrity": "sha1-K9OB8usgECAQXNUOpZ2mMJBpRoY=", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/cors-gate": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/cors-gate/-/cors-gate-1.1.3.tgz", + "integrity": "sha512-RFqvbbpj02lqKDhqasBEkgzmT3RseCH3DKy5sT2W9S1mhctABKQP3ktKcnKN0h8t4pJ2SneI3hPl3TGNi/VmZA==", + "dev": true + }, + "node_modules/cpu-features": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz", + "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "dependencies": { + "buildcheck": "~0.0.6", + "nan": "^2.19.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "dev": true, + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-6.0.0.tgz", + "integrity": "sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==", + "dev": true, + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/crc32-stream/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/crc32-stream/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dev": true, + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/crc32-stream/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/crc32-stream/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, "dependencies": { - "object-assign": "^4", - "vary": "^1" - }, - "engines": { - "node": ">=0.10.0" + "safe-buffer": "~5.2.0" } }, - "node_modules/cors-gate": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/cors-gate/-/cors-gate-1.1.3.tgz", - "integrity": "sha512-RFqvbbpj02lqKDhqasBEkgzmT3RseCH3DKy5sT2W9S1mhctABKQP3ktKcnKN0h8t4pJ2SneI3hPl3TGNi/VmZA==", - "dev": true - }, "node_modules/cross-fetch": { "version": "3.1.8", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", @@ -4061,9 +4819,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "dependencies": { "path-key": "^3.1.0", @@ -4363,6 +5121,137 @@ "node": ">=4.0.0" } }, + "node_modules/docker-compose": { + "version": "0.24.8", + "resolved": "https://registry.npmjs.org/docker-compose/-/docker-compose-0.24.8.tgz", + "integrity": "sha512-plizRs/Vf15H+GCVxq2EUvyPK7ei9b/cVesHvjnX4xaXjM9spHe2Ytq0BitndFgvTJ3E3NljPNUEl7BAN43iZw==", + "dev": true, + "dependencies": { + "yaml": "^2.2.2" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/docker-compose/node_modules/yaml": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.7.0.tgz", + "integrity": "sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA==", + "dev": true, + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/docker-modem": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-3.0.8.tgz", + "integrity": "sha512-f0ReSURdM3pcKPNS30mxOHSbaFLcknGmQjwSfmbcdOw1XWKXVhukM3NJHhr7NpY9BIyyWQb0EBo3KQvvuU5egQ==", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "readable-stream": "^3.5.0", + "split-ca": "^1.0.1", + "ssh2": "^1.11.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/docker-modem/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "dev": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/docker-modem/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/docker-modem/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/docker-modem/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/docker-modem/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/dockerode": { + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-3.3.5.tgz", + "integrity": "sha512-/0YNa3ZDNeLr/tSckmD69+Gq+qVNhvKfAHNeZJBnp7EOP6RGKV8ORrJHkUn20So5wU+xxT7+1n5u8PjHbfjbSA==", + "dev": true, + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "docker-modem": "^3.0.0", + "tar-fs": "~2.0.1" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/dockerode/node_modules/tar-fs": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz", + "integrity": "sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==", + "dev": true, + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.0.0" + } + }, "node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -4426,6 +5315,12 @@ "stream-shift": "^1.0.0" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true + }, "node_modules/ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -4982,12 +5877,30 @@ "node": ">= 0.6" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/eventemitter3": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz", "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==", "dev": true }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "dev": true, + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/expect-ct": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/expect-ct/-/expect-ct-0.2.0.tgz", @@ -5244,6 +6157,12 @@ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "dev": true + }, "node_modules/fast-json-patch": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/fast-json-patch/-/fast-json-patch-3.1.1.tgz", @@ -6906,6 +7825,21 @@ "integrity": "sha512-QZ9qOMdF+QLHxy1QIpUHUU1D5pS2CG2P69LF6L6CPjPYA/XMOmKV3PZpawHoAjHNyB0swdVTRxdYT4tbBbxqwg==", "dev": true }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -7056,6 +7990,18 @@ "node": ">=8" } }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "dev": true, + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, "node_modules/level-codec": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-7.0.1.tgz", @@ -7454,6 +8400,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "dev": true, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/mkdirp": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", @@ -7465,6 +8420,12 @@ "mkdirp": "bin/cmd.js" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true + }, "node_modules/mocha": { "version": "8.2.1", "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.2.1.tgz", @@ -8154,6 +9115,13 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "node_modules/nan": { + "version": "2.22.1", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.1.tgz", + "integrity": "sha512-pfRR4ZcNTSm2ZFHaztuvbICf+hyiG6ecA06SfAxoPmuHjvMu0KUIae7Y8GyVkbBqeEIidsmXeYooWIX9+qjfRQ==", + "dev": true, + "optional": true + }, "node_modules/nanoid": { "version": "3.1.12", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.12.tgz", @@ -8840,6 +9808,12 @@ "node": ">=8" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true + }, "node_modules/package-json/node_modules/semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", @@ -9069,6 +10043,28 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true + }, "node_modules/path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", @@ -9205,6 +10201,15 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "dev": true, + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/process-nextick-args": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", @@ -9234,6 +10239,54 @@ "node": ">=0.12" } }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proper-lockfile/node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/properties-reader": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/properties-reader/-/properties-reader-2.3.0.tgz", + "integrity": "sha512-z597WicA7nDZxK12kZqHr2TcvwNU1GCfA5UwfDY/HDp3hXPoPlb5rlEx9bwGTiJnc0OqbBTkU975jDToth8Gxw==", + "dev": true, + "dependencies": { + "mkdirp": "^1.0.4" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/steveukx/properties?sponsor=1" + } + }, + "node_modules/properties-reader/node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true, + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -9442,6 +10495,36 @@ "util-deprecate": "~1.0.1" } }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "dev": true, + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/readdirp": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.1.0.tgz", @@ -10261,19 +11344,63 @@ "spdx-ranges": "^2.0.0" } }, - "node_modules/split-on-first": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz", - "integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==", + "node_modules/split-ca": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", + "dev": true + }, + "node_modules/split-on-first": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz", + "integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==", + "engines": { + "node": ">=6" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" + }, + "node_modules/ssh-remote-port-forward": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/ssh-remote-port-forward/-/ssh-remote-port-forward-1.0.4.tgz", + "integrity": "sha512-x0LV1eVDwjf1gmG7TTnfqIzf+3VPRz7vrNIjX6oYLbeCrf/PeVY6hkT68Mg+q02qXxQhrLjB0jfgvhevoCRmLQ==", + "dev": true, + "dependencies": { + "@types/ssh2": "^0.5.48", + "ssh2": "^1.4.0" + } + }, + "node_modules/ssh-remote-port-forward/node_modules/@types/ssh2": { + "version": "0.5.52", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-0.5.52.tgz", + "integrity": "sha512-lbLLlXxdCZOSJMCInKH2+9V/77ET2J6NPQHpFI0kda61Dd1KglJs+fPQBchizmzYSOJBgdTajhPqBO1xxLywvg==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/ssh2-streams": "*" + } + }, + "node_modules/ssh2": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", + "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "asn1": "^0.2.6", + "bcrypt-pbkdf": "^1.0.2" + }, "engines": { - "node": ">=6" + "node": ">=10.16.0" + }, + "optionalDependencies": { + "cpu-features": "~0.0.10", + "nan": "^2.20.0" } }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" - }, "node_modules/sshpk": { "version": "1.15.2", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.15.2.tgz", @@ -10325,6 +11452,19 @@ "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", "dev": true }, + "node_modules/streamx": { + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.22.0.tgz", + "integrity": "sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw==", + "dev": true, + "dependencies": { + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, "node_modules/strict-uri-encode": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", @@ -10351,6 +11491,48 @@ "node": ">=4" } }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/string-width-cjs/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", @@ -10363,6 +11545,19 @@ "node": ">=4" } }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi/node_modules/ansi-regex": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz", @@ -10635,6 +11830,31 @@ "node": ">=0.10.0" } }, + "node_modules/tar-fs": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.8.tgz", + "integrity": "sha512-ZoROL70jptorGAlgAYiLoBLItEKw/fUxg9BSYK/dF/GAGYFJOJJJMvjPAKDJraCXFwadD456FCuvLWgfhMsPwg==", + "dev": true, + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-fs/node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "dev": true, + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/tar-stream": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", @@ -10720,6 +11940,70 @@ "node": ">=8" } }, + "node_modules/testcontainers": { + "version": "10.18.0", + "resolved": "https://registry.npmjs.org/testcontainers/-/testcontainers-10.18.0.tgz", + "integrity": "sha512-MnwWsPjsN5QVe+lSU1LwLZVOyjgwSwv1INzkw8FekdwgvOtvJ7FThQEkbmzRcguQootgwmA9FG54NoTChZDRvA==", + "dev": true, + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "@types/dockerode": "^3.3.29", + "archiver": "^7.0.1", + "async-lock": "^1.4.1", + "byline": "^5.0.0", + "debug": "^4.3.5", + "docker-compose": "^0.24.8", + "dockerode": "^3.3.5", + "get-port": "^5.1.1", + "proper-lockfile": "^4.1.2", + "properties-reader": "^2.3.0", + "ssh-remote-port-forward": "^1.0.4", + "tar-fs": "^3.0.6", + "tmp": "^0.2.3", + "undici": "^5.28.5" + } + }, + "node_modules/testcontainers/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "dev": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/testcontainers/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/testcontainers/node_modules/tmp": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", + "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==", + "dev": true, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/text-decoder": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "dev": true, + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -11002,6 +12286,18 @@ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz", "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=" }, + "node_modules/undici": { + "version": "5.28.5", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.5.tgz", + "integrity": "sha512-zICwjrDrcrUE0pyyJc1I2QzBkLM8FINsgOrt6WjA+BgajVq9Nxu2PbFFXUrAggLfDXlZGZBVZYw7WNV5KiBiBA==", + "dev": true, + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=14.0" + } + }, "node_modules/unique-string": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", @@ -11386,6 +12682,92 @@ "node": ">=6" } }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/wrap-ansi-cjs/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wrap-ansi/node_modules/ansi-regex": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", @@ -11723,6 +13105,89 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zip-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-6.0.1.tgz", + "integrity": "sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==", + "dev": true, + "dependencies": { + "archiver-utils": "^5.0.0", + "compress-commons": "^6.0.2", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/zip-stream/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/zip-stream/node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dev": true, + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/zip-stream/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/zip-stream/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/zstd-codec": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/zstd-codec/-/zstd-codec-0.1.4.tgz", diff --git a/package.json b/package.json index 236305297..ad68df157 100644 --- a/package.json +++ b/package.json @@ -93,7 +93,8 @@ "prettier": "3.2.5", "proxyquire": "^2.0.1", "sinon": "^5.0.0", - "supertest": "7.0.0", + "supertest": "^7.0.0", + "testcontainers": "^10.18.0", "typescript": "5.0.4" } } diff --git a/providers/caching/redis.js b/providers/caching/redis.js index e10c7de7f..efa8278b6 100644 --- a/providers/caching/redis.js +++ b/providers/caching/redis.js @@ -44,14 +44,14 @@ class RedisCache { await this._client.del(item) } - static buildRedisClient({ apiKey, service }) { + static buildRedisClient({ apiKey, service, port = 6380, tls = true }) { return createClient({ username: 'default', password: apiKey, socket: { host: service, - port: 6380, - tls: true + port, + tls }, pingInterval: 5 * 60 * 1000 // https://learn.microsoft.com/en-us/azure/azure-cache-for-redis/cache-best-practices-connection#idle-timeout }) diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js index 64d679c26..a1feb6723 100644 --- a/test/lib/rateLimit.js +++ b/test/lib/rateLimit.js @@ -12,6 +12,7 @@ const supertest = require('supertest') const express = require('express') const sandbox = require('sinon').createSandbox() const { RedisCache } = require('../../providers/caching/redis') +const { GenericContainer } = require('testcontainers') const logger = { info: () => {}, @@ -76,9 +77,19 @@ describe('Rate Limiter', () => { }) describe('Redis Based Rate Limiter', () => { - const apiKey = process.env['CACHING_REDIS_API_KEY'] - const service = process.env['CACHING_REDIS_SERVICE'] - const redis = { apiKey, service } + let client, rateLimiter + let container, redis + + before(async () => { + container = await new GenericContainer('redis').withExposedPorts(6379).start() + const service = container.getHost() + const port = container.getMappedPort(6379) + redis = { service, port, tls: false } + }) + + after(async () => { + await container.stop() + }) describe('Handling errors', () => { let rateLimiter @@ -111,16 +122,17 @@ describe('Rate Limiter', () => { }) describe('Rate Limit Integration Tests', () => { - let client, rateLimiter - - beforeEach(async () => { + before(async () => { rateLimiter = new RedisBasedRateLimiter({ limit, redis, logger }) const app = await buildApp(rateLimiter) client = supertest(app) }) - afterEach(async () => { + after(async () => { await rateLimiter.done() + }) + + afterEach(async () => { await new Promise(resolve => setTimeout(resolve, limit.windowMs)) }) diff --git a/test/providers/caching/redis.js b/test/providers/caching/redis.js index 3f2a5bb6c..50dd8d7ff 100644 --- a/test/providers/caching/redis.js +++ b/test/providers/caching/redis.js @@ -6,6 +6,7 @@ const sandbox = sinon.createSandbox() const assert = require('assert') const redisCache = require('../../../providers/caching/redis') const { RedisCache } = require('../../../providers/caching/redis') +const { GenericContainer } = require('testcontainers') const logger = { info: () => {}, @@ -80,16 +81,25 @@ describe('Redis Cache', () => { }) describe('Integration Test', () => { - const apiKey = process.env['CACHING_REDIS_API_KEY'] - const service = process.env['CACHING_REDIS_SERVICE'] + let container, service, port + + before(async () => { + container = await new GenericContainer('redis').withExposedPorts(6379).start() + service = container.getHost() + port = container.getMappedPort(6379) + }) + + after(async () => { + await container.stop() + }) describe('Redis Client Test', () => { let client - beforeEach(async () => { - client = await RedisCache.initializeClient({ apiKey, service }, logger) + before(async () => { + client = await RedisCache.initializeClient({ service, port, tls: false }, logger) }) - afterEach(async () => { + after(async () => { await client.quit() }) @@ -116,7 +126,7 @@ describe('Redis Cache', () => { value = await client.get('tee') assert.ok(value === 'value') - await new Promise(resolve => setTimeout(resolve, 1200)) + await new Promise(resolve => setTimeout(resolve, 1010)) value = await client.get('tee') assert.ok(value === null) }).timeout(3000) @@ -124,12 +134,12 @@ describe('Redis Cache', () => { describe('Redis Cache Test', () => { let cache - beforeEach(async () => { - cache = redisCache({ apiKey, service, logger }) + before(async () => { + cache = redisCache({ service, port, tls: false, logger }) await cache.initialize() }) - afterEach(async () => { + after(async () => { await cache.done() }) @@ -156,7 +166,7 @@ describe('Redis Cache', () => { value = await cache.get('wee') assert.ok(value === 'value') - await new Promise(resolve => setTimeout(resolve, 1200)) + await new Promise(resolve => setTimeout(resolve, 1010)) value = await cache.get('wee') assert.ok(value === null) }).timeout(3000) From fa422ef5a062f733ed7f2f609e10d49ba14d42c0 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Tue, 25 Feb 2025 09:11:37 -0800 Subject: [PATCH 06/22] Refactor --- providers/caching/redis.js | 6 +---- test/lib/rateLimit.js | 4 +-- test/providers/caching/redis.js | 43 +++++++++++++++++---------------- 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/providers/caching/redis.js b/providers/caching/redis.js index efa8278b6..0b25091f0 100644 --- a/providers/caching/redis.js +++ b/providers/caching/redis.js @@ -48,11 +48,7 @@ class RedisCache { return createClient({ username: 'default', password: apiKey, - socket: { - host: service, - port, - tls - }, + socket: { host: service, port, tls }, pingInterval: 5 * 60 * 1000 // https://learn.microsoft.com/en-us/azure/azure-cache-for-redis/cache-best-practices-connection#idle-timeout }) } diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js index a1feb6723..65a0e85f4 100644 --- a/test/lib/rateLimit.js +++ b/test/lib/rateLimit.js @@ -77,7 +77,6 @@ describe('Rate Limiter', () => { }) describe('Redis Based Rate Limiter', () => { - let client, rateLimiter let container, redis before(async () => { @@ -104,7 +103,7 @@ describe('Rate Limiter', () => { rateLimiter = new RedisBasedRateLimiter({ limit, logger }) await rateLimiter.initialize() } catch (error) { - assert.ok(error.message === 'Redis configuration is missing') + assert.strictEqual(error.message, 'Redis configuration is missing') } }) @@ -122,6 +121,7 @@ describe('Rate Limiter', () => { }) describe('Rate Limit Integration Tests', () => { + let client, rateLimiter before(async () => { rateLimiter = new RedisBasedRateLimiter({ limit, redis, logger }) const app = await buildApp(rateLimiter) diff --git a/test/providers/caching/redis.js b/test/providers/caching/redis.js index 50dd8d7ff..d92e2dd23 100644 --- a/test/providers/caching/redis.js +++ b/test/providers/caching/redis.js @@ -66,7 +66,7 @@ describe('Redis Cache', () => { await cache.set('foo', 'bar') await cache.delete('foo') const result = await cache.get('foo') - assert.ok(result === null) + assert.strictEqual(result, null) }) it('throws error if redis connection fails', async () => { @@ -75,18 +75,19 @@ describe('Redis Cache', () => { try { await cache.initialize() } catch (error) { - assert.equal(error.message, 'Connection failed') + assert.strictEqual(error.message, 'Connection failed') } }) }) describe('Integration Test', () => { - let container, service, port + let container, redisConfig before(async () => { container = await new GenericContainer('redis').withExposedPorts(6379).start() - service = container.getHost() - port = container.getMappedPort(6379) + const service = container.getHost() + const port = container.getMappedPort(6379) + redisConfig = { service, port, tls: false } }) after(async () => { @@ -96,7 +97,7 @@ describe('Redis Cache', () => { describe('Redis Client Test', () => { let client before(async () => { - client = await RedisCache.initializeClient({ service, port, tls: false }, logger) + client = await RedisCache.initializeClient(redisConfig, logger) }) after(async () => { @@ -105,37 +106,37 @@ describe('Redis Cache', () => { it('retrieves empty initially', async () => { const value = await client.get('boo') - assert.ok(value === null) + assert.strictEqual(value, null) }) it('sets, gets and removes a value', async () => { await client.set('foo', 'bar') let value = await client.get('foo') - assert.ok(value === 'bar') + assert.strictEqual(value, 'bar') //clear the value await client.del('foo') value = await client.get('foo') - assert.ok(value === null) + assert.strictEqual(value, null) }) it('sets value and exipres', async () => { let value = await client.get('tee') - assert.ok(value === null) + assert.strictEqual(value, null) await client.set('tee', 'value', { EX: 1 }) value = await client.get('tee') - assert.ok(value === 'value') + assert.strictEqual(value, 'value') await new Promise(resolve => setTimeout(resolve, 1010)) value = await client.get('tee') - assert.ok(value === null) - }).timeout(3000) + assert.strictEqual(value, null) + }).timeout(2000) }) describe('Redis Cache Test', () => { let cache before(async () => { - cache = redisCache({ service, port, tls: false, logger }) + cache = redisCache({ ...redisConfig, logger }) await cache.initialize() }) @@ -145,31 +146,31 @@ describe('Redis Cache', () => { it('retrieves empty initially', async () => { const value = await cache.get('boo') - assert.ok(value === null) + assert.strictEqual(value, null) }) it('sets, gets and removes a value', async () => { await cache.set('foo', 'bar') let value = await cache.get('foo') - assert.ok(value === 'bar') + assert.strictEqual(value, 'bar') //clear the value await cache.delete('foo') value = await cache.get('foo') - assert.ok(value === null) + assert.strictEqual(value, null) }) it('sets value and exipres', async () => { let value = await cache.get('wee') - assert.ok(value === null) + assert.strictEqual(value, null) await cache.set('wee', 'value', 1) value = await cache.get('wee') - assert.ok(value === 'value') + assert.strictEqual(value, 'value') await new Promise(resolve => setTimeout(resolve, 1010)) value = await cache.get('wee') - assert.ok(value === null) - }).timeout(3000) + assert.strictEqual(value, null) + }).timeout(2000) }) }) }) From 6bc99781039d137b9c24f5f744a08c6842632920 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Tue, 25 Feb 2025 10:01:56 -0800 Subject: [PATCH 07/22] Use default sandbox --- test/lib/rateLimit.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js index 65a0e85f4..3eeffc960 100644 --- a/test/lib/rateLimit.js +++ b/test/lib/rateLimit.js @@ -10,7 +10,7 @@ const { } = require('../../lib/rateLimit.js') const supertest = require('supertest') const express = require('express') -const sandbox = require('sinon').createSandbox() +const sinon = require('sinon') const { RedisCache } = require('../../providers/caching/redis') const { GenericContainer } = require('testcontainers') @@ -95,7 +95,7 @@ describe('Rate Limiter', () => { afterEach(async () => { await rateLimiter.done() - sandbox.restore() + sinon.restore() }) it('throws error if redis configuration is missing', async () => { @@ -108,7 +108,7 @@ describe('Rate Limiter', () => { }) it('throws error if connecting to redis fails', async () => { - sandbox.stub(RedisCache, 'buildRedisClient').returns({ + sinon.stub(RedisCache, 'buildRedisClient').returns({ connect: () => Promise.reject(new Error('Connection failed')) }) try { From b8f52ec9f91fc1a11deab335b8077c00da6d6567 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Tue, 25 Feb 2025 20:49:14 -0800 Subject: [PATCH 08/22] Turn of logging in definitionServiceTest --- test/business/definitionServiceTest.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/business/definitionServiceTest.js b/test/business/definitionServiceTest.js index db15f8b6c..7370f0fb0 100644 --- a/test/business/definitionServiceTest.js +++ b/test/business/definitionServiceTest.js @@ -349,9 +349,9 @@ describe('Integration test', () => { let logger, upgradeHandler beforeEach(() => { logger = { - debug: (format, ...args) => console.debug(util.format(format, ...args)), - error: (format, ...args) => console.error(util.format(format, ...args)), - info: (format, ...args) => console.info(util.format(format, ...args)) + debug: () => {}, + error: () => {}, + info: () => {} } }) From ba04bf7d8ec21ae09f05a1ef5985c6c24b97a5d3 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Wed, 26 Feb 2025 13:51:11 -0800 Subject: [PATCH 09/22] Fix unit tests --- test/business/noticeServiceTest.js | 11 +++++++++++ test/lib/rateLimit.js | 4 +--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/test/business/noticeServiceTest.js b/test/business/noticeServiceTest.js index b2fb3f866..047435841 100644 --- a/test/business/noticeServiceTest.js +++ b/test/business/noticeServiceTest.js @@ -5,8 +5,19 @@ const { expect } = require('chai') const sinon = require('sinon') const NoticeService = require('../../business/noticeService') const spdxLicenseList = require('spdx-license-list/full') +const logger = require('../../providers/logging/logger') + +const mockLogger = { + info: () => {}, + error: () => {}, + debug: () => {} +} describe('Notice Service', () => { + before(() => { + logger(mockLogger) + }) + it('generates simple notice', async () => { const { service, coordinates } = setup({ 'npm/npmjs/-/test/1.0.0': { diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js index 3eeffc960..7916e03a5 100644 --- a/test/lib/rateLimit.js +++ b/test/lib/rateLimit.js @@ -234,9 +234,7 @@ async function tryBeyondLimit(max, client) { let counter = 0 while (counter < max + 10) { const response = await client.get('/') - if (!response.ok) { - break - } + if (!response.ok) break counter++ } return counter From 84129f3eebd73a02d1932a29dc51bbb9537a2e4a Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Wed, 26 Feb 2025 14:37:00 -0800 Subject: [PATCH 10/22] Add default logger fallback --- bin/www | 2 +- providers/caching/redis.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/www b/bin/www index e1188ad57..3b3eebb12 100644 --- a/bin/www +++ b/bin/www @@ -125,4 +125,4 @@ async function connect(config) { process.exit(1) }) return { rateLimiter, batchRateLimiter, logger } -} \ No newline at end of file +} diff --git a/providers/caching/redis.js b/providers/caching/redis.js index 0b25091f0..d4b73a25d 100644 --- a/providers/caching/redis.js +++ b/providers/caching/redis.js @@ -3,13 +3,14 @@ const { createClient } = require('redis') const pako = require('pako') +const logger = require('../logging/logger') const objectPrefix = '*!~%' class RedisCache { constructor(options) { this.options = options - this.logger = options.logger + this.logger = options.logger || logger() } async initialize() { From 12f85d3f90ebffef470fc13c6802cd18af5d2959 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Thu, 27 Feb 2025 09:36:26 -0800 Subject: [PATCH 11/22] Share redis client across api and batch rate limiting and caching --- app.js | 11 +- bin/www | 14 ++- lib/rateLimit.js | 53 ++++------ providers/caching/memory.js | 3 +- providers/caching/redis.js | 4 + test/app.js | 17 ++- test/lib/rateLimit.js | 203 +++++++++++++++++------------------- 7 files changed, 141 insertions(+), 164 deletions(-) diff --git a/app.js b/app.js index 8db4a9021..a7372d20c 100644 --- a/app.js +++ b/app.js @@ -13,9 +13,11 @@ const requestId = require('request-id/express') const passport = require('passport') const swaggerUi = require('swagger-ui-express') const routesVersioning = require('express-routes-versioning')() +const { createApiLimiter, createBatchApiLimiter } = require('./lib/rateLimit') + const v1 = '1.0.0' -function createApp(config, { logger, rateLimiter, batchRateLimiter }) { +function createApp(config, { logger, cachingService }) { const initializers = [] process.on('unhandledRejection', exception => logger.error('unhandledRejection', exception)) @@ -43,9 +45,6 @@ function createApp(config, { logger, rateLimiter, batchRateLimiter }) { const searchService = config.search.service() initializers.push(async () => searchService.initialize()) - const cachingService = config.caching.service() - initializers.push(async () => cachingService.initialize()) - const curationService = config.curation.service(null, curationStore, config.endpoints, cachingService, harvestStore) const curationQueue = config.curation.queue() @@ -128,7 +127,7 @@ function createApp(config, { logger, rateLimiter, batchRateLimiter }) { app.set('trust-proxy', true) - app.use(rateLimiter.middleware) + app.use(createApiLimiter(config, cachingService).middleware) // Use a (potentially lower) different API limit // for batch API request @@ -136,7 +135,7 @@ function createApp(config, { logger, rateLimiter, batchRateLimiter }) { // * POST /definitions // * POST /curations // * POST /notices - const batchApiLimiter = batchRateLimiter.middleware + const batchApiLimiter = createBatchApiLimiter(config, cachingService).middleware app.post('/definitions', batchApiLimiter) app.post('/curations', batchApiLimiter) app.post('/notices', batchApiLimiter) diff --git a/bin/www b/bin/www index 3b3eebb12..fe94943fe 100644 --- a/bin/www +++ b/bin/www @@ -4,7 +4,6 @@ const config = require('./config') const loggerFactory = require('../providers/logging/logger') -const { createApiLimiter, createBatchApiLimiter } = require('../lib/rateLimit') const createApp = require('../app') const debug = require('debug')('service:server') const http = require('http') @@ -94,11 +93,11 @@ function onListening(server) { * @param server - The server instance to be shut down. * @param {Object} resources - The resources to be cleaned up during shutdown. */ -const onShutdown = (server, { rateLimiter, batchRateLimiter, logger }) => { +const onShutdown = (server, { cachingService, logger }) => { logger.info('Shutdown started') server.close(() => { logger.info('Server closed') - Promise.allSettled([rateLimiter.done(), batchRateLimiter.done()]).then(results => { + Promise.allSettled([cachingService.done()]).then(results => { const errorResults = results.filter(result => result.status === 'rejected') if (errorResults.length === 0) { logger.info('Shutdown complete') @@ -118,11 +117,10 @@ const onShutdown = (server, { rateLimiter, batchRateLimiter, logger }) => { */ async function connect(config) { const logger = loggerFactory(config.logging.logger()) - const rateLimiter = createApiLimiter(config, logger) - const batchRateLimiter = createBatchApiLimiter(config, logger) - await Promise.all([rateLimiter.initialize(), batchRateLimiter.initialize()]).catch(error => { - logger.error('Error initializing rate limiters', error) + const cachingService = config.caching.service() + await Promise.all([cachingService.initialize()]).catch(error => { + logger.error('Error connecting to cachingService', error) process.exit(1) }) - return { rateLimiter, batchRateLimiter, logger } + return { cachingService, logger } } diff --git a/lib/rateLimit.js b/lib/rateLimit.js index ac102d0e4..ce19450b4 100644 --- a/lib/rateLimit.js +++ b/lib/rateLimit.js @@ -5,32 +5,28 @@ const { createClient } = require('redis') const { RedisStore } = require('rate-limit-redis') const { rateLimit } = require('express-rate-limit') const { RedisCache } = require('../providers/caching/redis') +const logger = require('../providers/logging/logger') class RateLimiter { constructor(opts) { this.options = opts - this.logger = opts.logger + this.logger = opts.logger || logger() } - async initialize(store) { + initialize(store) { if (!this._limiter) { this.logger.debug('Creating rate limiter: %o', this.options.limit) const options = RateLimiter.buildOptions(this.options.limit, store) this._limiter = rateLimit(options) - this.logger.info('Rate limiter initialized') + this.logger.debug('Rate limiter initialized') } } get middleware() { - if (!this._limiter) throw new Error('Rate limiter not initialized') + this.initialize() return this._limiter } - async done() { - //do nothing - this.logger.info('Rate limiter done') - } - static buildOptions({ windowMs, max }, store) { //TODO: use standardHeaders? const opts = { @@ -49,21 +45,15 @@ class RateLimiter { } class RedisBasedRateLimiter extends RateLimiter { - async initialize() { - if (!this._limiter) { - this._client = await this._initializeClient() - const store = RedisBasedRateLimiter.buildRedisStore(this._client, this.options.redis) - await super.initialize(store) - } - } - - async done() { - return this._client?.quit().then(() => super.done()) + constructor(opts) { + super(opts) + this._client = opts.redis?.client } - async _initializeClient() { - if (!this.options.redis) throw new Error('Redis configuration is missing') - return RedisCache.initializeClient(this.options.redis, this.logger) + initialize() { + if (!this._client) throw new Error('Redis client is missing') + const store = RedisBasedRateLimiter.buildRedisStore(this._client, this.options.redis) + super.initialize(store) } static buildRedisStore(client, { prefix }) { @@ -78,16 +68,9 @@ function createRateLimiter(config) { return config.redis ? new RedisBasedRateLimiter(config) : new RateLimiter(config) } -function buildOpts({ windowSeconds, max }, logger, { caching_redis_service, caching_redis_api_key } = {}, prefix) { +function buildOpts({ windowSeconds, max }, cachingService, prefix, logger) { const limit = { windowMs: windowSeconds * 1000, max } - let redis - if (caching_redis_service) { - redis = { - service: caching_redis_service, - apiKey: caching_redis_api_key, - prefix - } - } + const redis = cachingService instanceof RedisCache ? { client: cachingService.client, prefix } : undefined return { limit, redis, logger } } @@ -95,12 +78,12 @@ function buildBatchOpts({ batchWindowSeconds, batchMax }, ...args) { return buildOpts({ windowSeconds: batchWindowSeconds, max: batchMax }, ...args) } -function createApiLimiter(config, logger) { - return createRateLimiter(buildOpts(config.limits, logger, config.caching, 'api')) +function createApiLimiter(config, cachingService, logger) { + return createRateLimiter(buildOpts(config.limits, cachingService, 'api', logger)) } -function createBatchApiLimiter(config, logger) { - return createRateLimiter(buildBatchOpts(config.limits, logger, config.caching, 'batch-api')) +function createBatchApiLimiter(config, cachingService, logger) { + return createRateLimiter(buildBatchOpts(config.limits, cachingService, 'batch-api', logger)) } module.exports = { diff --git a/providers/caching/memory.js b/providers/caching/memory.js index 99217d151..dfdc9c93c 100644 --- a/providers/caching/memory.js +++ b/providers/caching/memory.js @@ -9,7 +9,8 @@ class MemoryCache { this.defaultTtlSeconds = options.defaultTtlSeconds } - initialize() {} + async initialize() {} + async done() {} get(item) { return this.cache.get(item) diff --git a/providers/caching/redis.js b/providers/caching/redis.js index d4b73a25d..aa72a8861 100644 --- a/providers/caching/redis.js +++ b/providers/caching/redis.js @@ -21,6 +21,10 @@ class RedisCache { return this._client?.quit() } + get client() { + return this._client + } + async get(item) { const cacheItem = await this._client.get(item) if (!cacheItem) return null diff --git a/test/app.js b/test/app.js index 55a3aa821..d2fe28a7b 100644 --- a/test/app.js +++ b/test/app.js @@ -6,6 +6,7 @@ process.env['CURATION_GITHUB_TOKEN'] = '123' process.env['GITLAB_TOKEN'] = 'abc' const init = require('express-init') const Application = require('../app') +const logger = require('../providers/logging/logger') const config = proxyquire('../bin/config', { ['painless-config']: { get: name => { @@ -19,15 +20,23 @@ const config = proxyquire('../bin/config', { } }) +const mockLogger = { + info: () => {}, + error: () => {}, + debug: () => {} +} + describe('Application', () => { - const middleware = async (req, res, next) => {} let resources = {} beforeEach(async () => { + logger(mockLogger) + resources = { - rateLimiter: { middleware }, - batchRateLimiter: { middleware }, - logger: console + logger: mockLogger, + cachingService: { + initialize: async () => {} + } } }) diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js index 7916e03a5..122a6de39 100644 --- a/test/lib/rateLimit.js +++ b/test/lib/rateLimit.js @@ -23,11 +23,11 @@ const logger = { const limit = { windowMs: 1000, max: 1 } describe('Rate Limiter', () => { - describe('Rate Limiter Tests', () => { - let client, rateLimiter + describe('Rate Limit Integration Tests', () => { + let client beforeEach(async () => { - rateLimiter = new RateLimiter({ limit, logger }) + const rateLimiter = new RateLimiter({ limit, logger }) const app = await buildApp(rateLimiter) client = supertest(app) }) @@ -45,7 +45,9 @@ describe('Rate Limiter', () => { const counter = await tryBeyondLimit(limit.max, client) assert.strictEqual(counter, limit.max, `Counter is ${counter}`) }) + }) + describe('Build Limit Options', () => { it('builds rate limit options', () => { const options = RateLimiter.buildOptions(limit) assert.deepStrictEqual(options, { @@ -77,59 +79,33 @@ describe('Rate Limiter', () => { }) describe('Redis Based Rate Limiter', () => { - let container, redis - - before(async () => { - container = await new GenericContainer('redis').withExposedPorts(6379).start() - const service = container.getHost() - const port = container.getMappedPort(6379) - redis = { service, port, tls: false } - }) - - after(async () => { - await container.stop() - }) - - describe('Handling errors', () => { - let rateLimiter - - afterEach(async () => { - await rateLimiter.done() - sinon.restore() - }) - - it('throws error if redis configuration is missing', async () => { - try { - rateLimiter = new RedisBasedRateLimiter({ limit, logger }) - await rateLimiter.initialize() - } catch (error) { - assert.strictEqual(error.message, 'Redis configuration is missing') - } - }) - - it('throws error if connecting to redis fails', async () => { - sinon.stub(RedisCache, 'buildRedisClient').returns({ - connect: () => Promise.reject(new Error('Connection failed')) - }) - try { - rateLimiter = new RedisBasedRateLimiter({ limit, redis, logger }) - await rateLimiter.initialize() - } catch (error) { - assert.equal(error.message, 'Connection failed') - } - }) + it('throws error if redis client is missing', async () => { + try { + //eslint-disable-next-line no-unused-vars + const middleware = new RedisBasedRateLimiter({ limit, logger }).middleware + assert.fail('Should have thrown error') + } catch (error) { + assert.strictEqual(error.message, 'Redis client is missing') + } }) describe('Rate Limit Integration Tests', () => { - let client, rateLimiter + let container, redisClient, client + before(async () => { - rateLimiter = new RedisBasedRateLimiter({ limit, redis, logger }) + container = await new GenericContainer('redis').withExposedPorts(6379).start() + const service = container.getHost() + const port = container.getMappedPort(6379) + const redisOpts = { service, port, tls: false } + redisClient = await RedisCache.initializeClient(redisOpts, logger) + const rateLimiter = new RedisBasedRateLimiter({ limit, redis: { client: redisClient }, logger }) const app = await buildApp(rateLimiter) client = supertest(app) }) after(async () => { - await rateLimiter.done() + await redisClient.quit() + await container.stop() }) afterEach(async () => { @@ -159,73 +135,81 @@ describe('Rate Limiter', () => { batchWindowSeconds: 10, batchMax: 10 } - const caching = { - caching_redis_service: 'host', - caching_redis_api_key: 'key' - } - it('builds a rate limiter', () => { - const rateLimiter = createApiLimiter({ limits }) - assert.ok(rateLimiter instanceof RateLimiter) - const expected = { - limit: { - windowMs: 1000, - max: 0 - }, - redis: undefined, - logger: undefined - } - assert.deepStrictEqual(rateLimiter.options, expected) - }) + describe('Create Rate Limiter without Caching', () => { + it('builds a rate limiter', () => { + const rateLimiter = createApiLimiter({ limits }, undefined, logger) + assert.ok(rateLimiter instanceof RateLimiter) + const expected = { + limit: { + windowMs: 1000, + max: 0 + }, + redis: undefined, + logger + } + assert.deepStrictEqual(rateLimiter.options, expected) + }) - it('builds a batch rate limiter', () => { - const batchRateLimiter = createBatchApiLimiter({ limits }) - assert.ok(batchRateLimiter instanceof RateLimiter) - const expected = { - limit: { - windowMs: 10000, - max: 10 - }, - redis: undefined, - logger: undefined - } - assert.deepStrictEqual(batchRateLimiter.options, expected) + it('builds a batch rate limiter', () => { + const batchRateLimiter = createBatchApiLimiter({ limits }, undefined, logger) + assert.ok(batchRateLimiter instanceof RateLimiter) + const expected = { + limit: { + windowMs: 10000, + max: 10 + }, + redis: undefined, + logger + } + assert.deepStrictEqual(batchRateLimiter.options, expected) + }) }) - it('builds a redis based rate limiter', () => { - const rateLimiter = createApiLimiter({ limits, caching }) - assert.ok(rateLimiter instanceof RedisBasedRateLimiter) - const expected = { - limit: { - windowMs: 1000, - max: 0 - }, - redis: { - service: 'host', - apiKey: 'key', - prefix: 'api' - }, - logger: undefined - } - assert.deepStrictEqual(rateLimiter.options, expected) - }) + describe('Create Rate Limiter with Caching', () => { + let caching + beforeEach(() => { + caching = new RedisCache({ logger }) + sinon.stub(caching, 'client').value({}) + }) - it('builds a redis based batch rate limiter', () => { - const batchRateLimiter = createBatchApiLimiter({ limits, caching }) - assert.ok(batchRateLimiter instanceof RedisBasedRateLimiter) - const expected = { - limit: { - windowMs: 10000, - max: 10 - }, - redis: { - service: 'host', - apiKey: 'key', - prefix: 'batch-api' - }, - logger: undefined - } - assert.deepStrictEqual(batchRateLimiter.options, expected) + afterEach(() => { + sinon.restore() + }) + + it('builds a redis based rate limiter', () => { + const rateLimiter = createApiLimiter({ limits }, caching, logger) + assert.ok(rateLimiter instanceof RedisBasedRateLimiter) + const expected = { + limit: { + windowMs: 1000, + max: 0 + }, + redis: { + client: {}, + prefix: 'api' + }, + logger + } + assert.deepStrictEqual(rateLimiter.options, expected) + }) + + it('builds a redis based batch rate limiter', () => { + const batchRateLimiter = createBatchApiLimiter({ limits }, caching, logger) + assert.ok(batchRateLimiter instanceof RedisBasedRateLimiter) + const expected = { + limit: { + windowMs: 10000, + max: 10 + }, + redis: { + client: {}, + prefix: 'batch-api' + }, + logger + } + assert.deepStrictEqual(batchRateLimiter.options, expected) + }) }) }) }) @@ -242,7 +226,6 @@ async function tryBeyondLimit(max, client) { async function buildApp(rateLimiter) { const app = express() - await rateLimiter.initialize() app.use(rateLimiter.middleware) app.get('/', (req, res) => res.send('Hello World!')) return app From a9ad0be382d030d9967d1a27ece7c1103b873ef2 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Wed, 5 Mar 2025 16:38:33 -0800 Subject: [PATCH 12/22] Add logging to integration test setup --- test/lib/rateLimit.js | 31 ++++++++++++++++++++----------- test/providers/caching/redis.js | 19 +++++++++++++------ 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js index 122a6de39..95a722d6e 100644 --- a/test/lib/rateLimit.js +++ b/test/lib/rateLimit.js @@ -92,20 +92,29 @@ describe('Rate Limiter', () => { describe('Rate Limit Integration Tests', () => { let container, redisClient, client - before(async () => { - container = await new GenericContainer('redis').withExposedPorts(6379).start() - const service = container.getHost() - const port = container.getMappedPort(6379) - const redisOpts = { service, port, tls: false } - redisClient = await RedisCache.initializeClient(redisOpts, logger) - const rateLimiter = new RedisBasedRateLimiter({ limit, redis: { client: redisClient }, logger }) - const app = await buildApp(rateLimiter) - client = supertest(app) + before(async function () { + this.timeout(4000) + try { + console.info(`Starting redis generic container, timestamp: ${new Date().toISOString()}`) + container = await new GenericContainer('redis').withExposedPorts(6379).start() + const service = container.getHost() + const port = container.getMappedPort(6379) + console.info(`Redis running at ${service}:${port}, timestamp: ${new Date().toISOString()}`) + const redisOpts = { service, port, tls: false } + redisClient = await RedisCache.initializeClient(redisOpts, logger) + console.info(`Redis client connected, timestamp: ${new Date().toISOString()}`) + const rateLimiter = new RedisBasedRateLimiter({ limit, redis: { client: redisClient }, logger }) + const app = await buildApp(rateLimiter) + console.info(`Test app started, timestamp: ${new Date().toISOString()}`) + client = supertest(app) + } catch (error) { + console.error(error) + } }) after(async () => { - await redisClient.quit() - await container.stop() + await redisClient?.quit() + await container?.stop() }) afterEach(async () => { diff --git a/test/providers/caching/redis.js b/test/providers/caching/redis.js index d92e2dd23..33e052528 100644 --- a/test/providers/caching/redis.js +++ b/test/providers/caching/redis.js @@ -83,15 +83,22 @@ describe('Redis Cache', () => { describe('Integration Test', () => { let container, redisConfig - before(async () => { - container = await new GenericContainer('redis').withExposedPorts(6379).start() - const service = container.getHost() - const port = container.getMappedPort(6379) - redisConfig = { service, port, tls: false } + before(async function () { + this.timeout(4000) + try { + console.info(`Starting redis generic container, timestamp: ${new Date().toISOString()}`) + container = await new GenericContainer('redis').withExposedPorts(6379).start() + const service = container.getHost() + const port = container.getMappedPort(6379) + console.info(`Redis running at ${service}:${port}, timestamp: ${new Date().toISOString()}`) + redisConfig = { service, port, tls: false } + } catch (error) { + console.error(error) + } }) after(async () => { - await container.stop() + await container?.stop() }) describe('Redis Client Test', () => { From e3d72a23183a8e546db39ded5b3d02c252c8674e Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Wed, 5 Mar 2025 16:45:06 -0800 Subject: [PATCH 13/22] Adjust timeout for integration test setup --- test/lib/rateLimit.js | 26 +++++++++----------------- test/providers/caching/redis.js | 16 +++++----------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js index 95a722d6e..707615f5d 100644 --- a/test/lib/rateLimit.js +++ b/test/lib/rateLimit.js @@ -93,23 +93,15 @@ describe('Rate Limiter', () => { let container, redisClient, client before(async function () { - this.timeout(4000) - try { - console.info(`Starting redis generic container, timestamp: ${new Date().toISOString()}`) - container = await new GenericContainer('redis').withExposedPorts(6379).start() - const service = container.getHost() - const port = container.getMappedPort(6379) - console.info(`Redis running at ${service}:${port}, timestamp: ${new Date().toISOString()}`) - const redisOpts = { service, port, tls: false } - redisClient = await RedisCache.initializeClient(redisOpts, logger) - console.info(`Redis client connected, timestamp: ${new Date().toISOString()}`) - const rateLimiter = new RedisBasedRateLimiter({ limit, redis: { client: redisClient }, logger }) - const app = await buildApp(rateLimiter) - console.info(`Test app started, timestamp: ${new Date().toISOString()}`) - client = supertest(app) - } catch (error) { - console.error(error) - } + this.timeout(10000) + container = await new GenericContainer('redis').withExposedPorts(6379).start() + const service = container.getHost() + const port = container.getMappedPort(6379) + const redisOpts = { service, port, tls: false } + redisClient = await RedisCache.initializeClient(redisOpts, logger) + const rateLimiter = new RedisBasedRateLimiter({ limit, redis: { client: redisClient }, logger }) + const app = await buildApp(rateLimiter) + client = supertest(app) }) after(async () => { diff --git a/test/providers/caching/redis.js b/test/providers/caching/redis.js index 33e052528..24df7f342 100644 --- a/test/providers/caching/redis.js +++ b/test/providers/caching/redis.js @@ -84,17 +84,11 @@ describe('Redis Cache', () => { let container, redisConfig before(async function () { - this.timeout(4000) - try { - console.info(`Starting redis generic container, timestamp: ${new Date().toISOString()}`) - container = await new GenericContainer('redis').withExposedPorts(6379).start() - const service = container.getHost() - const port = container.getMappedPort(6379) - console.info(`Redis running at ${service}:${port}, timestamp: ${new Date().toISOString()}`) - redisConfig = { service, port, tls: false } - } catch (error) { - console.error(error) - } + this.timeout(10000) + container = await new GenericContainer('redis').withExposedPorts(6379).start() + const service = container.getHost() + const port = container.getMappedPort(6379) + redisConfig = { service, port, tls: false } }) after(async () => { From a4c61bf569a4932e9a15468125ae41a2574c8f20 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Thu, 6 Mar 2025 09:19:53 -0800 Subject: [PATCH 14/22] Fix linting errors --- lib/rateLimit.js | 1 - test/business/definitionServiceTest.js | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/rateLimit.js b/lib/rateLimit.js index ce19450b4..43aef90e0 100644 --- a/lib/rateLimit.js +++ b/lib/rateLimit.js @@ -1,7 +1,6 @@ // (c) Copyright 2025, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. // SPDX-License-Identifier: MIT -const { createClient } = require('redis') const { RedisStore } = require('rate-limit-redis') const { rateLimit } = require('express-rate-limit') const { RedisCache } = require('../providers/caching/redis') diff --git a/test/business/definitionServiceTest.js b/test/business/definitionServiceTest.js index 7370f0fb0..29ed98a05 100644 --- a/test/business/definitionServiceTest.js +++ b/test/business/definitionServiceTest.js @@ -18,7 +18,6 @@ const AggregatorService = require('../../business/aggregator') const DefinitionQueueUpgrader = require('../../providers/upgrade/defUpgradeQueue') const memoryQueue = require('../../providers/upgrade/memoryQueueConfig') const { DefinitionVersionChecker } = require('../../providers/upgrade/defVersionCheck') -const util = require('util') describe('Definition Service', () => { it('invalidates single coordinate', async () => { From d0160f47496f0218be5a0d2c29102dae870579a2 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Thu, 6 Mar 2025 09:59:33 -0800 Subject: [PATCH 15/22] Disable the unstable unit test. The application test starts a timer on the curation queue. This timer is not cleaned up, which causes intermittent failures in the "Curation queue processing" unit tests. Disable the unit test. --- test/app.js | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/test/app.js b/test/app.js index d2fe28a7b..a51ecd345 100644 --- a/test/app.js +++ b/test/app.js @@ -27,21 +27,15 @@ const mockLogger = { } describe('Application', () => { - let resources = {} - beforeEach(async () => { logger(mockLogger) - - resources = { - logger: mockLogger, - cachingService: { - initialize: async () => {} - } - } }) - it('should initialize', done => { - const app = Application(config, resources) + it.skip('should initialize', done => { + const app = Application(config, { + logger: mockLogger, + cachingService: {} + }) init(app, error => { if (error) { done(error) From 99a9aa4bb2f991d0b65b57142c526c42c896d724 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Thu, 6 Mar 2025 18:04:29 -0800 Subject: [PATCH 16/22] Minor Refactor --- bin/www | 2 +- lib/rateLimit.js | 16 +++++++++------- test/lib/rateLimit.js | 15 ++++++++++----- test/providers/caching/redis.js | 23 +++++++++++++++++------ 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/bin/www b/bin/www index fe94943fe..d64dc27fd 100644 --- a/bin/www +++ b/bin/www @@ -93,7 +93,7 @@ function onListening(server) { * @param server - The server instance to be shut down. * @param {Object} resources - The resources to be cleaned up during shutdown. */ -const onShutdown = (server, { cachingService, logger }) => { +function onShutdown(server, { cachingService, logger }) { logger.info('Shutdown started') server.close(() => { logger.info('Server closed') diff --git a/lib/rateLimit.js b/lib/rateLimit.js index 43aef90e0..888f89d28 100644 --- a/lib/rateLimit.js +++ b/lib/rateLimit.js @@ -12,22 +12,24 @@ class RateLimiter { this.logger = opts.logger || logger() } + // Initialize the rate limiter with optional store initialize(store) { if (!this._limiter) { - this.logger.debug('Creating rate limiter: %o', this.options.limit) + this.logger.debug('Creating rate limiter', this.options.limit) const options = RateLimiter.buildOptions(this.options.limit, store) this._limiter = rateLimit(options) this.logger.debug('Rate limiter initialized') } + return this } + // Return the rate limiter middleware get middleware() { - this.initialize() - return this._limiter + return this.initialize()._limiter } + // Build rate limiter options static buildOptions({ windowMs, max }, store) { - //TODO: use standardHeaders? const opts = { standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers legacyHeaders: false // Disable the `X-RateLimit-*` headers @@ -52,7 +54,7 @@ class RedisBasedRateLimiter extends RateLimiter { initialize() { if (!this._client) throw new Error('Redis client is missing') const store = RedisBasedRateLimiter.buildRedisStore(this._client, this.options.redis) - super.initialize(store) + return super.initialize(store) } static buildRedisStore(client, { prefix }) { @@ -63,8 +65,8 @@ class RedisBasedRateLimiter extends RateLimiter { } } -function createRateLimiter(config) { - return config.redis ? new RedisBasedRateLimiter(config) : new RateLimiter(config) +function createRateLimiter(opts) { + return opts.redis ? new RedisBasedRateLimiter(opts) : new RateLimiter(opts) } function buildOpts({ windowSeconds, max }, cachingService, prefix, logger) { diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js index 707615f5d..117c1bffe 100644 --- a/test/lib/rateLimit.js +++ b/test/lib/rateLimit.js @@ -94,11 +94,7 @@ describe('Rate Limiter', () => { before(async function () { this.timeout(10000) - container = await new GenericContainer('redis').withExposedPorts(6379).start() - const service = container.getHost() - const port = container.getMappedPort(6379) - const redisOpts = { service, port, tls: false } - redisClient = await RedisCache.initializeClient(redisOpts, logger) + ;({ container, redisClient } = await setupRedis()) const rateLimiter = new RedisBasedRateLimiter({ limit, redis: { client: redisClient }, logger }) const app = await buildApp(rateLimiter) client = supertest(app) @@ -215,6 +211,15 @@ describe('Rate Limiter', () => { }) }) +async function setupRedis() { + const container = await new GenericContainer('redis').withExposedPorts(6379).start() + const service = container.getHost() + const port = container.getMappedPort(6379) + const redisOpts = { service, port, tls: false } + const redisClient = await RedisCache.initializeClient(redisOpts, logger) + return { container, redisClient } +} + async function tryBeyondLimit(max, client) { let counter = 0 while (counter < max + 10) { diff --git a/test/providers/caching/redis.js b/test/providers/caching/redis.js index 24df7f342..238331988 100644 --- a/test/providers/caching/redis.js +++ b/test/providers/caching/redis.js @@ -42,7 +42,7 @@ describe('Redis Cache', () => { await cache.initialize() await cache.set('foo', 'bar') const result = await cache.get('foo') - assert.equal(result, 'bar') + assert.strictEqual(result, 'bar') }) it('works well for an object', async () => { @@ -50,14 +50,14 @@ describe('Redis Cache', () => { await cache.initialize() await cache.set('foo', { temp: 3 }) const result = await cache.get('foo') - assert.equal(result.temp, 3) + assert.strictEqual(result.temp, 3) }) it('returns null for missing entry', async () => { const cache = redisCache({ logger }) await cache.initialize() const result = await cache.get('bar') - assert.equal(result, null) + assert.strictEqual(result, null) }) it('deletes a key', async () => { @@ -131,7 +131,7 @@ describe('Redis Cache', () => { await new Promise(resolve => setTimeout(resolve, 1010)) value = await client.get('tee') assert.strictEqual(value, null) - }).timeout(2000) + }) }) describe('Redis Cache Test', () => { @@ -150,7 +150,7 @@ describe('Redis Cache', () => { assert.strictEqual(value, null) }) - it('sets, gets and removes a value', async () => { + it('sets, gets and removes a string', async () => { await cache.set('foo', 'bar') let value = await cache.get('foo') assert.strictEqual(value, 'bar') @@ -160,6 +160,17 @@ describe('Redis Cache', () => { assert.strictEqual(value, null) }) + it('sets, gets and removes a object', async () => { + const obj = { foo: 'bar' } + await cache.set('foo', obj) + let value = await cache.get('foo') + assert.deepStrictEqual(value, obj) + //clear the value + await cache.delete('foo') + value = await cache.get('foo') + assert.strictEqual(value, null) + }) + it('sets value and exipres', async () => { let value = await cache.get('wee') assert.strictEqual(value, null) @@ -171,7 +182,7 @@ describe('Redis Cache', () => { await new Promise(resolve => setTimeout(resolve, 1010)) value = await cache.get('wee') assert.strictEqual(value, null) - }).timeout(2000) + }) }) }) }) From f0fc99c362db69331db3d3bcc6e7e7ba2220f530 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Mon, 10 Mar 2025 10:49:03 -0700 Subject: [PATCH 17/22] Allow more time to set up integration tests --- test/lib/rateLimit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js index 117c1bffe..fdc7da553 100644 --- a/test/lib/rateLimit.js +++ b/test/lib/rateLimit.js @@ -93,7 +93,7 @@ describe('Rate Limiter', () => { let container, redisClient, client before(async function () { - this.timeout(10000) + this.timeout(15000) ;({ container, redisClient } = await setupRedis()) const rateLimiter = new RedisBasedRateLimiter({ limit, redis: { client: redisClient }, logger }) const app = await buildApp(rateLimiter) From 1d38d1cc31bc1cda77cb5aa1b0d60a7266cf3f51 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Thu, 13 Mar 2025 18:00:46 -0700 Subject: [PATCH 18/22] Use MiddlewareDelegate to delay construction of rate limiter after redis connection --- app.js | 13 +++-- bin/www | 96 ++++++++------------------------- lib/rateLimit.js | 49 ++++++++++++++++- providers/caching/redis.js | 11 +++- test/providers/caching/redis.js | 6 +++ 5 files changed, 95 insertions(+), 80 deletions(-) diff --git a/app.js b/app.js index a7372d20c..91286c2f6 100644 --- a/app.js +++ b/app.js @@ -12,14 +12,16 @@ const serializeError = require('serialize-error') const requestId = require('request-id/express') const passport = require('passport') const swaggerUi = require('swagger-ui-express') +const loggerFactory = require('./providers/logging/logger') const routesVersioning = require('express-routes-versioning')() -const { createApiLimiter, createBatchApiLimiter } = require('./lib/rateLimit') +const { setupApiRateLimiterAfterCachingInit, setupBatchApiRateLimiterAfterCachingInit } = require('./lib/rateLimit') const v1 = '1.0.0' -function createApp(config, { logger, cachingService }) { +function createApp(config) { const initializers = [] + const logger = loggerFactory(config.logging.logger()) process.on('unhandledRejection', exception => logger.error('unhandledRejection', exception)) config.auth.service.permissionsSetup() @@ -45,6 +47,9 @@ function createApp(config, { logger, cachingService }) { const searchService = config.search.service() initializers.push(async () => searchService.initialize()) + const cachingService = config.caching.service() + initializers.push(async () => cachingService.initialize()) + const curationService = config.curation.service(null, curationStore, config.endpoints, cachingService, harvestStore) const curationQueue = config.curation.queue() @@ -127,7 +132,7 @@ function createApp(config, { logger, cachingService }) { app.set('trust-proxy', true) - app.use(createApiLimiter(config, cachingService).middleware) + app.use(setupApiRateLimiterAfterCachingInit(config, cachingService)) // Use a (potentially lower) different API limit // for batch API request @@ -135,7 +140,7 @@ function createApp(config, { logger, cachingService }) { // * POST /definitions // * POST /curations // * POST /notices - const batchApiLimiter = createBatchApiLimiter(config, cachingService).middleware + const batchApiLimiter = setupBatchApiRateLimiterAfterCachingInit(config, cachingService) app.post('/definitions', batchApiLimiter) app.post('/curations', batchApiLimiter) app.post('/notices', batchApiLimiter) diff --git a/bin/www b/bin/www index d64dc27fd..fee9cba45 100644 --- a/bin/www +++ b/bin/www @@ -3,45 +3,34 @@ // SPDX-License-Identifier: MIT const config = require('./config') -const loggerFactory = require('../providers/logging/logger') -const createApp = require('../app') +const app = require('../app')(config) const debug = require('debug')('service:server') const http = require('http') const init = require('express-init') -connect(config).then(resources => { - const app = createApp(config, resources) - - /** - * Get port from environment and store in Express. - */ - const port = normalizePort(process.env.PORT || '4000') - app.set('port', port) - - /** - * Create HTTP server. - */ - const server = http.createServer(app) +/** + * Get port from environment and store in Express. + */ +const port = normalizePort(process.env.PORT || '4000') +app.set('port', port) - /** - * Initialize the apps (if they have async init functions) and start listening - */ - init(app, error => { - if (error) { - console.log('Error initializing the Express app: ' + error) - throw new Error(error) - } - server.listen(port) - server.on('error', onError) - server.on('listening', () => onListening(server)) - console.log(`Service listening on port: ${port}`) - }) +/** + * Create HTTP server. + */ +const server = http.createServer(app) - /** - * Handles graceful shutdown - */ - process.on('SIGINT', () => onShutdown(server, resources)) - process.on('SIGTERM', () => onShutdown(server, resources)) +/** + * Initialize the apps (if they have async init functions) and start listening + */ +init(app, error => { + if (error) { + console.log('Error initializing the Express app: ' + error) + throw new Error(error) + } + server.listen(port) + server.on('error', onError) + server.on('listening', onListening) + console.log(`Service listening on port: ${port}`) }) /** @@ -80,47 +69,8 @@ function onError(error) { /** * Event listener for HTTP server "listening" event. */ -function onListening(server) { +function onListening() { const addr = server.address() const bind = typeof addr === 'string' ? 'pipe ' + addr : 'port ' + addr.port debug('Listening on ' + bind) } - -/** - * - * Handles gradelful shutdown - * - * @param server - The server instance to be shut down. - * @param {Object} resources - The resources to be cleaned up during shutdown. - */ -function onShutdown(server, { cachingService, logger }) { - logger.info('Shutdown started') - server.close(() => { - logger.info('Server closed') - Promise.allSettled([cachingService.done()]).then(results => { - const errorResults = results.filter(result => result.status === 'rejected') - if (errorResults.length === 0) { - logger.info('Shutdown complete') - process.exit(0) - } - errorResults.forEach(errorResult => logger.error(errorResult.reason)) - process.exit(1) - }) - }) -} - -/** - * Connect to the required services and return the resources - * - * @param {Object} config - The configuration object - * @returns {Promise} - The resources object - */ -async function connect(config) { - const logger = loggerFactory(config.logging.logger()) - const cachingService = config.caching.service() - await Promise.all([cachingService.initialize()]).catch(error => { - logger.error('Error connecting to cachingService', error) - process.exit(1) - }) - return { cachingService, logger } -} diff --git a/lib/rateLimit.js b/lib/rateLimit.js index 888f89d28..0447b6cd3 100644 --- a/lib/rateLimit.js +++ b/lib/rateLimit.js @@ -87,9 +87,56 @@ function createBatchApiLimiter(config, cachingService, logger) { return createRateLimiter(buildBatchOpts(config.limits, cachingService, 'batch-api', logger)) } +class AbstractMiddlewareDelegate { + constructor(opts) { + this.options = opts + this.logger = opts.logger || logger() + } + + get middleware() { + return (request, response, next) => { + this._createMiddleware().then(middleware => middleware(request, response, next)) + } + } + + async _createMiddleware() { + if (!this._innerMiddleware) { + this.logger.debug('Creating inner middleware') + this._innerMiddleware = await this.createInnerMiddleware() + } + return this._innerMiddleware + } + + async createInnerMiddleware() { + throw new Error('Not implemented') + } +} + +class ApiMiddlewareDelegate extends AbstractMiddlewareDelegate { + async createInnerMiddleware() { + await this.options.cachingService.initialize() + return createApiLimiter(this.options.config, this.options.cachingService).middleware + } +} + +class BatchApiMiddlewareDelegate extends AbstractMiddlewareDelegate { + async createInnerMiddleware() { + await this.options.cachingService.initialize() + return createBatchApiLimiter(this.options.config, this.options.cachingService).middleware + } +} + +const setupApiRateLimiterAfterCachingInit = (config, cachingService) => + new ApiMiddlewareDelegate({ config, cachingService }).middleware + +const setupBatchApiRateLimiterAfterCachingInit = (config, cachingService) => + new BatchApiMiddlewareDelegate({ config, cachingService }).middleware + module.exports = { createApiLimiter, createBatchApiLimiter, RedisBasedRateLimiter, - RateLimiter + RateLimiter, + setupApiRateLimiterAfterCachingInit, + setupBatchApiRateLimiterAfterCachingInit } diff --git a/providers/caching/redis.js b/providers/caching/redis.js index aa72a8861..fcb59d46b 100644 --- a/providers/caching/redis.js +++ b/providers/caching/redis.js @@ -13,8 +13,15 @@ class RedisCache { this.logger = options.logger || logger() } - async initialize() { - this._client = await RedisCache.initializeClient(this.options, this.logger) + // Initialize the Redis client and return a promise + initialize() { + if (this._client) return Promise.resolve() + if (!this._clientReady) { + this._clientReady = RedisCache.initializeClient(this.options, this.logger).then(client => { + this._client = client + }) + } + return this._clientReady } async done() { diff --git a/test/providers/caching/redis.js b/test/providers/caching/redis.js index 238331988..3de7c458d 100644 --- a/test/providers/caching/redis.js +++ b/test/providers/caching/redis.js @@ -78,6 +78,12 @@ describe('Redis Cache', () => { assert.strictEqual(error.message, 'Connection failed') } }) + + it('initalizes client only once', async () => { + const cache = redisCache({ logger }) + await Promise.all([cache.initialize(), cache.initialize()]) + assert.ok(RedisCache.buildRedisClient.calledOnce) + }) }) describe('Integration Test', () => { From ea769d690fe6da8b58cd0ed367de1aded98f76ca Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Thu, 13 Mar 2025 18:24:43 -0700 Subject: [PATCH 19/22] Address review comments --- test/business/definitionServiceTest.js | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/business/definitionServiceTest.js b/test/business/definitionServiceTest.js index 29ed98a05..f688427a6 100644 --- a/test/business/definitionServiceTest.js +++ b/test/business/definitionServiceTest.js @@ -344,15 +344,13 @@ describe('Integration test', () => { describe('Handle schema version upgrade', () => { const coordinates = EntityCoordinates.fromString('npm/npmjs/-/test/1.0') const definition = { _meta: { schemaVersion: '1.7.0' }, coordinates } + const logger = { + debug: () => {}, + error: () => {}, + info: () => {} + } - let logger, upgradeHandler - beforeEach(() => { - logger = { - debug: () => {}, - error: () => {}, - info: () => {} - } - }) + let upgradeHandler const handleVersionedDefinition = function () { describe('verify schema version', () => { From 7edecf196534928bd751e86c723f76f017713edf Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Thu, 13 Mar 2025 20:22:39 -0700 Subject: [PATCH 20/22] Add default values and error handling when constructing rate-limiting middleware Restored the integration test code for app. The test itself is still disabled because createApp starts recursive timers, which interfere with other unit tests. --- lib/rateLimit.js | 38 +++++++++++++++++++++++--------------- test/app.js | 16 +--------------- test/lib/rateLimit.js | 33 ++++++++++++++++++++------------- 3 files changed, 44 insertions(+), 43 deletions(-) diff --git a/lib/rateLimit.js b/lib/rateLimit.js index 0447b6cd3..c6df98dc8 100644 --- a/lib/rateLimit.js +++ b/lib/rateLimit.js @@ -69,22 +69,22 @@ function createRateLimiter(opts) { return opts.redis ? new RedisBasedRateLimiter(opts) : new RateLimiter(opts) } -function buildOpts({ windowSeconds, max }, cachingService, prefix, logger) { +function buildOpts({ windowSeconds = 0, max = 0 } = {}, cachingService, prefix, logger) { const limit = { windowMs: windowSeconds * 1000, max } const redis = cachingService instanceof RedisCache ? { client: cachingService.client, prefix } : undefined return { limit, redis, logger } } -function buildBatchOpts({ batchWindowSeconds, batchMax }, ...args) { +function buildBatchOpts({ batchWindowSeconds, batchMax } = {}, ...args) { return buildOpts({ windowSeconds: batchWindowSeconds, max: batchMax }, ...args) } -function createApiLimiter(config, cachingService, logger) { - return createRateLimiter(buildOpts(config.limits, cachingService, 'api', logger)) +function createApiLimiter({ config, cachingService, logger } = {}) { + return createRateLimiter(buildOpts(config?.limits, cachingService, 'api', logger)) } -function createBatchApiLimiter(config, cachingService, logger) { - return createRateLimiter(buildBatchOpts(config.limits, cachingService, 'batch-api', logger)) +function createBatchApiLimiter({ config, cachingService, logger } = {}) { + return createRateLimiter(buildBatchOpts(config?.limits, cachingService, 'batch-api', logger)) } class AbstractMiddlewareDelegate { @@ -95,14 +95,20 @@ class AbstractMiddlewareDelegate { get middleware() { return (request, response, next) => { - this._createMiddleware().then(middleware => middleware(request, response, next)) + this._createMiddleware() + .catch(error => next(error)) + .then(middleware => middleware(request, response, next)) } } async _createMiddleware() { if (!this._innerMiddleware) { - this.logger.debug('Creating inner middleware') - this._innerMiddleware = await this.createInnerMiddleware() + try { + this._innerMiddleware = await this.createInnerMiddleware() + } catch (error) { + this.logger.error('Error creating inner middleware: %s', error) + throw error + } } return this._innerMiddleware } @@ -114,23 +120,25 @@ class AbstractMiddlewareDelegate { class ApiMiddlewareDelegate extends AbstractMiddlewareDelegate { async createInnerMiddleware() { + this.logger.debug('Creating api rate-limiting middleware') await this.options.cachingService.initialize() - return createApiLimiter(this.options.config, this.options.cachingService).middleware + return createApiLimiter(this.options).middleware } } class BatchApiMiddlewareDelegate extends AbstractMiddlewareDelegate { async createInnerMiddleware() { + this.logger.debug('Creating batch api rate-limiting middleware') await this.options.cachingService.initialize() - return createBatchApiLimiter(this.options.config, this.options.cachingService).middleware + return createBatchApiLimiter(this.options).middleware } } -const setupApiRateLimiterAfterCachingInit = (config, cachingService) => - new ApiMiddlewareDelegate({ config, cachingService }).middleware +const setupApiRateLimiterAfterCachingInit = (config, cachingService, logger) => + new ApiMiddlewareDelegate({ config, cachingService, logger }).middleware -const setupBatchApiRateLimiterAfterCachingInit = (config, cachingService) => - new BatchApiMiddlewareDelegate({ config, cachingService }).middleware +const setupBatchApiRateLimiterAfterCachingInit = (config, cachingService, logger) => + new BatchApiMiddlewareDelegate({ config, cachingService, logger }).middleware module.exports = { createApiLimiter, diff --git a/test/app.js b/test/app.js index a51ecd345..fd0879d35 100644 --- a/test/app.js +++ b/test/app.js @@ -6,7 +6,6 @@ process.env['CURATION_GITHUB_TOKEN'] = '123' process.env['GITLAB_TOKEN'] = 'abc' const init = require('express-init') const Application = require('../app') -const logger = require('../providers/logging/logger') const config = proxyquire('../bin/config', { ['painless-config']: { get: name => { @@ -20,22 +19,9 @@ const config = proxyquire('../bin/config', { } }) -const mockLogger = { - info: () => {}, - error: () => {}, - debug: () => {} -} - describe('Application', () => { - beforeEach(async () => { - logger(mockLogger) - }) - it.skip('should initialize', done => { - const app = Application(config, { - logger: mockLogger, - cachingService: {} - }) + const app = Application(config) init(app, error => { if (error) { done(error) diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js index fdc7da553..4ba5a35dd 100644 --- a/test/lib/rateLimit.js +++ b/test/lib/rateLimit.js @@ -2,12 +2,7 @@ // SPDX-License-Identifier: MIT const assert = require('assert') -const { - RateLimiter, - RedisBasedRateLimiter, - createApiLimiter, - createBatchApiLimiter -} = require('../../lib/rateLimit.js') +const { RateLimiter, RedisBasedRateLimiter, createApiLimiter, createBatchApiLimiter } = require('../../lib/rateLimit') const supertest = require('supertest') const express = require('express') const sinon = require('sinon') @@ -134,8 +129,9 @@ describe('Rate Limiter', () => { } describe('Create Rate Limiter without Caching', () => { + const options = { config: { limits }, cachingService: undefined, logger } it('builds a rate limiter', () => { - const rateLimiter = createApiLimiter({ limits }, undefined, logger) + const rateLimiter = createApiLimiter(options) assert.ok(rateLimiter instanceof RateLimiter) const expected = { limit: { @@ -149,7 +145,7 @@ describe('Rate Limiter', () => { }) it('builds a batch rate limiter', () => { - const batchRateLimiter = createBatchApiLimiter({ limits }, undefined, logger) + const batchRateLimiter = createBatchApiLimiter(options) assert.ok(batchRateLimiter instanceof RateLimiter) const expected = { limit: { @@ -161,13 +157,24 @@ describe('Rate Limiter', () => { } assert.deepStrictEqual(batchRateLimiter.options, expected) }) + + it('builds a api rate limiter with default', () => { + const batchRateLimiter = createApiLimiter() + assert.ok(batchRateLimiter instanceof RateLimiter) + }) + + it('builds a batch rate limiter with default', () => { + const batchRateLimiter = createBatchApiLimiter() + assert.ok(batchRateLimiter instanceof RateLimiter) + }) }) describe('Create Rate Limiter with Caching', () => { - let caching + let options beforeEach(() => { - caching = new RedisCache({ logger }) - sinon.stub(caching, 'client').value({}) + const cachingService = new RedisCache({ logger }) + sinon.stub(cachingService, 'client').value({}) + options = { config: { limits }, cachingService, logger } }) afterEach(() => { @@ -175,7 +182,7 @@ describe('Rate Limiter', () => { }) it('builds a redis based rate limiter', () => { - const rateLimiter = createApiLimiter({ limits }, caching, logger) + const rateLimiter = createApiLimiter(options) assert.ok(rateLimiter instanceof RedisBasedRateLimiter) const expected = { limit: { @@ -192,7 +199,7 @@ describe('Rate Limiter', () => { }) it('builds a redis based batch rate limiter', () => { - const batchRateLimiter = createBatchApiLimiter({ limits }, caching, logger) + const batchRateLimiter = createBatchApiLimiter(options) assert.ok(batchRateLimiter instanceof RedisBasedRateLimiter) const expected = { limit: { From 5777bb0e2edfee7296074509f39955ac5be26cae Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Fri, 14 Mar 2025 12:15:09 -0700 Subject: [PATCH 21/22] Add more tests --- providers/caching/redis.js | 4 +- test/lib/rateLimit.js | 187 ++++++++++++++++++++------------ test/providers/caching/redis.js | 10 +- 3 files changed, 131 insertions(+), 70 deletions(-) diff --git a/providers/caching/redis.js b/providers/caching/redis.js index fcb59d46b..4a3c13faf 100644 --- a/providers/caching/redis.js +++ b/providers/caching/redis.js @@ -25,7 +25,9 @@ class RedisCache { } async done() { - return this._client?.quit() + const client = this._client + this._client = null + return client?.quit() } get client() { diff --git a/test/lib/rateLimit.js b/test/lib/rateLimit.js index 4ba5a35dd..7211ef4f8 100644 --- a/test/lib/rateLimit.js +++ b/test/lib/rateLimit.js @@ -2,11 +2,19 @@ // SPDX-License-Identifier: MIT const assert = require('assert') -const { RateLimiter, RedisBasedRateLimiter, createApiLimiter, createBatchApiLimiter } = require('../../lib/rateLimit') +const { + RateLimiter, + RedisBasedRateLimiter, + createApiLimiter, + createBatchApiLimiter, + setupApiRateLimiterAfterCachingInit, + setupBatchApiRateLimiterAfterCachingInit +} = require('../../lib/rateLimit') const supertest = require('supertest') const express = require('express') const sinon = require('sinon') const { RedisCache } = require('../../providers/caching/redis') +const MemoryCache = require('../../providers/caching/memory') const { GenericContainer } = require('testcontainers') const logger = { @@ -18,30 +26,6 @@ const logger = { const limit = { windowMs: 1000, max: 1 } describe('Rate Limiter', () => { - describe('Rate Limit Integration Tests', () => { - let client - - beforeEach(async () => { - const rateLimiter = new RateLimiter({ limit, logger }) - const app = await buildApp(rateLimiter) - client = supertest(app) - }) - - it('allows requests under the limit', async () => { - await client - .get('/') - .expect(200) - .expect('Hello World!') - .expect('RateLimit-Limit', '1') - .expect('RateLimit-Remaining', '0') - }) - - it('blocks requests over the limit', async () => { - const counter = await tryBeyondLimit(limit.max, client) - assert.strictEqual(counter, limit.max, `Counter is ${counter}`) - }) - }) - describe('Build Limit Options', () => { it('builds rate limit options', () => { const options = RateLimiter.buildOptions(limit) @@ -83,41 +67,6 @@ describe('Rate Limiter', () => { assert.strictEqual(error.message, 'Redis client is missing') } }) - - describe('Rate Limit Integration Tests', () => { - let container, redisClient, client - - before(async function () { - this.timeout(15000) - ;({ container, redisClient } = await setupRedis()) - const rateLimiter = new RedisBasedRateLimiter({ limit, redis: { client: redisClient }, logger }) - const app = await buildApp(rateLimiter) - client = supertest(app) - }) - - after(async () => { - await redisClient?.quit() - await container?.stop() - }) - - afterEach(async () => { - await new Promise(resolve => setTimeout(resolve, limit.windowMs)) - }) - - it('allows requests under the limit', async () => { - await client - .get('/') - .expect(200) - .expect('Hello World!') - .expect('RateLimit-Limit', '1') - .expect('RateLimit-Remaining', '0') - }) - - it('blocks requests over the limit', async () => { - const counter = await tryBeyondLimit(limit.max, client) - assert.strictEqual(counter, limit.max, `Counter is ${counter}`) - }) - }) }) describe('Create Rate Limiter', () => { @@ -128,7 +77,7 @@ describe('Rate Limiter', () => { batchMax: 10 } - describe('Create Rate Limiter without Caching', () => { + describe('Memory Based', () => { const options = { config: { limits }, cachingService: undefined, logger } it('builds a rate limiter', () => { const rateLimiter = createApiLimiter(options) @@ -169,7 +118,7 @@ describe('Rate Limiter', () => { }) }) - describe('Create Rate Limiter with Caching', () => { + describe('Redis Based', () => { let options beforeEach(() => { const cachingService = new RedisCache({ logger }) @@ -216,15 +165,117 @@ describe('Rate Limiter', () => { }) }) }) + + describe('Create MiddlewareDelegate', () => { + let cachingService + + beforeEach(() => { + cachingService = { initialize: sinon.stub().resolves() } + }) + + it('creates api rate limit middleware function', () => { + const middleware = setupApiRateLimiterAfterCachingInit({}, cachingService, logger) + assert.equal(typeof middleware, 'function') + assert.ok(cachingService.initialize.notCalled) + }) + + it('creates batch api rate limit middleware function', () => { + const middleware = setupBatchApiRateLimiterAfterCachingInit({}, cachingService, logger) + assert.equal(typeof middleware, 'function') + assert.ok(cachingService.initialize.notCalled) + }) + }) + + describe('Rate Limit Integration Test', () => { + let client + + afterEach(async () => { + await new Promise(resolve => setTimeout(resolve, limit.windowMs)) + }) + + const verifyRateLimiting = function () { + it('allows requests under the limit', async () => { + await client + ?.get('/') + .expect(200) + .expect('Hello World!') + .expect('RateLimit-Limit', '1') + .expect('RateLimit-Remaining', '0') + }) + + it('blocks requests over the limit', async () => { + const counter = await tryBeyondLimit(limit.max, client) + assert.strictEqual(counter, limit.max, `Counter is ${counter}`) + }) + } + + describe('Memory Based Rate Limiter', () => { + before(async () => { + const rateLimiter = new RateLimiter({ limit, logger }) + client = await buildTestAppClient(rateLimiter.middleware) + }) + + verifyRateLimiting() + }) + + describe('Redis Based Rate Limiter', () => { + let container, redisClient, redisOpts + + before(async function () { + this.timeout(15000) + ;({ redisOpts, container } = await startRedis()) + redisClient = await RedisCache.initializeClient(redisOpts, logger) + const rateLimiter = new RedisBasedRateLimiter({ limit, redis: { client: redisClient }, logger }) + client = await buildTestAppClient(rateLimiter.middleware) + }) + + after(async function () { + await redisClient.quit() + await container.stop() + }) + + verifyRateLimiting() + }) + + describe('MiddlewareDelegate - Redis Based', () => { + const config = { limits: { windowSeconds: 1, max: 1 } } + let container, cachingService, redisOpts + + before(async function () { + this.timeout(15000) + ;({ container, redisOpts } = await startRedis()) + cachingService = new RedisCache({ ...redisOpts, logger }) + const middleware = setupApiRateLimiterAfterCachingInit(config, cachingService, logger) + client = await buildTestAppClient(middleware) + }) + + after(async () => { + await cachingService.done() + await container.stop() + }) + + verifyRateLimiting() + }) + + describe('MiddlewareDelegate - Memory Based', () => { + const config = { limits: { windowSeconds: 1, max: 1 } } + + before(async function () { + const middleware = setupApiRateLimiterAfterCachingInit(config, MemoryCache(), logger) + client = await buildTestAppClient(middleware) + }) + + verifyRateLimiting() + }) + }) }) -async function setupRedis() { +async function startRedis() { const container = await new GenericContainer('redis').withExposedPorts(6379).start() const service = container.getHost() const port = container.getMappedPort(6379) const redisOpts = { service, port, tls: false } - const redisClient = await RedisCache.initializeClient(redisOpts, logger) - return { container, redisClient } + return { redisOpts, container } } async function tryBeyondLimit(max, client) { @@ -237,9 +288,9 @@ async function tryBeyondLimit(max, client) { return counter } -async function buildApp(rateLimiter) { +async function buildTestAppClient(rateLimiter) { const app = express() - app.use(rateLimiter.middleware) + app.use(rateLimiter) app.get('/', (req, res) => res.send('Hello World!')) - return app + return supertest(app) } diff --git a/test/providers/caching/redis.js b/test/providers/caching/redis.js index 3de7c458d..edbd75ca3 100644 --- a/test/providers/caching/redis.js +++ b/test/providers/caching/redis.js @@ -28,7 +28,8 @@ describe('Redis Cache', () => { store[key] = null }, connect: async () => Promise.resolve(mockClient), - on: () => {} + on: () => {}, + quit: sinon.stub().resolves() } sandbox.stub(RedisCache, 'buildRedisClient').returns(mockClient) }) @@ -84,6 +85,13 @@ describe('Redis Cache', () => { await Promise.all([cache.initialize(), cache.initialize()]) assert.ok(RedisCache.buildRedisClient.calledOnce) }) + + it('calls client.quit only once', async () => { + const cache = redisCache({ logger }) + await cache.initialize() + await Promise.all([cache.done(), cache.done()]) + assert.ok(mockClient.quit.calledOnce) + }) }) describe('Integration Test', () => { From f101ef3574966908869878678b66ea441fba42b0 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Fri, 14 Mar 2025 12:19:57 -0700 Subject: [PATCH 22/22] Move health check ahead of rate limiting and refactor --- app.js | 2 +- lib/rateLimit.js | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/app.js b/app.js index 91286c2f6..6eb9ecab0 100644 --- a/app.js +++ b/app.js @@ -131,6 +131,7 @@ function createApp(config) { app.use(config.auth.service.middleware()) app.set('trust-proxy', true) + app.use('/', require('./routes/index')(config.buildsha, config.appVersion)) app.use(setupApiRateLimiterAfterCachingInit(config, cachingService)) @@ -147,7 +148,6 @@ function createApp(config) { app.use(require('./middleware/querystring')) - app.use('/', require('./routes/index')(config.buildsha, config.appVersion)) app.use('/origins/github', require('./routes/originGitHub')()) app.use('/origins/crate', require('./routes/originCrate')()) const repoAccess = require('./lib/condaRepoAccess')() diff --git a/lib/rateLimit.js b/lib/rateLimit.js index c6df98dc8..5a9455abd 100644 --- a/lib/rateLimit.js +++ b/lib/rateLimit.js @@ -75,16 +75,14 @@ function buildOpts({ windowSeconds = 0, max = 0 } = {}, cachingService, prefix, return { limit, redis, logger } } -function buildBatchOpts({ batchWindowSeconds, batchMax } = {}, ...args) { - return buildOpts({ windowSeconds: batchWindowSeconds, max: batchMax }, ...args) -} - function createApiLimiter({ config, cachingService, logger } = {}) { return createRateLimiter(buildOpts(config?.limits, cachingService, 'api', logger)) } function createBatchApiLimiter({ config, cachingService, logger } = {}) { - return createRateLimiter(buildBatchOpts(config?.limits, cachingService, 'batch-api', logger)) + const { batchWindowSeconds, batchMax } = config?.limits || {} + const opts = buildOpts({ windowSeconds: batchWindowSeconds, max: batchMax }, cachingService, 'batch-api', logger) + return createRateLimiter(opts) } class AbstractMiddlewareDelegate { @@ -106,7 +104,7 @@ class AbstractMiddlewareDelegate { try { this._innerMiddleware = await this.createInnerMiddleware() } catch (error) { - this.logger.error('Error creating inner middleware: %s', error) + this.logger.error('Error creating inner middleware', error) throw error } }