diff --git a/src/cmap/connect.ts b/src/cmap/connect.ts index 544ec471a99..711335f3d1d 100644 --- a/src/cmap/connect.ts +++ b/src/cmap/connect.ts @@ -224,6 +224,7 @@ export interface HandshakeDocument extends Document { compression: string[]; saslSupportedMechs?: string; loadBalanced?: boolean; + backpressure: true; } /** @@ -241,6 +242,7 @@ export async function prepareHandshakeDocument( const handshakeDoc: HandshakeDocument = { [serverApi?.version || options.loadBalanced === true ? 'hello' : LEGACY_HELLO_COMMAND]: 1, + backpressure: true, helloOk: true, client: clientMetadata, compression: compressors diff --git a/src/cmap/connection.ts b/src/cmap/connection.ts index 9652e3a5e4f..2a3e5a3b9a0 100644 --- a/src/cmap/connection.ts +++ b/src/cmap/connection.ts @@ -582,6 +582,9 @@ export class Connection extends TypedEventEmitter { this.throwIfAborted(); } } catch (error) { + if (options.session != null && !(error instanceof MongoServerError)) { + updateSessionFromResponse(options.session, MongoDBResponse.empty); + } if (this.shouldEmitAndLogCommand) { this.emitAndLogCommand( this.monitorCommands, diff --git a/src/index.ts b/src/index.ts index 8f5c4cfa60e..ca66d31e201 100644 --- a/src/index.ts +++ b/src/index.ts @@ -87,6 +87,7 @@ export { MongoWriteConcernError, WriteConcernErrorResult } from './error'; +export { TokenBucket } from './token_bucket'; export { AbstractCursor, // Actual driver classes exported diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index b6ddbba2b66..f68ce5b1d9f 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -1,3 +1,5 @@ +import { setTimeout } from 'node:timers/promises'; + import { MIN_SUPPORTED_SNAPSHOT_READS_WIRE_VERSION } from '../cmap/wire_protocol/constants'; import { isRetryableReadError, @@ -10,6 +12,7 @@ import { MongoInvalidArgumentError, MongoNetworkError, MongoNotConnectedError, + MongoOperationTimeoutError, MongoRuntimeError, MongoServerError, MongoTransactionError, @@ -26,9 +29,15 @@ import { import type { Topology } from '../sdam/topology'; import type { ClientSession } from '../sessions'; import { TimeoutContext } from '../timeout'; -import { abortable, maxWireVersion, supportsRetryableWrites } from '../utils'; +import { RETRY_COST, TOKEN_REFRESH_RATE } from '../token_bucket'; +import { + abortable, + ExponentialBackoffProvider, + maxWireVersion, + supportsRetryableWrites +} from '../utils'; import { AggregateOperation } from './aggregate'; -import { AbstractOperation, Aspect } from './operation'; +import { AbstractOperation, Aspect, RetryContext } from './operation'; const MMAPv1_RETRY_WRITES_ERROR_CODE = MONGODB_ERROR_CODES.IllegalOperation; const MMAPv1_RETRY_WRITES_ERROR_MESSAGE = @@ -50,7 +59,7 @@ type ResultTypeFromOperation = ReturnType< * The expectation is that this function: * - Connects the MongoClient if it has not already been connected, see {@link autoConnect} * - Creates a session if none is provided and cleans up the session it creates - * - Tries an operation and retries under certain conditions, see {@link tryOperation} + * - Tries an operation and retries under certain conditions, see {@link executeOperationWithRetries} * * @typeParam T - The operation's type * @typeParam TResult - The type of the operation's result, calculated from T @@ -120,7 +129,7 @@ export async function executeOperation< }); try { - return await tryOperation(operation, { + return await executeOperationWithRetries(operation, { topology, timeoutContext, session, @@ -184,7 +193,10 @@ type RetryOptions = { * * @param operation - The operation to execute * */ -async function tryOperation>( +async function executeOperationWithRetries< + T extends AbstractOperation, + TResult = ResultTypeFromOperation +>( operation: T, { topology, timeoutContext, session, readPreference }: RetryOptions ): Promise { @@ -233,11 +245,27 @@ async function tryOperation timeoutContext.remainingTimeMS) { + // TODO: is this the right error to throw? + throw new MongoOperationTimeoutError( + `MongoDB SystemOverload exponential backoff would exceed timeoutMS deadline: remaining CSOT deadline=${timeoutContext.remainingTimeMS}, backoff delayMS=${delayMS}`, + { + cause: previousOperationError + } + ); + } + + if (!topology.tokenBucket.consume(RETRY_COST)) { + throw previousOperationError; + } + + await setTimeout(delayMS); } if ( @@ -285,19 +337,34 @@ async function tryOperation 0 and we are command batching we need to reset the batch. - if (tries > 0 && operation.hasAspect(Aspect.COMMAND_BATCHING)) { + const isRetry = attempt > 0; + + // If attempt > 0 and we are command batching we need to reset the batch. + if (isRetry && operation.hasAspect(Aspect.COMMAND_BATCHING)) { operation.resetBatch(); } try { const result = await server.command(operation, timeoutContext); + topology.tokenBucket.deposit( + isRetry + ? // on successful retry, deposit the retry cost + the refresh rate. + TOKEN_REFRESH_RATE + RETRY_COST + : // otherwise, just deposit the refresh rate. + TOKEN_REFRESH_RATE + ); return operation.handleOk(result); } catch (error) { return operation.handleError(error); } } catch (operationError) { if (!(operationError instanceof MongoError)) throw operationError; + + if (!operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { + // if an operation fails with an error that does not contain the SystemOverloadError, deposit 1 token. + topology.tokenBucket.deposit(RETRY_COST); + } + if ( previousOperationError != null && operationError.hasErrorLabel(MongoErrorLabel.NoWritesPerformed) @@ -312,8 +379,5 @@ async function tryOperation { /** Specifies the time an operation will run until it throws a timeout error. */ timeoutMS?: number; + retryContext?: RetryContext; + private _session: ClientSession | undefined; static aspects?: Set; diff --git a/src/sdam/topology.ts b/src/sdam/topology.ts index af25f87fe41..370c3d33308 100644 --- a/src/sdam/topology.ts +++ b/src/sdam/topology.ts @@ -35,6 +35,7 @@ import { type Abortable, TypedEventEmitter } from '../mongo_types'; import { ReadPreference, type ReadPreferenceLike } from '../read_preference'; import type { ClientSession } from '../sessions'; import { Timeout, TimeoutContext, TimeoutError } from '../timeout'; +import { TokenBucket } from '../token_bucket'; import type { Transaction } from '../transactions'; import { addAbortListener, @@ -207,18 +208,15 @@ export type TopologyEvents = { * @internal */ export class Topology extends TypedEventEmitter { - /** @internal */ s: TopologyPrivate; - /** @internal */ waitQueue: List; - /** @internal */ hello?: Document; - /** @internal */ _type?: string; + tokenBucket = new TokenBucket(1000); + client!: MongoClient; - /** @internal */ private connectionLock?: Promise; /** @event */ diff --git a/src/sessions.ts b/src/sessions.ts index e282c8617a0..9e740b90744 100644 --- a/src/sessions.ts +++ b/src/sessions.ts @@ -23,6 +23,7 @@ import { import type { MongoClient, MongoOptions } from './mongo_client'; import { TypedEventEmitter } from './mongo_types'; import { executeOperation } from './operations/execute_operation'; +import { RetryContext } from './operations/operation'; import { RunCommandOperation } from './operations/run_command'; import { ReadConcernLevel } from './read_concern'; import { ReadPreference } from './read_preference'; @@ -466,7 +467,11 @@ export class ClientSession } else { const wcKeys = Object.keys(wc); if (wcKeys.length > 2 || (!wcKeys.includes('wtimeoutMS') && !wcKeys.includes('wTimeoutMS'))) - // if the write concern was specified with wTimeoutMS, then we set both wtimeoutMS and wTimeoutMS, guaranteeing at least two keys, so if we have more than two keys, then we can automatically assume that we should add the write concern to the command. If it has 2 or fewer keys, we need to check that those keys aren't the wtimeoutMS or wTimeoutMS options before we add the write concern to the command + // if the write concern was specified with wTimeoutMS, then we set both wtimeoutMS + // and wTimeoutMS, guaranteeing at least two keys, so if we have more than two keys, + // then we can automatically assume that we should add the write concern to the command. + // If it has 2 or fewer keys, we need to check that those keys aren't the wtimeoutMS + // or wTimeoutMS options before we add the write concern to the command WriteConcern.apply(command, { ...wc, wtimeoutMS: undefined }); } } @@ -487,11 +492,14 @@ export class ClientSession command.recoveryToken = this.transaction.recoveryToken; } + const retryContext = new RetryContext(5); + const operation = new RunCommandOperation(new MongoDBNamespace('admin'), command, { session: this, readPreference: ReadPreference.primary, bypassPinningCheck: true }); + operation.retryContext = retryContext; const timeoutContext = this.timeoutContext ?? @@ -516,15 +524,13 @@ export class ClientSession this.unpin({ force: true }); try { - await executeOperation( - this.client, - new RunCommandOperation(new MongoDBNamespace('admin'), command, { - session: this, - readPreference: ReadPreference.primary, - bypassPinningCheck: true - }), - timeoutContext - ); + const op = new RunCommandOperation(new MongoDBNamespace('admin'), command, { + session: this, + readPreference: ReadPreference.primary, + bypassPinningCheck: true + }); + op.retryContext = retryContext; + await executeOperation(this.client, op, timeoutContext); return; } catch (retryCommitError) { // If the retry failed, we process that error instead of the original @@ -957,6 +963,11 @@ export class ServerSession { id: ServerSessionId; lastUse: number; txnNumber: number; + + /* + * Indicates that a network error has been encountered while using this session. + * Once a session is marked as dirty, it is always dirty. + */ isDirty: boolean; /** @internal */ @@ -1050,16 +1061,15 @@ export class ServerSessionPool { * @param session - The session to release to the pool */ release(session: ServerSession): void { - const sessionTimeoutMinutes = this.client.topology?.logicalSessionTimeoutMinutes ?? 10; + if (this.client.topology?.loadBalanced) { + if (session.isDirty) return; - if (this.client.topology?.loadBalanced && !sessionTimeoutMinutes) { this.sessions.unshift(session); - } - - if (!sessionTimeoutMinutes) { return; } + const sessionTimeoutMinutes = this.client.topology?.logicalSessionTimeoutMinutes ?? 10; + this.sessions.prune(session => session.hasTimedOut(sessionTimeoutMinutes)); if (!session.hasTimedOut(sessionTimeoutMinutes)) { @@ -1147,9 +1157,9 @@ export function applySession( command.autocommit = false; if (session.transaction.state === TxnState.STARTING_TRANSACTION) { - session.transaction.transition(TxnState.TRANSACTION_IN_PROGRESS); command.startTransaction = true; + // TODO: read concern only applied if it is not the same as the server's default const readConcern = session.transaction.options.readConcern || session?.clientOptions?.readConcern; if (readConcern) { @@ -1185,4 +1195,17 @@ export function updateSessionFromResponse(session: ClientSession, document: Mong session.snapshotTime = atClusterTime; } } + + if (session.transaction.state === TxnState.STARTING_TRANSACTION) { + if (document.ok === 1) { + session.transaction.transition(TxnState.TRANSACTION_IN_PROGRESS); + } else { + const error = new MongoServerError(document.toObject()); + const isBackpressureError = error.hasErrorLabel(MongoErrorLabel.RetryableError); + + if (!isBackpressureError) { + session.transaction.transition(TxnState.TRANSACTION_IN_PROGRESS); + } + } + } } diff --git a/src/token_bucket.ts b/src/token_bucket.ts new file mode 100644 index 00000000000..9d542bf719d --- /dev/null +++ b/src/token_bucket.ts @@ -0,0 +1,23 @@ +/** + * @internal + */ +export class TokenBucket { + private budget: number; + constructor(allowance: number) { + this.budget = allowance; + } + deposit(tokens: number) { + this.budget += tokens; + } + + consume(tokens: number): boolean { + if (tokens > this.budget) return false; + + this.budget -= tokens; + return true; + } +} + +export const TOKEN_REFRESH_RATE = 0.1; +export const INITIAL_SIZE = 1000; +export const RETRY_COST = 1; diff --git a/src/utils.ts b/src/utils.ts index c0f0864dbf8..6b7b4a0c468 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1431,3 +1431,19 @@ export async function abortable( abortListener?.[kDispose](); } } + +export class ExponentialBackoffProvider { + constructor( + public readonly maxBackoff: number, + public readonly baseBackoff: number, + public readonly backoffIncreaseRate: number, + public iteration = 0 + ) {} + + getNextBackoffDuration(): number { + return ( + Math.random() * + Math.min(this.maxBackoff, this.baseBackoff * this.backoffIncreaseRate ** this.iteration) + ); + } +} diff --git a/sync.sh b/sync.sh new file mode 100644 index 00000000000..b99d9ed1086 --- /dev/null +++ b/sync.sh @@ -0,0 +1,4 @@ + + +cp ~/dev/specifications/source/client-backpressure/tests/* ~/dev/node-mongodb-native/test/spec/client-backpressure +cp ~/dev/specifications/source/transactions/tests/unified/backpressure* ~/dev/node-mongodb-native/test/spec/transactions/unified/ \ No newline at end of file diff --git a/test/integration/client-backpressure/client-backpressure.prose.test.ts b/test/integration/client-backpressure/client-backpressure.prose.test.ts new file mode 100644 index 00000000000..56d274fffe1 --- /dev/null +++ b/test/integration/client-backpressure/client-backpressure.prose.test.ts @@ -0,0 +1,60 @@ +import { expect } from 'chai'; +import * as sinon from 'sinon'; + +import { type Collection, type MongoClient, MongoServerError } from '../../../src'; +import { clearFailPoint, configureFailPoint, measureDuration } from '../../tools/utils'; + +describe('Client Backpressure (Prose)', function () { + let client: MongoClient; + let collection: Collection; + + beforeEach(async function () { + client = this.configuration.newClient(); + await client.connect(); + + collection = client.db('foo').collection('bar'); + }); + + afterEach(async function () { + await client.close(); + await clearFailPoint(this.configuration); + }); + + it( + 'Test 1: Operation Retry Uses Exponential Backoff', + { + requires: { + mongodb: '4.4' + } + }, + async function () { + await configureFailPoint(this.configuration, { + configureFailPoint: 'failCommand', + mode: 'alwaysOn', + data: { + failCommands: ['insert'], + errorCode: 2, + errorLabels: ['SystemOverloadedError', 'RetryableError'] + } + }); + + const stub = sinon.stub(Math, 'random'); + + stub.returns(0); + + const { duration: durationNoBackoff } = await measureDuration(async () => { + const error = await collection.insertOne({ a: 1 }).catch(e => e); + expect(error).to.be.instanceof(MongoServerError); + }); + + stub.returns(1); + + const { duration: durationBackoff } = await measureDuration(async () => { + const error = await collection.insertOne({ a: 1 }).catch(e => e); + expect(error).to.be.instanceof(MongoServerError); + }); + + expect(durationBackoff - durationNoBackoff).to.be.within(3100 - 1000, 3100 + 1000); + } + ); +}); diff --git a/test/integration/client-backpressure/client-backpressure.spec.test.ts b/test/integration/client-backpressure/client-backpressure.spec.test.ts new file mode 100644 index 00000000000..9485c46a96c --- /dev/null +++ b/test/integration/client-backpressure/client-backpressure.spec.test.ts @@ -0,0 +1,16 @@ +import { loadSpecTests } from '../../spec'; +import { runUnifiedSuite } from '../../tools/unified-spec-runner/runner'; +import { type Test } from '../../tools/unified-spec-runner/schema'; + +const skippedTests = { + 'collection.dropIndexes retries at most maxAttempts=5 times': + 'TODO(NODE-6517): dropIndexes squashes all errors other than ns not found' +}; + +function shouldSkip({ description }: Test) { + return skippedTests[description] ?? false; +} + +describe('Client Backpressure (spec)', function () { + runUnifiedSuite(loadSpecTests('client-backpressure'), shouldSkip); +}); diff --git a/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts b/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts index a8ac32b99cf..dd04c8d1826 100644 --- a/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts +++ b/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts @@ -1,7 +1,14 @@ import { expect } from 'chai'; import * as sinon from 'sinon'; -import { type ClientMetadata, type DriverInfo, Int32, MongoClient } from '../../../src'; +import { + type ClientMetadata, + type Document, + type DriverInfo, + type HandshakeDocument, + Int32, + MongoClient +} from '../../../src'; import { Connection } from '../../../src/cmap/connection'; import { getFAASEnv, isDriverInfoEqual } from '../../../src/cmap/handshake/client_metadata'; import { LEGACY_HELLO_COMMAND } from '../../../src/constants'; @@ -939,3 +946,38 @@ describe('Client Metadata Update Prose Tests', function () { } }); }); + +// TODO: add prose test descriptions here to align the test with the spec. +describe('Backpressure Metadata', function () { + let client: MongoClient; + let spy: sinon.SinonSpy>; + + beforeEach(async function () { + client = this.configuration.newClient(); + spy = sinon.spy(Connection.prototype, 'command'); + await client.connect(); + + // run an operation to force a connection establishment, + // if we're testing noauth load balanced mode. + await client.db('foo').collection('bar').insertOne({ name: 'bumpy' }); + }); + + afterEach(async function () { + sinon.restore(); + await client?.close(); + }); + + it('includes backpressure in the handshake document', function () { + const isHello = (cmd: Document): cmd is HandshakeDocument => + `hello` in cmd || LEGACY_HELLO_COMMAND in cmd; + + const hellos = spy.args.map(([_ns, command, _options]) => command).filter(isHello); + + expect(hellos.length).to.be.greaterThan(0); + + expect( + hellos.every(hello => hello.backpressure === true), + `some handshake documents did not specify backpressure: true` + ); + }); +}); diff --git a/test/spec/client-backpressure/README.md b/test/spec/client-backpressure/README.md new file mode 100644 index 00000000000..9f49b1ed203 --- /dev/null +++ b/test/spec/client-backpressure/README.md @@ -0,0 +1,57 @@ +# Client Backpressure Tests + +______________________________________________________________________ + +## Introduction + +The YAML and JSON files in this directory are platform-independent tests meant to exercise a driver's implementation of +retryable reads. These tests utilize the [Unified Test Format](../../unified-test-format/unified-test-format.md). + +Several prose tests, which are not easily expressed in YAML, are also presented in this file. Those tests will need to +be manually implemented by each driver. + +### Prose Tests + +#### Test 1: Operation Retry Uses Exponential Backoff + +Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. + +1. Let `client` be a `MongoClient` +2. Let `collection` be a collection +3. Now, run transactions without backoff: + 1. Configure the random number generator used for jitter to always return `0` -- this effectively disables backoff. + + 2. Configure the following failPoint: + + ```javascript + { + configureFailPoint: 'failCommand', + mode: 'alwaysOn', + data: { + failCommands: ['insert'], + errorCode: 2, + errorLabels: ['SystemOverloadedError', 'RetryableError'] + } + } + ``` + + 3. Insert the document `{ a: 1 }`. Expect that the command errors. Measure the duration of the command execution. + + ```javascript + const start = performance.now(); + expect( + await coll.insertOne({ a: 1 }).catch(e => e) + ).to.be.an.instanceof(MongoServerError); + const end = performance.now(); + ``` + + 4. Configure the random number generator used for jitter to always return `1`. + + 5. Execute step 3 again. + + 6. Compare the two time between the two runs. + ```python + assertTrue(with_backoff_time - no_backoff_time >= 2.1) + ``` + The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between the two + runs. diff --git a/test/spec/client-backpressure/backpressure-retry-loop.json b/test/spec/client-backpressure/backpressure-retry-loop.json new file mode 100644 index 00000000000..2542344b382 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-loop.json @@ -0,0 +1,2759 @@ +{ + "description": "tests that operations respect overload backoff retry loop", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "client": { + "id": "internal_client", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "internal_db", + "client": "internal_client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "retryable-writes-tests", + "database": "internal_db", + "collectionName": "coll" + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [] + } + ], + "_yamlAnchors": { + "bulWriteInsertNamespace": "retryable-writes-tests.coll" + }, + "tests": [ + { + "description": "client.listDatabases retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabases", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.listDatabaseNames retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabaseNames" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.createChangeStream retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite retries using operation loop", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandSucceededEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "database.aggregate retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "database.listCollections retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollections", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.listCollectionNames retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollectionNames", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.runCommand retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandSucceededEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.createChangeStream retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "aggregate", + "arguments": { + "pipeline": [] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "countDocuments", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "estimatedDocumentCount" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandSucceededEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.distinct retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "distinct", + "arguments": { + "fieldName": "x", + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandSucceededEvent": { + "commandName": "distinct" + } + } + ] + } + ] + }, + { + "description": "collection.find retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.findOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOne", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexes retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexes" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexNames retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexNames" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.createChangeStream retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.insertOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.insertMany retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.deleteOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteOne", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.deleteMany retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.replaceOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateOne", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateMany retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateMany", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndDelete", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndReplace", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndUpdate", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.bulkWrite retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.createIndex retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "createIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndex retries using operation loop", + "operations": [ + { + "object": "retryable-writes-tests", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndex", + "arguments": { + "name": "x_11" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndexes retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndexes" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/client-backpressure/backpressure-retry-loop.yml b/test/spec/client-backpressure/backpressure-retry-loop.yml new file mode 100644 index 00000000000..6a3033989b4 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-loop.yml @@ -0,0 +1,1444 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations respect overload backoff retry loop + +schemaVersion: '1.3' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + + - + client: + id: &internal_client internal_client + useMultipleMongoses: false + + - + database: + id: &internal_db internal_db + client: *internal_client + databaseName: &database_name retryable-writes-tests + + - + collection: + id: &internal_collection retryable-writes-tests + database: *internal_db + collectionName: &collection_name coll + + - + database: + id: &database database + client: *client + databaseName: *database_name + + - + collection: + id: &collection collection + database: *database + collectionName: *collection_name + +initialData: +- collectionName: *collection_name + databaseName: *database_name + documents: [] + +_yamlAnchors: + bulWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll + +tests: + + - + description: 'client.listDatabases retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listDatabases] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: listDatabases + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandSucceededEvent: + commandName: listDatabases + + - + description: 'client.listDatabaseNames retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listDatabases] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: listDatabaseNames + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandSucceededEvent: + commandName: listDatabases + + - + description: 'client.createChangeStream retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: createChangeStream + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'client.clientBulkWrite retries using operation loop' + runOnRequirements: + - minServerVersion: '8.0' # client bulk write added to server in 8.0 + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [bulkWrite] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: *client_bulk_write_ns + document: { _id: 8, x: 88 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandSucceededEvent: + commandName: bulkWrite + + - + description: 'database.aggregate retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: aggregate + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'database.listCollections retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listCollections] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: listCollections + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandSucceededEvent: + commandName: listCollections + + - + description: 'database.listCollectionNames retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listCollections] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: listCollectionNames + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandSucceededEvent: + commandName: listCollections + + - + description: 'database.runCommand retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [ping] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: runCommand + arguments: + command: { ping: 1 } + commandName: ping + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandSucceededEvent: + commandName: ping + + - + description: 'database.createChangeStream retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: createChangeStream + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.aggregate retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: aggregate + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.countDocuments retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: countDocuments + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.estimatedDocumentCount retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [count] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: estimatedDocumentCount + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandSucceededEvent: + commandName: count + + - + description: 'collection.distinct retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [distinct] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: distinct + arguments: + fieldName: x + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandSucceededEvent: + commandName: distinct + + - + description: 'collection.find retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [find] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: find + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + + - + description: 'collection.findOne retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [find] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOne + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + + - + description: 'collection.listIndexes retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: listIndexes + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandSucceededEvent: + commandName: listIndexes + + - + description: 'collection.listIndexNames retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: listIndexNames + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandSucceededEvent: + commandName: listIndexes + + - + description: 'collection.createChangeStream retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: createChangeStream + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.insertOne retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: insertOne + arguments: + document: { _id: 2, x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.insertMany retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: insertMany + arguments: + documents: + - { _id: 2, x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.deleteOne retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [delete] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: deleteOne + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandSucceededEvent: + commandName: delete + + - + description: 'collection.deleteMany retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [delete] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: deleteMany + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandSucceededEvent: + commandName: delete + + - + description: 'collection.replaceOne retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: replaceOne + arguments: + filter: {} + replacement: { x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.updateOne retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: updateOne + arguments: + filter: {} + update: { $set: { x: 22 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.updateMany retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: updateMany + arguments: + filter: {} + update: { $set: { x: 22 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.findOneAndDelete retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOneAndDelete + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.findOneAndReplace retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOneAndReplace + arguments: + filter: {} + replacement: { x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.findOneAndUpdate retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOneAndUpdate + arguments: + filter: {} + update: { $set: { x: 22 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.bulkWrite retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: bulkWrite + arguments: + requests: + - insertOne: + document: { _id: 2, x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.createIndex retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [createIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandSucceededEvent: + commandName: createIndexes + + - + description: 'collection.dropIndex retries using operation loop' + operations: + - + object: *internal_collection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [dropIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: dropIndex + arguments: + name: "x_11" + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandSucceededEvent: + commandName: dropIndexes + + - + description: 'collection.dropIndexes retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [dropIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: dropIndexes + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandSucceededEvent: + commandName: dropIndexes diff --git a/test/spec/client-backpressure/backpressure-retry-loop.yml.template b/test/spec/client-backpressure/backpressure-retry-loop.yml.template new file mode 100644 index 00000000000..df1afe95cf7 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-loop.yml.template @@ -0,0 +1,116 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations respect overload backoff retry loop + +schemaVersion: '1.3' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + + - + client: + id: &internal_client internal_client + useMultipleMongoses: false + + - + database: + id: &internal_db internal_db + client: *internal_client + databaseName: &database_name retryable-writes-tests + + - + collection: + id: &internal_collection retryable-writes-tests + database: *internal_db + collectionName: &collection_name coll + + - + database: + id: &database database + client: *client + databaseName: *database_name + + - + collection: + id: &collection collection + database: *database + collectionName: *collection_name + +initialData: +- collectionName: *collection_name + databaseName: *database_name + documents: [] + +_yamlAnchors: + bulWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll + +tests: +{% for operation in operations %} + - + description: '{{operation.object}}.{{operation.operation_name}} retries using operation loop' + {%- if ((operation.operation_name == 'clientBulkWrite')) %} + runOnRequirements: + - minServerVersion: '8.0' # client bulk write added to server in 8.0 + {%- endif %} + operations: + {%- if operation.operation_name == "dropIndex" %} + - + object: *internal_collection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + {%- endif %} + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [{{operation.command_name}}] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *{{operation.object}} + name: {{operation.operation_name}} + {%- if operation.arguments|length > 0 %} + arguments: + {%- for arg in operation.arguments %} + {{arg}} + {%- endfor -%} + {%- endif %} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandSucceededEvent: + commandName: {{operation.command_name}} +{% endfor -%} diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.json b/test/spec/client-backpressure/backpressure-retry-max-attempts.json new file mode 100644 index 00000000000..1de8cb38d4e --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.json @@ -0,0 +1,3448 @@ +{ + "description": "tests that operations retry at most maxAttempts=5 times", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "client": { + "id": "fail_point_client", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "_yamlAnchors": { + "bulkWriteInsertNamespace": "retryable-writes-tests.coll" + }, + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "client.listDatabases retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabases", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.listDatabaseNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabaseNames", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite retries at most maxAttempts=5 times", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "database.aggregate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "database.listCollections retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollections", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.listCollectionNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollectionNames", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.runCommand retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "aggregate", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "countDocuments", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "estimatedDocumentCount", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.distinct retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "distinct", + "arguments": { + "fieldName": "x", + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + } + ] + } + ] + }, + { + "description": "collection.find retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.findOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOne", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexes retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexes", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexNames", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.insertOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.insertMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.deleteOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteOne", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.deleteMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteMany", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.replaceOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateOne", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateMany", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndDelete", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndReplace", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndUpdate", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.bulkWrite retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.createIndex retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndex retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndex", + "arguments": { + "name": "x_11" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndexes retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndexes", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml new file mode 100644 index 00000000000..3800b20a333 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml @@ -0,0 +1,1844 @@ +# Tests in this file are generated from backpressure-retry-max-attempts.yml.template. + +description: tests that operations retry at most maxAttempts=5 times + +schemaVersion: '1.3' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + + - + client: + id: &fail_point_client fail_point_client + useMultipleMongoses: false + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +_yamlAnchors: + bulkWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: + + - + description: 'client.listDatabases retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listDatabases] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: listDatabases + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + + + - + description: 'client.listDatabaseNames retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listDatabases] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: listDatabaseNames + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + + + - + description: 'client.createChangeStream retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: createChangeStream + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + + - + description: 'client.clientBulkWrite retries at most maxAttempts=5 times' + runOnRequirements: + - minServerVersion: '8.0' # client bulk write added to server in 8.0 + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [bulkWrite] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: *client_bulk_write_ns + document: { _id: 8, x: 88 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + + + - + description: 'database.aggregate retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: aggregate + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + + - + description: 'database.listCollections retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listCollections] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: listCollections + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + + + - + description: 'database.listCollectionNames retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listCollections] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: listCollectionNames + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + + + - + description: 'database.runCommand retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [ping] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: runCommand + arguments: + command: { ping: 1 } + commandName: ping + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + + + - + description: 'database.createChangeStream retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: createChangeStream + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + + - + description: 'collection.aggregate retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: aggregate + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + + - + description: 'collection.countDocuments retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: countDocuments + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + + - + description: 'collection.estimatedDocumentCount retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [count] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: estimatedDocumentCount + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + + + - + description: 'collection.distinct retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [distinct] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: distinct + arguments: + fieldName: x + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + + + - + description: 'collection.find retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [find] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: find + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + + + - + description: 'collection.findOne retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [find] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOne + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + + + - + description: 'collection.listIndexes retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: listIndexes + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + + + - + description: 'collection.listIndexNames retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: listIndexNames + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + + + - + description: 'collection.createChangeStream retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: createChangeStream + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + + - + description: 'collection.insertOne retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: insertOne + arguments: + document: { _id: 2, x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + + - + description: 'collection.insertMany retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: insertMany + arguments: + documents: + - { _id: 2, x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + + - + description: 'collection.deleteOne retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [delete] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: deleteOne + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + + + - + description: 'collection.deleteMany retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [delete] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: deleteMany + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + + + - + description: 'collection.replaceOne retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: replaceOne + arguments: + filter: {} + replacement: { x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + + - + description: 'collection.updateOne retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: updateOne + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + + - + description: 'collection.updateMany retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: updateMany + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + + - + description: 'collection.findOneAndDelete retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOneAndDelete + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + + - + description: 'collection.findOneAndReplace retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOneAndReplace + arguments: + filter: {} + replacement: { x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + + - + description: 'collection.findOneAndUpdate retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOneAndUpdate + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + + - + description: 'collection.bulkWrite retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: bulkWrite + arguments: + requests: + - insertOne: + document: { _id: 2, x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + + - + description: 'collection.createIndex retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [createIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + + + - + description: 'collection.dropIndex retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [dropIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: dropIndex + arguments: + name: "x_11" + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + + + - + description: 'collection.dropIndexes retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [dropIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: dropIndexes + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template new file mode 100644 index 00000000000..3117d44b890 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template @@ -0,0 +1,111 @@ +# Tests in this file are generated from backpressure-retry-max-attempts.yml.template. + +description: tests that operations retry at most maxAttempts=5 times + +schemaVersion: '1.3' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + + - + client: + id: &fail_point_client fail_point_client + useMultipleMongoses: false + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +_yamlAnchors: + bulkWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: +{% for operation in operations %} + - + description: '{{operation.object}}.{{operation.operation_name}} retries at most maxAttempts=5 times' + {%- if ((operation.operation_name == 'clientBulkWrite')) %} + runOnRequirements: + - minServerVersion: '8.0' # client bulk write added to server in 8.0 + {%- endif %} + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [{{operation.command_name}}] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *{{operation.object}} + name: {{operation.operation_name}} + {%- if operation.arguments|length > 0 %} + arguments: + {%- for arg in operation.arguments %} + {{arg}} + {%- endfor -%} + {%- endif %} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + +{% endfor -%} diff --git a/test/spec/client-backpressure/getMore-retried.json b/test/spec/client-backpressure/getMore-retried.json new file mode 100644 index 00000000000..f60ad4187d5 --- /dev/null +++ b/test/spec/client-backpressure/getMore-retried.json @@ -0,0 +1,286 @@ +{ + "description": "getMore-retries-backpressure", + "schemaVersion": "1.3", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandFailedEvent", + "commandSucceededEvent" + ] + } + }, + { + "client": { + "id": "failPointClient" + } + }, + { + "database": { + "id": "db", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "coll", + "database": "db", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "default", + "collectionName": "default", + "documents": [ + { + "a": 1 + }, + { + "a": 2 + }, + { + "a": 3 + } + ] + } + ], + "tests": [ + { + "description": "getMores are retried", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "getMore" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "find", + "arguments": { + "batchSize": 2, + "filter": {}, + "sort": { + "a": 1 + } + }, + "object": "coll", + "expectResult": [ + { + "a": 1 + }, + { + "a": 2 + }, + { + "a": 3 + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandSucceededEvent": { + "commandName": "getMore" + } + } + ] + } + ] + }, + { + "description": "getMores are retried maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "getMore" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "find", + "arguments": { + "batchSize": 2, + "filter": {} + }, + "object": "coll", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "killCursors" + } + }, + { + "commandSucceededEvent": { + "commandName": "killCursors" + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/client-backpressure/getMore-retried.yml b/test/spec/client-backpressure/getMore-retried.yml new file mode 100644 index 00000000000..aaef0335007 --- /dev/null +++ b/test/spec/client-backpressure/getMore-retried.yml @@ -0,0 +1,147 @@ +description: getMore-retries-backpressure +schemaVersion: "1.3" + +createEntities: + - client: + id: &client client0 + observeEvents: + - commandStartedEvent + - commandFailedEvent + - commandSucceededEvent + - client: + id: &failPointClient failPointClient + - database: + id: db + client: *client + databaseName: &dbName default + - collection: + id: &collection coll + database: db + collectionName: &collectionName default +initialData: + - databaseName: *dbName + collectionName: *collectionName + documents: + - { a: 1 } + - { a: 2 } + - { a: 3 } + +tests: + - description: "getMores are retried" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [getMore] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: find + arguments: + # batch size of 2 with 3 docs in the collection ensures exactly one find + one getMore exhaust the cursor + batchSize: 2 + filter: {} + # ensure stable ordering of result documents + sort: { a: 1 } + object: *collection + expectResult: + - { a: 1 } + - { a: 2 } + - { a: 3 } + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + # first attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # second attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # third attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # success + - commandStartedEvent: + commandName: getMore + - commandSucceededEvent: + commandName: getMore + + - description: "getMores are retried maxAttempts=5 times" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [getMore] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: find + arguments: + batchSize: 2 + filter: {} + object: *collection + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + # first attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # second attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # third attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # fourth attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # fifth attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # final attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + - commandStartedEvent: + commandName: killCursors + - commandSucceededEvent: + commandName: killCursors \ No newline at end of file diff --git a/test/spec/transactions/unified/backpressure-retryable-abort.json b/test/spec/transactions/unified/backpressure-retryable-abort.json new file mode 100644 index 00000000000..53fc9c6f090 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-abort.json @@ -0,0 +1,357 @@ +{ + "description": "backpressure-retryable-abort", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "abortTransaction retries if backpressure labels are added", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "abortTransaction" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "abortTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "abortTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "abortTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + }, + { + "description": "abortTransaction is retried maxAttempts=5 times if backpressure labels are added", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "abortTransaction" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "abortTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + } + ] +} diff --git a/test/spec/transactions/unified/backpressure-retryable-abort.yml b/test/spec/transactions/unified/backpressure-retryable-abort.yml new file mode 100644 index 00000000000..85532e1f605 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-abort.yml @@ -0,0 +1,213 @@ +description: backpressure-retryable-abort +schemaVersion: "1.3" +runOnRequirements: + - minServerVersion: "4.4" + topologies: + - replicaset + - sharded + - load-balanced +createEntities: + - + client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - + database: + id: &database0 database0 + client: *client0 + databaseName: &database_name transaction-tests + - + collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection_name test + - + session: + id: &session0 session0 + client: *client0 + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: [] +tests: + - description: abortTransaction retries if backpressure labels are added + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 2 + data: + failCommands: + - abortTransaction + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + abortTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: abortTransaction + databaseName: admin + - commandStartedEvent: + command: + abortTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: abortTransaction + databaseName: admin + - commandStartedEvent: + command: + abortTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: abortTransaction + databaseName: admin + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] + - description: abortTransaction is retried maxAttempts=5 times if backpressure labels are added + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - abortTransaction + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + abortTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: abortTransaction + databaseName: admin + - commandStartedEvent: + commandName: abortTransaction + - commandStartedEvent: + commandName: abortTransaction + - commandStartedEvent: + commandName: abortTransaction + - commandStartedEvent: + commandName: abortTransaction + - commandStartedEvent: + commandName: abortTransaction + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] diff --git a/test/spec/transactions/unified/backpressure-retryable-commit.json b/test/spec/transactions/unified/backpressure-retryable-commit.json new file mode 100644 index 00000000000..ae873561a99 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-commit.json @@ -0,0 +1,374 @@ +{ + "description": "backpressure-retryable-commit", + "schemaVersion": "1.4", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "sharded", + "replicaset", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "commitTransaction retries if backpressure labels are added", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "commitTransaction" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "session0", + "name": "commitTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ] + }, + { + "description": "commitTransaction is retried maxAttempts=5 times if backpressure labels are added", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "commitTransaction" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "session0", + "name": "commitTransaction", + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + } + ] +} diff --git a/test/spec/transactions/unified/backpressure-retryable-commit.yml b/test/spec/transactions/unified/backpressure-retryable-commit.yml new file mode 100644 index 00000000000..8099e1c1eba --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-commit.yml @@ -0,0 +1,220 @@ +description: backpressure-retryable-commit +schemaVersion: "1.4" +runOnRequirements: + - minServerVersion: "4.4" + topologies: + - sharded + - replicaset + - load-balanced +createEntities: + - + client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - + database: + id: &database0 database0 + client: *client0 + databaseName: &database_name transaction-tests + - + collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection_name test + - + session: + id: &session0 session0 + client: *client0 + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: [] +tests: + - description: commitTransaction retries if backpressure labels are added + runOnRequirements: + - serverless: forbid + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 2 + data: + failCommands: + - commitTransaction + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: *session0 + name: commitTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: + - _id: 1 + - description: commitTransaction is retried maxAttempts=5 times if backpressure labels are added + runOnRequirements: + - serverless: forbid + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - commitTransaction + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: *session0 + name: commitTransaction + expectError: + isError: true + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + - commandStartedEvent: + commandName: commitTransaction + - commandStartedEvent: + commandName: commitTransaction + - commandStartedEvent: + commandName: commitTransaction + - commandStartedEvent: + commandName: commitTransaction + - commandStartedEvent: + commandName: commitTransaction + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] diff --git a/test/spec/transactions/unified/backpressure-retryable-reads.json b/test/spec/transactions/unified/backpressure-retryable-reads.json new file mode 100644 index 00000000000..731762830e3 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-reads.json @@ -0,0 +1,328 @@ +{ + "description": "backpressure-retryable-reads", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "reads are retried if backpressure labels are added", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": {}, + "session": "session0" + } + }, + { + "object": "session0", + "name": "commitTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "test", + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "find", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "test", + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "find", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ] + }, + { + "description": "reads are retried maxAttempts=5 times if backpressure labels are added", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": {}, + "session": "session0" + }, + "expectError": { + "isError": true + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/transactions/unified/backpressure-retryable-reads.yml b/test/spec/transactions/unified/backpressure-retryable-reads.yml new file mode 100644 index 00000000000..18bbdaadbfa --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-reads.yml @@ -0,0 +1,192 @@ +description: backpressure-retryable-reads +schemaVersion: "1.3" +runOnRequirements: + - minServerVersion: "4.4" + topologies: + - replicaset + - sharded + - load-balanced +createEntities: + - + client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - + database: + id: &database0 database0 + client: *client0 + databaseName: &database_name transaction-tests + - + collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection_name test + - + session: + id: &session0 session0 + client: *client0 + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: [] +tests: + - description: reads are retried if backpressure labels are added + operations: + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - find + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: find + arguments: + filter: {} + session: *session0 + - object: *session0 + name: commitTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + find: test + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + autocommit: false + writeConcern: + $$exists: false + commandName: find + databaseName: *database_name + - commandStartedEvent: + command: + find: test + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + autocommit: false + writeConcern: + $$exists: false + commandName: find + databaseName: *database_name + - commandStartedEvent: + command: + abortTransaction: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + - description: reads are retried maxAttempts=5 times if backpressure labels are added + operations: + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - find + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: find + arguments: + filter: {} + session: *session0 + expectError: + isError: true + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: abortTransaction diff --git a/test/spec/transactions/unified/backpressure-retryable-writes.json b/test/spec/transactions/unified/backpressure-retryable-writes.json new file mode 100644 index 00000000000..0817e03f2f7 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-writes.json @@ -0,0 +1,440 @@ +{ + "description": "backpressure-retryable-writes", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "writes are retried if backpressure labels are added", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 2 + } + } + }, + { + "object": "session0", + "name": "commitTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + }, + { + "description": "writes are retried maxAttempts=5 times if backpressure labels are added", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 2 + } + }, + "expectError": { + "isError": true + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + }, + { + "description": "retry succeeds if backpressure labels are added to the first operation in a transaction", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 2 + } + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + } + ] +} diff --git a/test/spec/transactions/unified/backpressure-retryable-writes.yml b/test/spec/transactions/unified/backpressure-retryable-writes.yml new file mode 100644 index 00000000000..630c9d9694b --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-writes.yml @@ -0,0 +1,250 @@ +description: backpressure-retryable-writes +schemaVersion: "1.3" +runOnRequirements: + - minServerVersion: "4.4" + topologies: + - replicaset + - sharded + - load-balanced +createEntities: + - + client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - + database: + id: &database0 database0 + client: *client0 + databaseName: &database_name transaction-tests + - + collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection_name test + - + session: + id: &session0 session0 + client: *client0 + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: [] +tests: + - description: writes are retried if backpressure labels are added + operations: + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - insert + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 2 + - object: *session0 + name: commitTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + insert: test + documents: + - _id: 2 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + insert: test + documents: + - _id: 2 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + abortTransaction: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1 } + - { _id: 2 } + - description: writes are retried maxAttempts=5 times if backpressure labels are added + operations: + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - insert + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 2 + expectError: + isError: true + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: abortTransaction + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] + - description: retry succeeds if backpressure labels are added to the first operation in a transaction + operations: + - object: *session0 + name: startTransaction + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - insert + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 2 + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: abortTransaction + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index 7b73c22b82b..9e74808e29c 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -132,6 +132,7 @@ const EXPECTED_EXPORTS = [ 'SeverityLevel', 'SrvPollingEvent', 'Timestamp', + 'TokenBucket', 'TopologyClosedEvent', 'TopologyDescriptionChangedEvent', 'TopologyOpeningEvent',