Add more flaky tests to retry logic

Signed-off-by: Prudhvi Godithi <[email protected]>

Author: prudhvigodithi

build.gradle

@@ -478,10 +478,25 @@ subprojects {
   tasks.withType(Test).configureEach {
     develocity.testRetry {
       if (BuildParams.isCi()) {
-        maxRetries = 3
-        maxFailures = 10
+        maxRetries.set(5)
+        maxFailures.set(15)
       }
-      failOnPassedAfterRetry = false
+      beforeRetry { descriptor ->
+        // Get the current test seed
+        String currentSeed = System.getProperty('tests.seed')
+        if (currentSeed != null) {
+          try {
+            long seedValue = Long.parseUnsignedLong(currentSeed, 16)
+            long newSeedValue = seedValue + 1
+            String newSeed = Long.toUnsignedString(newSeedValue, 16).toUpperCase(Locale.ROOT)
+            systemProperty 'tests.seed', newSeed
+            logger.lifecycle("Retrying ${descriptor.className}.${descriptor.methodName} with new seed: ${newSeed} (was: ${currentSeed})")
+          } catch (NumberFormatException e) {
+            logger.warn("Failed to parse seed value [${currentSeed}], will use the same seed for retry", e)
+          }
+        }
+      }
+      failOnPassedAfterRetry.set(false)
       filter {
         includeClasses.add("org.opensearch.action.admin.cluster.node.tasks.ResourceAwareTasksTests")
         includeClasses.add("org.opensearch.action.admin.cluster.tasks.PendingTasksBlocksIT")
@@ -557,6 +572,56 @@ subprojects {
         includeClasses.add("org.opensearch.test.rest.ClientYamlTestSuiteIT")
         includeClasses.add("org.opensearch.upgrade.DetectEsInstallationTaskTests")
         includeClasses.add("org.opensearch.cluster.MinimumClusterManagerNodesIT")
+        includeClasses.add("org.opensearch.indices.IndicesRequestCacheIT")
+        includeClasses.add("org.opensearch.action.admin.indices.create.RemoteCloneIndexIT")
+        includeClasses.add("org.opensearch.remotestore.RemoteStorePinnedTimestampsGarbageCollectionIT")
+        includeClasses.add("org.opensearch.remotemigration.RemoteMigrationIndexMetadataUpdateIT")
+        includeClasses.add("org.opensearch.search.approximate.ApproximatePointRangeQueryTests")
+        includeClasses.add("org.opensearch.action.admin.indices.create.RemoteSplitIndexIT")
+        includeClasses.add("org.opensearch.rest.ReactorNetty4StreamingStressIT")
+        includeClasses.add("org.opensearch.http.SearchRestCancellationIT")
+        includeClasses.add("org.opensearch.remotemigration.RemotePrimaryLocalRecoveryIT")
+        includeClasses.add("org.opensearch.cluster.service.MasterServiceTests")
+        includeClasses.add("org.opensearch.upgrades.RecoveryIT")
+        includeClasses.add("org.opensearch.upgrades.FullClusterRestartIT")
+        includeClasses.add("org.opensearch.indexing.IndexActionIT")
+        includeClasses.add("org.opensearch.indices.IndicesRequestCacheCleanupIT")
+        includeClasses.add("org.opensearch.script.expression.MoreExpressionIT")
+        includeClasses.add("org.opensearch.repositories.url.RepositoryURLClientYamlTestSuiteIT")
+        includeClasses.add("org.opensearch.join.query.ChildQuerySearchIT")
+        includeClasses.add("org.opensearch.recovery.RecoveryWhileUnderLoadIT")
+        includeClasses.add("org.opensearch.repositories.s3.S3BlobContainerRetriesTests")
+        includeClasses.add("org.opensearch.index.translog.RemoteFsTimestampAwareTranslogTests")
+        includeClasses.add("org.opensearch.search.simple.SimpleSearchIT")
+        includeClasses.add("org.opensearch.index.reindex.UpdateByQueryBasicTests")
+        includeClasses.add("org.opensearch.discovery.ClusterDisruptionIT")
+        includeClasses.add("org.opensearch.snapshots.DeleteSnapshotV2IT")
+        includeClasses.add("org.opensearch.cluster.routing.allocation.IndexShardHotSpotTests")
+        includeClasses.add("org.opensearch.upgrades.TranslogPolicyIT")
+        includeClasses.add("org.opensearch.snapshots.SharedClusterSnapshotRestoreIT")
+        includeClasses.add("org.opensearch.upgrades.UpgradeClusterClientYamlTestSuiteIT")
+        includeClasses.add("org.opensearch.cache.store.disk.EhCacheDiskCacheTests")
+        includeClasses.add("org.opensearch.backwards.PluginInfoIT")
+        includeClasses.add("org.opensearch.search.query.QueryPhaseTests")
+        includeClasses.add("org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandlerTests")
+        includeClasses.add("org.opensearch.remotestore.RemoteStoreRestoreIT")
+        includeClasses.add("org.opensearch.qa.verify_version_constants.VerifyVersionConstantsIT")
+        includeClasses.add("org.opensearch.cluster.routing.allocation.AllocationConstraintsTest")
+        includeClasses.add("org.opensearch.common.cache.stats.DefaultCacheStatsHolderTests")
+        includeClasses.add("org.opensearch.search.aggregations.startree.RangeAggregatorTests")
+        includeClasses.add("org.opensearch.index.reindex.ReindexBasicTests")
+        includeClasses.add("org.opensearch.gateway.remote.RemoteClusterStateCleanupManagerIT")
+        includeClasses.add("org.opensearch.http.reactor.netty4.ssl.SecureReactorNetty4HttpServerTransportTests")
+        includeClasses.add("org.opensearch.cluster.decommission.DecommissionControllerTests")
+        includeClasses.add("org.opensearch.plugin.kafka.IngestFromKafkaIT")
+        includeClasses.add("org.opensearch.plugin.kinesis.IngestFromKinesisIT")
+        includeClasses.add("org.opensearch.remotemigration.ResizeIndexMigrationTestCase")
+        includeClasses.add("org.opensearch.indices.replication.SegmentReplicationResizeRequestIT")
+        includeClasses.add("org.opensearch.index.reindex.CancelTests")
+        includeClasses.add("org.opensearch.snapshots.DeleteSnapshotIT")
+        includeClasses.add("org.opensearch.search.aggregations.EquivalenceIT")
+        includeClasses.add("org.opensearch.search.aggregations.startree")
+        includeClasses.add("org.opensearch.backwards.IndexingIT")
       }
     }
   }

settings.gradle

@@ -10,7 +10,7 @@
  */
 
 plugins {
-  id "com.gradle.develocity" version "3.19.1"
+  id "com.gradle.develocity" version "4.0"
 }
 
 ext.disableBuildCache = hasProperty('DISABLE_BUILD_CACHE') || System.getenv().containsKey('DISABLE_BUILD_CACHE')