Test investigation: [CI] ILMDownsampleDisruptionIT testILMDownsampleRollingRestart (#137363)

We are adding extra logging to investigate [[CI]
ILMDownsampleDisruptionIT
testILMDownsampleRollingRestart](#136585).

The current logging aims to help us understand what happens when the
test tier down tries to delete the downsampled index.

Author: gmarouli

muted-tests.yml

@@ -456,9 +456,6 @@ tests:
 - class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
   method: test {p0=search.vectors/200_dense_vector_docvalue_fields/Enable docvalue_fields parameter for dense_vector fields}
   issue: https://github.com/elastic/elasticsearch/issues/136443
-- class: org.elasticsearch.xpack.downsample.ILMDownsampleDisruptionIT
-  method: testILMDownsampleRollingRestart
-  issue: https://github.com/elastic/elasticsearch/issues/136585
 - class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
   method: testManyConcat
   issue: https://github.com/elastic/elasticsearch/issues/136728

test/framework/src/main/java/org/elasticsearch/test/TestCluster.java

@@ -234,6 +234,7 @@ public void wipeIndices(String... indices) {
 
     private void wipeIndicesAsync(String[] indices, ActionListener<Void> listener) {
         assert indices != null && indices.length > 0;
+        logger.info("---- wiping indices [{}]", Strings.arrayToCommaDelimitedString(indices));
         SubscribableListener.<AcknowledgedResponse>newForked(
             l -> client().admin()
                 .indices()
@@ -248,13 +249,15 @@ private void wipeIndicesAsync(String[] indices, ActionListener<Void> listener) {
 
     private void handleWipeIndicesFailure(Exception exception, boolean wipingAllIndices, ActionListener<AcknowledgedResponse> listener) {
         Throwable unwrapped = ExceptionsHelper.unwrap(exception, IndexNotFoundException.class, IllegalArgumentException.class);
+        logger.error("---- initial wiping of indices failed", exception);
         if (unwrapped instanceof IndexNotFoundException) {
             // ignore
             listener.onResponse(AcknowledgedResponse.TRUE);
         } else if (unwrapped instanceof IllegalArgumentException) {
             // Happens if `action.destructive_requires_name` is set to true
             // which is the case in the CloseIndexDisableCloseAllTests
             if (wipingAllIndices) {
+                logger.info("---- retry wiping indices using their concrete names", exception);
                 SubscribableListener
 
                     .<ClusterStateResponse>newForked(l -> client().admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).execute(l))
@@ -274,6 +277,11 @@ private void handleWipeIndicesFailure(Exception exception, boolean wipingAllIndi
                 // TODO: this is clearly wrong but at least
                 // org.elasticsearch.xpack.watcher.test.integration.BootStrapTests.testTriggeredWatchLoading depends on this
                 // quietly passing when it tries to delete an alias instead of its backing indices
+                logger.warn(
+                    "This failure is ok, if this test is BootStrapTests.testTriggeredWatchLoading."
+                        + " Otherwise, please investigate this failure:",
+                    exception
+                );
                 listener.onResponse(AcknowledgedResponse.TRUE);
 
             }

x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/ILMDownsampleDisruptionIT.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.test.junit.annotations.TestIssueLogging;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin;
@@ -129,6 +130,10 @@ public void setup(final String sourceIndex, int numOfShards, int numOfReplicas,
         assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).actionGet());
     }
 
+    @TestIssueLogging(
+        value = "org.elasticsearch.cluster.service.MasterService:TRACE",
+        issueUrl = "https://github.com/elastic/elasticsearch/issues/136585"
+    )
     public void testILMDownsampleRollingRestart() throws Exception {
         final InternalTestCluster cluster = internalCluster();
         cluster.startMasterOnlyNodes(1);