HasFrozenCacheAllocationDecider returns No decision for unknown node (#137232)

Since SearchableSnapshotAllocator updates frozen cache service before
allocation, the unknown state for a node can only happen when the node
leaves the cluster. In this case, it makes more sense to reject the
allocation instead of throttling since the node may not come back. This
is also more consistent with the behaviour of
SearchableSnapshotAllocator which does not consider any node that is not
currently in the cluster (effectively a No). If the node does come back,
it will go through again the cache info fetching steps.

Relates: #136794

Author: ywangd

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/HasFrozenCacheAllocationDecider.java

@@ -48,6 +48,12 @@ public class HasFrozenCacheAllocationDecider extends AllocationDecider {
         "there was an error fetching the searchable snapshot shared cache state from this node"
     );
 
+    private static final Decision NO_UNKNOWN_NODE = Decision.single(
+        Decision.Type.NO,
+        NAME,
+        "this node is unknown to the searchable snapshot shared cache state"
+    );
+
     private final FrozenCacheInfoService frozenCacheService;
 
     public HasFrozenCacheAllocationDecider(FrozenCacheInfoService frozenCacheService) {
@@ -74,7 +80,8 @@ public Decision shouldAutoExpandToNode(IndexMetadata indexMetadata, DiscoveryNod
         return canAllocateToNode(indexMetadata, node);
     }
 
-    private Decision canAllocateToNode(IndexMetadata indexMetadata, DiscoveryNode discoveryNode) {
+    // Package private for tests
+    Decision canAllocateToNode(IndexMetadata indexMetadata, DiscoveryNode discoveryNode) {
         if (indexMetadata.isPartialSearchableSnapshot() == false) {
             return Decision.ALWAYS;
         }
@@ -83,7 +90,8 @@ private Decision canAllocateToNode(IndexMetadata indexMetadata, DiscoveryNode di
             case HAS_CACHE -> HAS_FROZEN_CACHE;
             case NO_CACHE -> NO_FROZEN_CACHE;
             case FAILED -> UNKNOWN_FROZEN_CACHE;
-            default -> STILL_FETCHING;
+            case FETCHING -> STILL_FETCHING;
+            case UNKNOWN -> NO_UNKNOWN_NODE;
         };
     }
 

x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/HasFrozenCacheAllocationDeciderTests.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.searchablesnapshots.allocation.decider;
+
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheInfoService;
+import org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheInfoService.NodeState;
+
+import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
+import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_PARTIAL_SETTING_KEY;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class HasFrozenCacheAllocationDeciderTests extends ESTestCase {
+
+    public void testCanAllocateToNode() {
+        final var frozenCacheService = mock(FrozenCacheInfoService.class);
+        final var decider = new HasFrozenCacheAllocationDecider(frozenCacheService);
+
+        final var indexMetadata = IndexMetadata.builder(randomIdentifier())
+            .settings(indexSettings(IndexVersion.current(), between(1, 10), between(0, 2)))
+            .build();
+
+        for (var nodeState : NodeState.values()) {
+            when(frozenCacheService.getNodeState(any(DiscoveryNode.class))).thenReturn(nodeState);
+            assertThat(decider.canAllocateToNode(indexMetadata, mock(DiscoveryNode.class)), equalTo(Decision.ALWAYS));
+        }
+
+        final var partialSearchableSnapshotIndexMetadata = IndexMetadata.builder(randomIdentifier())
+            .settings(
+                indexSettings(IndexVersion.current(), between(1, 10), between(0, 2)).put(INDEX_STORE_TYPE_SETTING.getKey(), "snapshot")
+                    .put(SEARCHABLE_SNAPSHOT_PARTIAL_SETTING_KEY, true)
+            )
+            .build();
+
+        for (var nodeState : NodeState.values()) {
+            when(frozenCacheService.getNodeState(any(DiscoveryNode.class))).thenReturn(nodeState);
+            final Decision decision = decider.canAllocateToNode(partialSearchableSnapshotIndexMetadata, mock(DiscoveryNode.class));
+            final Decision.Type expectedType;
+            if (nodeState == NodeState.HAS_CACHE) {
+                expectedType = Decision.Type.YES;
+            } else if (nodeState == NodeState.FETCHING) {
+                expectedType = Decision.Type.THROTTLE;
+            } else {
+                expectedType = Decision.Type.NO;
+            }
+            assertThat("incorrect decision for " + nodeState, decision.type(), is(expectedType));
+        }
+    }
+
+}