Stop repair scheduler if two major versions are detected

Do not allow repair scheduler to run repair if two major versions are detected. This can happen during major version upgrade, and we don't want repair to run since the streaming data for repair can be incompatible across major versions.

Users can override this using a cassandra yaml config if they want to.

Also, this does not affect minor version upgrades, or repair jobs run manually.

https://issues.apache.org/jira/browse/CASSANDRA-20048

Author: himanshujindal

CHANGES.txt

@@ -220,6 +220,7 @@
  * Add ELAPSED command to cqlsh (CASSANDRA-18861)
  * Add the ability to disable bulk loading of SSTables (CASSANDRA-18781)
  * Clean up obsolete functions and simplify cql_version handling in cqlsh (CASSANDRA-18787)
+ * Stop repair scheduler if two major versions are detected (CASSANDRA-20048)
 Merged from 5.0:
  * Sort SSTable TOC entries for determinism (CASSANDRA-20494)
  * Do not source cassandra-env.sh unnecessarily in nodetool and other tooling (CASSANDRA-20745)

conf/cassandra.yaml

@@ -2782,6 +2782,9 @@ storage_compatibility_mode: NONE
 #   # The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata
 #   # for a specified duration to ensure they are indeed removed before adjustments are made to the schedule.
 #   history_clear_delete_hosts_buffer_interval: 2h
+#   # By default repair is disabled if there are mixed major versions detected - which would happen
+#   # if a major version upgrade is being performed on the cluster, but a user can enable it using this flag
+#   mixed_major_version_repair_enabled: false
 #   # NOTE: Each of the below settings can be overridden per repair type under repair_type_overrides
 #   global_settings:
 #     # If true, attempts to group tables in the same keyspace into one repair; otherwise, each table is repaired

conf/cassandra_latest.yaml

@@ -2467,6 +2467,9 @@ storage_compatibility_mode: NONE
 #   # The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata
 #   # for a specified duration to ensure they are indeed removed before adjustments are made to the schedule.
 #   history_clear_delete_hosts_buffer_interval: 2h
+#   # By default repair is disabled if there are mixed major versions detected - which would happen
+#   # if a major version upgrade is being performed on the cluster, but a user can enable it using this flag
+#   mixed_major_version_repair_enabled: false
 #   # NOTE: Each of the below settings can be overridden per repair type under repair_type_overrides
 #   global_settings:
 #     # If true, attempts to group tables in the same keyspace into one repair; otherwise, each table is repaired

doc/modules/cassandra/pages/managing/operating/auto_repair.adoc

@@ -167,6 +167,10 @@ is time to schedule repairs.
 | history_clear_delete_hosts_buffer_interval | 2h | The scheduler needs to adjust its order when nodes leave the ring.
 Deleted hosts are tracked in metadata for a specified duration to ensure they are indeed removed before adjustments
 are made to the schedule.
+| mixed_major_version_repair_enabled | false | Enable/Disable running repairs on the cluster when there are mixed
+major versions detected, which usually occurs when the cluster is being upgraded. Repairs between nodes of
+different major versions is not something that is tested, so this may lead to data compatibility issues.
+It is strongly discouraged to set this to true without doing extensive testing beforehand.
 |===
 
 

src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java

@@ -165,6 +165,11 @@ public void repair(AutoRepairConfig.RepairType repairType)
             logger.debug("Auto-repair is disabled for repair type {}", repairType);
             return;
         }
+        if (!config.isMixedMajorVersionRepairEnabled() && AutoRepairUtils.hasMultipleLiveMajorVersions())
+        {
+            logger.info("Auto-repair is disabled when nodes in the cluster have different major versions");
+            return;
+        }
         AutoRepairService.instance.checkCanRun(repairType);
         AutoRepairState repairState = repairStates.get(repairType);
         try

src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java

@@ -59,6 +59,8 @@ public class AutoRepairConfig implements Serializable
     // Minimum duration for the execution of a single repair task. This prevents the scheduler from overwhelming
     // the node by scheduling too many repair tasks in a short period of time.
     public volatile DurationSpec.LongSecondsBound repair_task_min_duration = new DurationSpec.LongSecondsBound("5s");
+    // by default repair is disabled if there are mixed major versions detected, but you can enable it using this flag
+    public volatile boolean mixed_major_version_repair_enabled = false;
 
     // global_settings overides Options.defaultOptions for all repair types
     public volatile Options global_settings;
@@ -149,6 +151,11 @@ public void setAutoRepairSchedulingEnabled(boolean enabled)
         this.enabled = enabled;
     }
 
+    public boolean isMixedMajorVersionRepairEnabled()
+    {
+        return mixed_major_version_repair_enabled;
+    }
+
     public DurationSpec.IntSecondsBound getAutoRepairHistoryClearDeleteHostsBufferInterval()
     {
         return history_clear_delete_hosts_buffer_interval;
@@ -366,6 +373,16 @@ public void setRepairRetryBackoff(RepairType repairType, String interval)
         getOptions(repairType).repair_retry_backoff = new DurationSpec.LongSecondsBound(interval);
     }
 
+    public boolean getMixedMajorVersionRepairEnabled()
+    {
+        return this.mixed_major_version_repair_enabled;
+    }
+
+    public void setMixedMajorVersionRepairEnabled(boolean enabled)
+    {
+        this.mixed_major_version_repair_enabled = enabled;
+    }
+
     @VisibleForTesting
     static IAutoRepairTokenRangeSplitter newAutoRepairTokenRangeSplitter(RepairType repairType, ParameterizedClass parameterizedClass) throws ConfigurationException
     {

src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java

@@ -425,6 +425,21 @@ public static CurrentRepairStatus getCurrentRepairStatus(RepairType repairType,
         return null;
     }
 
+    /**
+     * Checks whether the cluster has multiple major versions
+     * @return
+     *  true if more than one major versions are detected
+     *  false if only one major version is detected
+     *
+     */
+    public static boolean hasMultipleLiveMajorVersions()
+    {
+        ClusterMetadata metadata = ClusterMetadata.current();
+        int maxMajorVersion = ClusterMetadata.current().directory.clusterMaxVersion.cassandraVersion.major;
+        int minMajorVersion = ClusterMetadata.current().directory.clusterMinVersion.cassandraVersion.major;
+        return maxMajorVersion != minMajorVersion;
+    }
+
     @VisibleForTesting
     protected static TreeSet<UUID> getHostIdsInCurrentRing(RepairType repairType, Collection<NodeAddresses> allNodesInRing)
     {

src/java/org/apache/cassandra/service/AutoRepairService.java

@@ -102,6 +102,7 @@ public String getAutoRepairConfiguration()
         appendConfig(sb, "repair_check_interval", config.getRepairCheckInterval());
         appendConfig(sb, "repair_task_min_duration", config.getRepairTaskMinDuration());
         appendConfig(sb, "history_clear_delete_hosts_buffer_interval", config.getAutoRepairHistoryClearDeleteHostsBufferInterval());
+        appendConfig(sb, "mixed_major_version_repair_enabled", config.getMixedMajorVersionRepairEnabled());
         for (RepairType repairType : RepairType.values())
         {
             sb.append(formatRepairTypeConfig(repairType, config));
@@ -271,6 +272,12 @@ public void setAutoRepairRetryBackoff(String repairType, String interval)
         config.setRepairRetryBackoff(RepairType.parse(repairType), interval);
     }
 
+    @Override
+    public void setMixedMajorVersionRepairEnabled(boolean enabled)
+    {
+        config.setMixedMajorVersionRepairEnabled(enabled);
+    }
+
     private String formatRepairTypeConfig(RepairType repairType, AutoRepairConfig config)
     {
         StringBuilder sb = new StringBuilder();

src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java

@@ -74,4 +74,6 @@ public interface AutoRepairServiceMBean
     public void setAutoRepairMaxRetriesCount(String repairType, int retries);
 
     public void setAutoRepairRetryBackoff(String repairType, String interval);
+
+    public void setMixedMajorVersionRepairEnabled(boolean enabled);
 }

src/java/org/apache/cassandra/tools/NodeProbe.java

@@ -2677,6 +2677,11 @@ public GuardrailsMBean getGuardrailsMBean()
     {
         return grProxy;
     }
+
+    public void setMixedMajorVersionRepairEnabled(boolean enabled)
+    {
+        autoRepairProxy.setMixedMajorVersionRepairEnabled(enabled);
+    }
 }
 
 class ColumnFamilyStoreMBeanIterator implements Iterator<Map.Entry<String, ColumnFamilyStoreMBean>>