HBASE-29474 RegionSplitter.rollingSplit is broken (#7174)

Avoid concurrent modification by iterating over a snapshot of the keys.
Also, revive the sorting logic for the ServerName list, which was mistakenly
removed in 5e91b45.

Signed-off-by: Duo Zhang <[email protected]>

Author: junegunn

hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java

@@ -21,9 +21,9 @@
 import java.math.BigInteger;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
 import org.apache.commons.lang3.ArrayUtils;
@@ -460,13 +460,15 @@ static void rollingSplit(TableName tableName, SplitAlgorithm splitAlgo, Configur
                 }
               }
 
+              // Sort the ServerNames by the number of regions they have
+              final List<ServerName> serversLeft = Lists.newArrayList(daughterRegions.keySet());
+              serversLeft.sort(Comparator.comparing(rsSizes::get));
+
               // Round-robin through the ServerName list. Choose the lightest-loaded servers
               // first to keep the master from load-balancing regions as we split.
-              for (Map.Entry<ServerName,
-                LinkedList<Pair<byte[], byte[]>>> daughterRegion : daughterRegions.entrySet()) {
+              for (final ServerName rsLoc : serversLeft) {
                 Pair<byte[], byte[]> dr = null;
-                ServerName rsLoc = daughterRegion.getKey();
-                LinkedList<Pair<byte[], byte[]>> regionList = daughterRegion.getValue();
+                final LinkedList<Pair<byte[], byte[]>> regionList = daughterRegions.get(rsLoc);
 
                 // Find a region in the ServerName list that hasn't been moved
                 LOG.debug("Finding a region on " + rsLoc);

hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitter.java

@@ -72,7 +72,7 @@ public class TestRegionSplitter {
 
   @BeforeClass
   public static void setup() throws Exception {
-    UTIL.startMiniCluster();
+    UTIL.startMiniCluster(2);
   }
 
   @AfterClass