add shortcut logic when get sub-partiton chunkRange pair

apache · Feb 18, 2025 · f7f364c · f7f364c
1 parent 8dae8ab
commit f7f364c
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/...park/spark-3-4/src/main/java/org/apache/spark/shuffle/celeborn/CelebornPartitionUtil.java b/...park/spark-3-4/src/main/java/org/apache/spark/shuffle/celeborn/CelebornPartitionUtil.java
@@ -91,6 +91,9 @@ public static Map<String, Pair<Integer, Integer>> splitSkewedPartitionLocations(
         j++;
       }
       partitionLocationOffset += p.getStorageInfo().getFileSize();
+      if (partitionLocationOffset > endOffset) {
+        break;
+      }
     }
     return chunkRange;
   }