Skip to content

Commit 7ebdc21

Browse files
committed
remove spark2 codec
1 parent 6276224 commit 7ebdc21

File tree

1 file changed

+2
-8
lines changed
  • client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn

1 file changed

+2
-8
lines changed

client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SparkUtils.java

+2-8
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@
4242
import org.apache.spark.SparkContext$;
4343
import org.apache.spark.TaskContext;
4444
import org.apache.spark.broadcast.Broadcast;
45-
import org.apache.spark.io.CompressionCodec;
4645
import org.apache.spark.scheduler.DAGScheduler;
4746
import org.apache.spark.scheduler.MapStatus;
4847
import org.apache.spark.scheduler.MapStatus$;
@@ -388,16 +387,14 @@ public static byte[] serializeGetReducerFileGroupResponse(
388387
scala.reflect.ClassManifestFactory.fromClass(
389388
GetReducerFileGroupResponse.class));
390389

391-
CompressionCodec codec = CompressionCodec.createCodec(sparkContext.conf());
392390
// Using `org.apache.commons.io.output.ByteArrayOutputStream` instead of the standard
393391
// one
394392
// This implementation doesn't reallocate the whole memory block but allocates
395393
// additional buffers. This way no buffers need to be garbage collected and
396394
// the contents don't have to be copied to the new buffer.
397395
org.apache.commons.io.output.ByteArrayOutputStream out =
398396
new org.apache.commons.io.output.ByteArrayOutputStream();
399-
try (ObjectOutputStream oos =
400-
new ObjectOutputStream(codec.compressedOutputStream(out))) {
397+
try (ObjectOutputStream oos = new ObjectOutputStream(out)) {
401398
oos.writeObject(broadcast);
402399
}
403400
byte[] _serializeResult = out.toByteArray();
@@ -426,10 +423,7 @@ public static GetReducerFileGroupResponse deserializeGetReducerFileGroupResponse
426423
GetReducerFileGroupResponse response = null;
427424

428425
try {
429-
CompressionCodec codec = CompressionCodec.createCodec(sparkContext.conf());
430-
try (ObjectInputStream objIn =
431-
new ObjectInputStream(
432-
codec.compressedInputStream(new ByteArrayInputStream(bytes)))) {
426+
try (ObjectInputStream objIn = new ObjectInputStream(new ByteArrayInputStream(bytes))) {
433427
Broadcast<GetReducerFileGroupResponse> broadcast =
434428
(Broadcast<GetReducerFileGroupResponse>) objIn.readObject();
435429
response = broadcast.value();

0 commit comments

Comments
 (0)