chore: remove unused import (#246)

apache · Oct 7, 2023 · de5f2bb · de5f2bb
1 parent d66ffe4
commit de5f2bb
Show file tree

Hide file tree

Showing 30 changed files with 49 additions and 130 deletions.
diff --git a/spark/.scalafix.conf b/spark/.scalafix.conf
@@ -0,0 +1,3 @@
+rules = [
+  RemoveUnused
+]
diff --git a/spark/pom.xml b/spark/pom.xml
@@ -197,11 +197,29 @@
                 <groupId>net.alchim31.maven</groupId>
                 <artifactId>scala-maven-plugin</artifactId>
                 <version>4.8.0</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
                 <configuration>
                     <jvmArgs>
                         <jvmArg>-Xms64m</jvmArg>
                         <jvmArg>-Xmx1024m</jvmArg>
                     </jvmArgs>
+                    <args>
+                        <arg>-Ywarn-unused</arg>
+                    </args>
+                    <compilerPlugins>
+                        <compilerPlugin>
+                            <groupId>org.scalameta</groupId>
+                            <artifactId>semanticdb-scalac_2.12.10</artifactId>
+                            <version>4.3.24</version>
+                        </compilerPlugin>
+                    </compilerPlugins>
                 </configuration>
             </plugin>
             <plugin>
@@ -225,6 +243,11 @@
                     </scala>
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>io.github.evis</groupId>
+                <artifactId>scalafix-maven-plugin_2.13</artifactId>
+                <version>0.1.8_0.11.0</version>
+            </plugin>
         </plugins>
     </build>
     <packaging>jar</packaging>

diff --git a/spark/src/main/scala/com/alibaba/graphar/EdgeInfo.scala b/spark/src/main/scala/com/alibaba/graphar/EdgeInfo.scala
@@ -16,8 +16,7 @@
 
 package com.alibaba.graphar
 
-import java.io.{File, FileInputStream}
-import org.apache.hadoop.fs.{Path, FileSystem}
+import org.apache.hadoop.fs.Path
 import org.apache.spark.sql.{SparkSession}
 import org.yaml.snakeyaml.{Yaml, DumperOptions}
 import org.yaml.snakeyaml.constructor.Constructor

diff --git a/spark/src/main/scala/com/alibaba/graphar/GraphInfo.scala b/spark/src/main/scala/com/alibaba/graphar/GraphInfo.scala
@@ -16,8 +16,7 @@
 
 package com.alibaba.graphar
 
-import java.io.{File, FileInputStream}
-import org.apache.hadoop.fs.{Path, FileSystem}
+import org.apache.hadoop.fs.Path
 import org.apache.spark.sql.{SparkSession}
 import org.yaml.snakeyaml.{Yaml, DumperOptions}
 import org.yaml.snakeyaml.constructor.Constructor

diff --git a/spark/src/main/scala/com/alibaba/graphar/VertexInfo.scala b/spark/src/main/scala/com/alibaba/graphar/VertexInfo.scala
@@ -16,8 +16,7 @@
 
 package com.alibaba.graphar
 
-import java.io.{File, FileInputStream}
-import org.apache.hadoop.fs.{Path, FileSystem}
+import org.apache.hadoop.fs.Path
 import org.apache.spark.sql.{SparkSession}
 import org.yaml.snakeyaml.{Yaml, DumperOptions}
 import org.yaml.snakeyaml.constructor.Constructor

diff --git a/spark/src/main/scala/com/alibaba/graphar/datasources/GarCommitProtocol.scala b/spark/src/main/scala/com/alibaba/graphar/datasources/GarCommitProtocol.scala
@@ -21,9 +21,7 @@ import com.alibaba.graphar.GeneralParams
 import org.json4s._
 import org.json4s.jackson.JsonMethods._
 
-import org.apache.spark.internal.io.FileCommitProtocol
 import org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
 import org.apache.hadoop.mapreduce._
 import org.apache.spark.internal.Logging
 

diff --git a/spark/src/main/scala/com/alibaba/graphar/datasources/GarDataSource.scala b/spark/src/main/scala/com/alibaba/graphar/datasources/GarDataSource.scala
@@ -29,7 +29,6 @@ import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.execution.datasources.csv.CSVFileFormat
 import org.apache.spark.sql.execution.datasources.orc.OrcFileFormat
 import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
-import org.apache.spark.sql.execution.datasources.v2._
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.sql.sources.DataSourceRegister

diff --git a/spark/src/main/scala/com/alibaba/graphar/datasources/GarWriterBuilder.scala b/spark/src/main/scala/com/alibaba/graphar/datasources/GarWriterBuilder.scala
@@ -26,12 +26,9 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.mapreduce.Job
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
-import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext}
+import org.apache.hadoop.mapreduce.Job
 
-import org.apache.spark.sql.execution.datasources.{
-  OutputWriter,
-  OutputWriterFactory
-}
+import org.apache.spark.sql.execution.datasources.OutputWriterFactory
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
@@ -49,7 +46,6 @@ import org.apache.spark.sql.execution.datasources.{
 import org.apache.spark.sql.execution.metric.SQLMetric
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{DataType, StructType}
-import org.apache.spark.sql.util.SchemaUtils
 import org.apache.spark.util.SerializableConfiguration
 import org.apache.spark.sql.execution.datasources.v2.FileBatchWrite
 import org.apache.spark.sql.catalyst.expressions.AttributeReference

diff --git a/spark/src/main/scala/com/alibaba/graphar/datasources/csv/CSVWriterBuilder.scala b/spark/src/main/scala/com/alibaba/graphar/datasources/csv/CSVWriterBuilder.scala
@@ -30,8 +30,6 @@ import org.apache.spark.sql.execution.datasources.{
 import org.apache.spark.sql.execution.datasources.csv.CsvOutputWriter
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{DataType, StructType}
-import org.apache.spark.sql.sources.Filter
-import org.apache.spark.sql.connector.write.SupportsOverwrite
 
 import com.alibaba.graphar.datasources.GarWriteBuilder
 

diff --git a/spark/src/main/scala/com/alibaba/graphar/example/GraphAr2Neo4j.scala b/spark/src/main/scala/com/alibaba/graphar/example/GraphAr2Neo4j.scala
@@ -16,18 +16,11 @@
 
 package com.alibaba.graphar.example
 
-import com.alibaba.graphar.datasources._
-import com.alibaba.graphar.reader.{VertexReader, EdgeReader}
 import com.alibaba.graphar.graph.GraphReader
 import com.alibaba.graphar.{GeneralParams, GraphInfo}
 import com.alibaba.graphar.util.Utils
 
-import java.io.{File, FileInputStream}
-import org.yaml.snakeyaml.Yaml
-import org.yaml.snakeyaml.constructor.Constructor
-import scala.beans.BeanProperty
 import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
-import org.apache.hadoop.fs.{Path, FileSystem}
 
 object GraphAr2Neo4j {
 

diff --git a/spark/src/main/scala/com/alibaba/graphar/example/Neo4j2GraphAr.scala b/spark/src/main/scala/com/alibaba/graphar/example/Neo4j2GraphAr.scala
@@ -16,10 +16,9 @@
 
 package com.alibaba.graphar.example
 
-import com.alibaba.graphar.datasources._
 import com.alibaba.graphar.graph.GraphWriter
 
-import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.SparkSession
 
 object Neo4j2GraphAr {
 

diff --git a/spark/src/main/scala/com/alibaba/graphar/graph/GraphReader.scala b/spark/src/main/scala/com/alibaba/graphar/graph/GraphReader.scala
@@ -16,19 +16,10 @@
 
 package com.alibaba.graphar.graph
 
-import com.alibaba.graphar.{
-  GeneralParams,
-  AdjListType,
-  GraphInfo,
-  VertexInfo,
-  EdgeInfo
-}
+import com.alibaba.graphar.{GraphInfo, VertexInfo, EdgeInfo}
 import com.alibaba.graphar.reader.{VertexReader, EdgeReader}
-import com.alibaba.graphar.util.IndexGenerator
 
 import org.apache.spark.sql.{DataFrame, SparkSession}
-import org.apache.spark.sql.types._
-import org.apache.spark.sql.functions._
 
 /**
  * The helper object for reading graph through the definitions of graph info.

diff --git a/spark/src/main/scala/com/alibaba/graphar/graph/GraphTransformer.scala b/spark/src/main/scala/com/alibaba/graphar/graph/GraphTransformer.scala
@@ -16,19 +16,11 @@
 
 package com.alibaba.graphar.graph
 
-import com.alibaba.graphar.{
-  GeneralParams,
-  AdjListType,
-  GraphInfo,
-  VertexInfo,
-  EdgeInfo
-}
+import com.alibaba.graphar.{AdjListType, GraphInfo, VertexInfo, EdgeInfo}
 import com.alibaba.graphar.reader.{VertexReader, EdgeReader}
 import com.alibaba.graphar.writer.{VertexWriter, EdgeWriter}
 
-import org.apache.spark.sql.{DataFrame, SparkSession}
-import org.apache.spark.sql.types._
-import org.apache.spark.sql.functions._
+import org.apache.spark.sql.SparkSession
 
 /**
  * The helper object for transforming graphs through the definitions of their

diff --git a/spark/src/main/scala/com/alibaba/graphar/graph/GraphWriter.scala b/spark/src/main/scala/com/alibaba/graphar/graph/GraphWriter.scala
@@ -16,22 +16,14 @@
 
 package com.alibaba.graphar.graph
 
-import com.alibaba.graphar.{
-  AdjListType,
-  GraphInfo,
-  VertexInfo,
-  EdgeInfo,
-  GeneralParams
-}
+import com.alibaba.graphar.{AdjListType, GraphInfo, GeneralParams}
 import com.alibaba.graphar.writer.{VertexWriter, EdgeWriter}
 import com.alibaba.graphar.util.IndexGenerator
 import com.alibaba.graphar.util.Utils
 
 import org.apache.spark.sql.{DataFrame, SparkSession}
 import org.apache.spark.sql.types._
-import org.apache.spark.sql.functions._
 
-import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileSystem, Path}
 import java.io.{BufferedWriter, OutputStreamWriter}
 

diff --git a/spark/src/main/scala/com/alibaba/graphar/reader/EdgeReader.scala b/spark/src/main/scala/com/alibaba/graphar/reader/EdgeReader.scala
@@ -17,19 +17,11 @@
 package com.alibaba.graphar.reader
 
 import com.alibaba.graphar.util.{IndexGenerator, DataFrameConcat}
-import com.alibaba.graphar.{
-  GeneralParams,
-  EdgeInfo,
-  FileType,
-  AdjListType,
-  PropertyGroup
-}
-import com.alibaba.graphar.datasources._
+import com.alibaba.graphar.{EdgeInfo, FileType, AdjListType, PropertyGroup}
 import com.alibaba.graphar.util.FileSystem
 
 import org.apache.spark.sql.{DataFrame, SparkSession}
 import org.apache.spark.sql.types._
-import org.apache.spark.sql.functions._
 
 /**
  * Reader for edge chunks.

diff --git a/spark/src/main/scala/com/alibaba/graphar/reader/VertexReader.scala b/spark/src/main/scala/com/alibaba/graphar/reader/VertexReader.scala
@@ -17,14 +17,11 @@
 package com.alibaba.graphar.reader
 
 import com.alibaba.graphar.util.{IndexGenerator, DataFrameConcat}
-import com.alibaba.graphar.{GeneralParams, VertexInfo, FileType, PropertyGroup}
-import com.alibaba.graphar.datasources._
+import com.alibaba.graphar.{VertexInfo, PropertyGroup}
 import com.alibaba.graphar.util.FileSystem
 
 import org.apache.spark.sql.{DataFrame, SparkSession}
 import org.apache.spark.sql.types._
-import org.apache.spark.sql.functions._
-import org.apache.spark.sql.Row
 
 /**
  * Reader for vertex chunks.

diff --git a/spark/src/main/scala/com/alibaba/graphar/util/DataFrameConcat.scala b/spark/src/main/scala/com/alibaba/graphar/util/DataFrameConcat.scala
@@ -16,11 +16,8 @@
 
 package com.alibaba.graphar.util
 
-import com.alibaba.graphar.GeneralParams
-
-import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.types._
-import org.apache.spark.sql.functions._
 import org.apache.spark.sql.Row
 import org.apache.spark.rdd.RDD
 

diff --git a/spark/src/main/scala/com/alibaba/graphar/util/FileSystem.scala b/spark/src/main/scala/com/alibaba/graphar/util/FileSystem.scala
@@ -19,7 +19,6 @@ package com.alibaba.graphar.util
 import org.json4s._
 import org.json4s.jackson.Serialization.write
 
-import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.DataFrame
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.conf.Configuration

diff --git a/spark/src/main/scala/com/alibaba/graphar/util/IndexGenerator.scala b/spark/src/main/scala/com/alibaba/graphar/util/IndexGenerator.scala
@@ -18,15 +18,11 @@ package com.alibaba.graphar.util
 
 import com.alibaba.graphar.GeneralParams
 
-import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.types._
-import org.apache.spark.sql.functions._
 import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.Row
-import org.apache.spark.rdd.RDD
 
 import scala.collection.SortedMap
-import scala.collection.mutable.ArrayBuffer
 
 /**
  * IndexGenerator is an object to help generating the indices for vertex/edge

diff --git a/spark/src/main/scala/com/alibaba/graphar/util/Patitioner.scala b/spark/src/main/scala/com/alibaba/graphar/util/Patitioner.scala
@@ -16,7 +16,6 @@
 
 package com.alibaba.graphar.util
 
-import org.apache.spark.sql.types._
 import org.apache.spark.Partitioner
 
 /**

diff --git a/spark/src/main/scala/com/alibaba/graphar/writer/EdgeWriter.scala b/spark/src/main/scala/com/alibaba/graphar/writer/EdgeWriter.scala
@@ -16,11 +16,7 @@
 
 package com.alibaba.graphar.writer
 
-import com.alibaba.graphar.util.{
-  FileSystem,
-  ChunkPartitioner,
-  EdgeChunkPartitioner
-}
+import com.alibaba.graphar.util.{FileSystem, EdgeChunkPartitioner}
 import com.alibaba.graphar.{
   GeneralParams,
   EdgeInfo,
@@ -38,8 +34,6 @@ import org.apache.spark.sql.types.{
   StructType,
   StructField
 }
-import org.apache.spark.util.Utils
-import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.functions._
 
 import scala.collection.SortedMap
@@ -54,7 +48,6 @@ object EdgeWriter {
       adjListType: AdjListType.Value,
       vertexNumOfPrimaryVertexLabel: Long
   ): (DataFrame, Seq[DataFrame], Array[Long], Map[Long, Int]) = {
-    import spark.implicits._
     val edgeSchema = edgeDf.schema
     val colName = if (
       adjListType == AdjListType.ordered_by_source || adjListType == AdjListType.unordered_by_source

diff --git a/spark/src/main/scala/com/alibaba/graphar/writer/VertexWriter.scala b/spark/src/main/scala/com/alibaba/graphar/writer/VertexWriter.scala
@@ -16,25 +16,14 @@
 
 package com.alibaba.graphar.writer
 
-import com.alibaba.graphar.util.{FileSystem, ChunkPartitioner, IndexGenerator}
-import com.alibaba.graphar.{
-  GeneralParams,
-  VertexInfo,
-  FileType,
-  AdjListType,
-  PropertyGroup
-}
+import com.alibaba.graphar.util.{FileSystem, ChunkPartitioner}
+import com.alibaba.graphar.{GeneralParams, VertexInfo, PropertyGroup}
 
-import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.Row
-import org.apache.spark.rdd.RDD
-import org.apache.spark.HashPartitioner
 import org.apache.spark.sql.types.{LongType, StructField}
 
-import scala.collection.SortedMap
 import scala.collection.mutable.ArrayBuffer
 
 /** Helper object for VertexWriter class. */

diff --git a/spark/src/test/scala/com/alibaba/graphar/ComputeExample.scala b/spark/src/test/scala/com/alibaba/graphar/ComputeExample.scala
@@ -18,13 +18,8 @@ package com.alibaba.graphar
 
 import com.alibaba.graphar.reader.{VertexReader, EdgeReader}
 
-import java.io.{File, FileInputStream}
-import scala.beans.BeanProperty
-import org.yaml.snakeyaml.Yaml
-import org.yaml.snakeyaml.constructor.Constructor
-import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.SparkSession
 import org.apache.spark.graphx._
-import org.apache.spark.rdd.RDD
 import org.scalatest.funsuite.AnyFunSuite
 
 class ComputeExampleSuite extends AnyFunSuite {

diff --git a/spark/src/test/scala/com/alibaba/graphar/TestGraphInfo.scala b/spark/src/test/scala/com/alibaba/graphar/TestGraphInfo.scala
@@ -16,11 +16,7 @@
 
 package com.alibaba.graphar
 
-import org.yaml.snakeyaml.Yaml
-import org.yaml.snakeyaml.constructor.Constructor
-import scala.beans.BeanProperty
 import org.scalatest.funsuite.AnyFunSuite
-import org.apache.hadoop.fs.{Path, FileSystem}
 import org.apache.spark.sql.SparkSession
 
 class GraphInfoSuite extends AnyFunSuite {
-Original file line number
+Diff line change
@@ Expand Up / @@ -16,7 +16,6 @@ @@
     package com.alibaba.graphar.util
-    import org.apache.spark.sql.types._
     import org.apache.spark.Partitioner
     /**
@@ Expand Down @@