From b98f081f947736eafcecb68957e9ea6465aa6cae Mon Sep 17 00:00:00 2001 From: Karl-WangSK Date: Thu, 29 Oct 2020 11:49:54 +0800 Subject: [PATCH] update --- .../{master.yml => build_and_test.yml} | 286 +- .github/workflows/test_report.yml | 24 + .gitignore | 3 + .sbtopts | 17 + R/pkg/NAMESPACE | 11 + R/pkg/R/DataFrame.R | 24 +- R/pkg/R/column.R | 106 +- R/pkg/R/deserialize.R | 13 +- R/pkg/R/functions.R | 152 +- R/pkg/R/generics.R | 40 +- R/pkg/R/utils.R | 4 + R/pkg/tests/fulltests/test_sparkSQL.R | 75 +- R/pkg/tests/fulltests/test_sparkSQL_arrow.R | 201 +- R/pkg/tests/run-all.R | 9 +- appveyor.yml | 4 +- bin/find-spark-home.cmd | 2 +- bin/load-spark-env.cmd | 6 +- bin/spark-class2.cmd | 6 +- binder/apt.txt | 1 + binder/postBuild | 24 + build/sbt-launch-lib.bash | 6 +- .../spark/util/kvstore/InMemoryStore.java | 5 +- common/network-common/pom.xml | 4 + .../spark/network/protocol/Encoders.java | 63 + .../spark/network/util/TransportConf.java | 2 +- common/network-shuffle/pom.xml | 9 + .../network/shuffle/BlockStoreClient.java | 95 + .../spark/network/shuffle/ErrorHandler.java | 85 + .../network/shuffle/ExternalBlockHandler.java | 104 +- .../shuffle/ExternalBlockStoreClient.java | 104 +- .../shuffle/ExternalShuffleBlockResolver.java | 8 - .../network/shuffle/MergedBlockMeta.java | 64 + .../shuffle/MergedShuffleFileManager.java | 116 + .../network/shuffle/OneForOneBlockPusher.java | 123 + .../network/shuffle/RetryingBlockFetcher.java | 27 +- .../protocol/BlockTransferMessage.java | 6 +- .../protocol/FinalizeShuffleMerge.java | 84 + .../shuffle/protocol/MergeStatuses.java | 118 + .../shuffle/protocol/PushBlockStream.java | 95 + .../network/shuffle/ErrorHandlerSuite.java | 51 + .../shuffle/ExternalBlockHandlerSuite.java | 40 +- .../ExternalShuffleBlockResolverSuite.java | 9 - .../ExternalShuffleIntegrationSuite.java | 6 +- .../shuffle/OneForOneBlockPusherSuite.java | 159 + common/network-yarn/pom.xml | 8 +- .../spark/util/sketch/Murmur3_x86_32.java | 10 +- .../spark/unsafe/hash/Murmur3_x86_32.java | 10 +- .../apache/spark/unsafe/types/UTF8String.java | 14 +- core/pom.xml | 16 +- .../apache/spark/api/java/StorageLevels.java | 1 + .../apache/spark/api/plugin/DriverPlugin.java | 2 +- .../spark/api/plugin/ExecutorPlugin.java | 42 + .../shuffle/sort/UnsafeShuffleWriter.java | 15 + .../io/LocalDiskShuffleMapOutputWriter.java | 6 +- .../spark/unsafe/map/BytesToBytesMap.java | 91 +- .../unsafe/sort/UnsafeExternalSorter.java | 188 +- .../unsafe/sort/UnsafeInMemorySorter.java | 56 +- .../unsafe/sort/UnsafeSorterIterator.java | 2 + .../unsafe/sort/UnsafeSorterSpillMerger.java | 5 + .../unsafe/sort/UnsafeSorterSpillReader.java | 5 + .../spark/ui/static/jquery-3.4.1.min.js | 2 - .../spark/ui/static/jquery-3.5.1.min.js | 2 + .../apache/spark/ui/static/timeline-view.js | 53 +- .../org/apache/spark/ui/static/utils.js | 7 +- .../spark/ExecutorAllocationClient.scala | 49 + .../spark/ExecutorAllocationManager.scala | 53 +- .../org/apache/spark/MapOutputTracker.scala | 21 +- .../scala/org/apache/spark/SparkConf.scala | 2 +- .../scala/org/apache/spark/SparkContext.scala | 33 +- .../scala/org/apache/spark/TestUtils.scala | 35 +- .../spark/broadcast/TorrentBroadcast.scala | 32 +- .../apache/spark/deploy/DeployMessage.scala | 38 +- .../apache/spark/deploy/SparkHadoopUtil.scala | 3 + .../org/apache/spark/deploy/SparkSubmit.scala | 36 +- .../deploy/StandaloneResourceUtils.scala | 10 +- .../deploy/client/StandaloneAppClient.scala | 6 +- .../client/StandaloneAppClientListener.scala | 2 +- .../history/ApplicationHistoryProvider.scala | 7 + .../deploy/history/FsHistoryProvider.scala | 54 +- .../spark/deploy/history/HistoryServer.scala | 5 + .../history/HistoryServerMemoryManager.scala | 5 +- .../spark/deploy/history/HybridStore.scala | 6 +- .../apache/spark/deploy/master/Master.scala | 32 +- .../spark/deploy/master/ui/MasterPage.scala | 10 +- .../HadoopFSDelegationTokenProvider.scala | 2 + .../apache/spark/deploy/worker/Worker.scala | 31 +- .../CoarseGrainedExecutorBackend.scala | 131 +- .../org/apache/spark/executor/Executor.scala | 81 +- .../apache/spark/internal/config/Tests.scala | 10 +- .../apache/spark/internal/config/Worker.scala | 6 - .../spark/internal/config/package.scala | 20 +- .../spark/internal/io/SparkHadoopWriter.scala | 4 + .../internal/plugin/PluginContainer.scala | 49 +- .../spark/network/BlockDataManager.scala | 5 + .../spark/network/BlockTransferService.scala | 3 +- .../network/netty/NettyBlockRpcServer.scala | 22 +- .../netty/NettyBlockTransferService.scala | 22 +- .../org/apache/spark/rdd/HadoopRDD.scala | 4 + .../ResourceDiscoveryScriptPlugin.scala | 2 +- .../apache/spark/resource/ResourceUtils.scala | 7 +- .../org/apache/spark/rpc/netty/Inbox.scala | 20 + .../BarrierJobAllocationFailed.scala | 4 +- .../apache/spark/scheduler/DAGScheduler.scala | 70 +- .../scheduler/ExecutorDecommissionInfo.scala | 22 +- .../spark/scheduler/ExecutorLossReason.scala | 11 +- .../scheduler/ExecutorResourceInfo.scala | 1 + .../org/apache/spark/scheduler/Pool.scala | 4 +- .../apache/spark/scheduler/Schedulable.scala | 1 + .../spark/scheduler/SchedulerBackend.scala | 2 +- .../spark/scheduler/ShuffleMapStage.scala | 2 +- .../org/apache/spark/scheduler/Task.scala | 6 +- .../spark/scheduler/TaskDescription.scala | 2 +- .../spark/scheduler/TaskResultGetter.scala | 2 +- .../spark/scheduler/TaskScheduler.scala | 2 +- .../spark/scheduler/TaskSchedulerImpl.scala | 178 +- .../spark/scheduler/TaskSetManager.scala | 103 +- .../cluster/CoarseGrainedClusterMessage.scala | 13 +- .../CoarseGrainedSchedulerBackend.scala | 204 +- .../cluster/StandaloneSchedulerBackend.scala | 15 +- .../scheduler/dynalloc/ExecutorMonitor.scala | 61 +- .../security/SecurityConfigurationLock.scala | 14 +- .../shuffle/IndexShuffleBlockResolver.scala | 43 +- .../apache/spark/shuffle/ShuffleManager.scala | 6 +- .../spark/status/api/v1/ApiRootResource.scala | 15 + .../api/v1/OneApplicationResource.scala | 9 +- .../org/apache/spark/status/api/v1/api.scala | 16 +- .../apache/spark/storage/BlockManager.scala | 63 +- .../storage/BlockManagerDecommissioner.scala | 154 +- .../spark/storage/BlockManagerMaster.scala | 6 +- .../storage/BlockManagerMasterEndpoint.scala | 49 +- .../storage/BlockManagerStorageEndpoint.scala | 2 +- .../storage/ShuffleBlockFetcherIterator.scala | 121 +- .../apache/spark/storage/StorageLevel.scala | 2 + .../scala/org/apache/spark/ui/SparkUI.scala | 5 + .../scala/org/apache/spark/ui/UIUtils.scala | 2 +- .../apache/spark/ui/jobs/AllJobsPage.scala | 3 +- .../org/apache/spark/ui/jobs/JobPage.scala | 3 +- .../org/apache/spark/util/HadoopFSUtils.scala | 360 + .../scala/org/apache/spark/util/Utils.scala | 25 +- .../map/AbstractBytesToBytesMapSuite.java | 27 +- .../sort/UnsafeExternalSorterSuite.java | 109 +- .../sort/UnsafeInMemorySorterSuite.java | 40 +- .../apache/spark/JavaSparkContextSuite.java | 7 +- .../spark/BarrierStageOnSubmittedSuite.scala | 36 + .../org/apache/spark/DistributedSuite.scala | 7 +- .../ExecutorAllocationManagerSuite.scala | 80 +- .../spark/ExternalShuffleServiceSuite.scala | 44 - .../org/apache/spark/SparkFunSuite.scala | 18 + .../apache/spark/TempLocalSparkContext.scala | 100 + .../apache/spark/benchmark/Benchmark.scala | 1 - .../deploy/DecommissionWorkerSuite.scala | 70 +- .../ExternalShuffleServiceMetricsSuite.scala | 3 +- .../spark/deploy/SparkSubmitSuite.scala | 83 +- .../spark/deploy/SparkSubmitUtilsSuite.scala | 16 +- .../spark/deploy/client/AppClientSuite.scala | 14 +- .../history/EventLogFileWritersSuite.scala | 6 +- .../history/FsHistoryProviderSuite.scala | 116 +- .../HistoryServerMemoryManagerSuite.scala | 55 + .../deploy/history/HistoryServerSuite.scala | 18 + .../deploy/history/HybridStoreSuite.scala | 232 + .../spark/deploy/master/MasterSuite.scala | 32 +- .../plugin/PluginContainerSuite.scala | 49 + .../spark/memory/TestMemoryManager.scala | 8 + .../spark/metrics/sink/StatsdSinkSuite.scala | 29 +- .../NettyBlockTransferServiceSuite.scala | 4 +- .../spark/rdd/LocalCheckpointSuite.scala | 1 + .../apache/spark/rpc/netty/InboxSuite.scala | 13 + .../scheduler/BarrierTaskContextSuite.scala | 4 +- .../spark/scheduler/DAGSchedulerSuite.scala | 79 +- .../ExternalClusterManagerSuite.scala | 4 +- .../spark/scheduler/TaskContextSuite.scala | 4 +- .../scheduler/TaskSchedulerImplSuite.scala | 141 +- .../spark/scheduler/TaskSetManagerSuite.scala | 104 +- .../WorkerDecommissionExtendedSuite.scala | 8 +- .../scheduler/WorkerDecommissionSuite.scala | 30 +- .../HostLocalShuffleReadingSuite.scala | 136 + .../sort/IndexShuffleBlockResolverSuite.scala | 5 + .../status/api/v1/ExecutorSummarySuite.scala | 51 + ...kManagerDecommissionIntegrationSuite.scala | 82 +- .../BlockManagerDecommissionUnitSuite.scala | 224 +- .../spark/storage/BlockManagerSuite.scala | 30 +- .../spark/storage/MemoryStoreSuite.scala | 29 +- .../ShuffleBlockFetcherIteratorSuite.scala | 91 +- .../spark/ui/ChromeUISeleniumSuite.scala | 4 +- .../spark/ui/RealBrowserUISeleniumSuite.scala | 56 +- .../org/apache/spark/ui/UISeleniumSuite.scala | 1 - .../apache/spark/util/JsonProtocolSuite.scala | 1 + .../spark/util/SizeEstimatorSuite.scala | 43 +- dev/.rat-excludes | 3 +- dev/appveyor-install-dependencies.ps1 | 4 +- dev/create-release/generate-contributors.py | 4 +- dev/create-release/release-build.sh | 9 +- dev/create-release/release-tag.sh | 6 +- dev/create-release/release-util.sh | 2 +- dev/create-release/releaseutils.py | 6 +- dev/create-release/spark-rm/Dockerfile | 5 +- dev/create-release/translate-contributors.py | 12 +- dev/deps/spark-deps-hadoop-2.7-hive-1.2 | 211 - dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 57 +- dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 106 +- dev/github_jira_sync.py | 2 +- dev/lint-python | 55 +- dev/merge_spark_pr.py | 2 +- dev/pip-sanity-check.py | 1 - dev/requirements.txt | 3 + dev/run-tests-jenkins.py | 2 - dev/run-tests.py | 17 +- dev/sparktestsupport/modules.py | 1 + dev/test-dependencies.sh | 6 +- dev/tox.ini | 5 + docs/README.md | 2 +- docs/_config.yml | 2 + docs/_data/menu-sql.yaml | 2 + docs/_layouts/global.html | 130 +- docs/api.md | 27 - docs/building-spark.md | 10 +- docs/configuration.md | 3 +- docs/css/bootstrap-responsive.css | 1040 - docs/css/bootstrap-responsive.min.css | 9 - docs/css/bootstrap.css | 5624 ----- docs/css/bootstrap.min.css | 14 +- docs/css/bootstrap.min.css.map | 1 + docs/css/main.css | 150 +- docs/img/pycharm-with-pyspark1.png | Bin 0 -> 160166 bytes docs/img/pycharm-with-pyspark2.png | Bin 0 -> 84813 bytes docs/img/pycharm-with-pyspark3.png | Bin 0 -> 15981 bytes docs/img/pyspark-components.png | Bin 0 -> 32727 bytes docs/img/pyspark-components.pptx | Bin 0 -> 354673 bytes docs/img/pyspark-remote-debug1.png | Bin 0 -> 91214 bytes docs/img/pyspark-remote-debug2.png | Bin 0 -> 10058 bytes docs/job-scheduling.md | 2 +- docs/js/main.js | 34 +- docs/js/vendor/bootstrap.bundle.min.js | 7 + docs/js/vendor/bootstrap.bundle.min.js.map | 1 + docs/js/vendor/bootstrap.js | 2027 -- docs/js/vendor/bootstrap.min.js | 6 - docs/js/vendor/jquery-3.4.1.min.js | 2 - docs/js/vendor/jquery-3.5.1.min.js | 2 + docs/monitoring.md | 24 +- docs/pyspark-migration-guide.md | 63 +- docs/quick-start.md | 2 +- docs/rdd-programming-guide.md | 4 +- docs/running-on-kubernetes.md | 36 +- docs/sql-data-sources-avro.md | 2 +- docs/sql-migration-guide.md | 20 + docs/sql-performance-tuning.md | 22 + docs/sql-pyspark-pandas-with-arrow.md | 349 +- docs/sql-ref-ansi-compliance.md | 6 - docs/sql-ref-datetime-pattern.md | 2 +- docs/sql-ref-null-semantics.md | 2 +- docs/sql-ref-syntax-aux-cache-cache-table.md | 1 + docs/sql-ref-syntax-aux-show-table.md | 6 +- docs/sql-ref-syntax-ddl-create-function.md | 4 +- ...-ref-syntax-ddl-create-table-datasource.md | 7 +- ...-ref-syntax-ddl-create-table-hiveformat.md | 32 + docs/sql-ref-syntax-qry-select-file.md | 76 + docs/sql-ref-syntax-qry-select-groupby.md | 2 +- docs/sql-ref-syntax-qry-select-hints.md | 2 +- docs/sql-ref-syntax-qry-select-tvf.md | 99 +- docs/sql-ref-syntax-qry-select.md | 2 + docs/sql-ref-syntax-qry.md | 1 + docs/sql-ref-syntax.md | 1 + docs/sql-ref.md | 2 +- .../structured-streaming-programming-guide.md | 8 +- docs/submitting-applications.md | 2 +- docs/tuning.md | 11 + docs/web-ui.md | 10 +- examples/pom.xml | 5 - .../ml/estimator_transformer_param_example.py | 8 +- .../main/python/ml/fm_classifier_example.py | 6 +- .../main/python/ml/fm_regressor_example.py | 6 +- .../src/main/python/ml/pipeline_example.py | 8 +- examples/src/main/python/sql/arrow.py | 26 +- examples/src/main/python/sql/basic.py | 2 +- examples/src/main/python/sql/datasource.py | 2 +- examples/src/main/python/sql/hive.py | 2 +- examples/src/main/python/status_api_demo.py | 1 - .../sql/avro/SparkAvroKeyOutputFormat.java | 4 +- .../spark/sql/avro/AvroDataToCatalyst.scala | 2 +- .../spark/sql/avro/AvroDeserializer.scala | 4 +- .../apache/spark/sql/avro/AvroOptions.scala | 4 +- .../spark/sql/avro/AvroSerializer.scala | 2 +- .../org/apache/spark/sql/avro/AvroUtils.scala | 4 +- .../spark/sql/avro/CatalystDataToAvro.scala | 2 +- .../spark/sql/avro/SchemaConverters.scala | 23 +- .../spark/sql/v2/avro/AvroDataSourceV2.scala | 6 +- .../org/apache/spark/sql/avro/AvroSuite.scala | 53 +- .../resources/mariadb_docker_entrypoint.sh | 2 +- .../spark/sql/jdbc/DB2IntegrationSuite.scala | 9 +- .../sql/jdbc/DB2KrbIntegrationSuite.scala | 11 +- .../sql/jdbc/DockerJDBCIntegrationSuite.scala | 15 +- .../sql/jdbc/MariaDBKrbIntegrationSuite.scala | 4 +- .../jdbc/MsSqlServerIntegrationSuite.scala | 10 +- .../sql/jdbc/MySQLIntegrationSuite.scala | 11 +- .../sql/jdbc/OracleIntegrationSuite.scala | 15 +- .../sql/jdbc/PostgresIntegrationSuite.scala | 49 +- .../jdbc/PostgresKrbIntegrationSuite.scala | 9 +- .../sql/jdbc/v2/DB2IntegrationSuite.scala | 76 + .../sql/jdbc/v2/MySQLIntegrationSuite.scala | 89 + .../sql/jdbc/v2/OracleIntegrationSuite.scala | 90 + .../jdbc/v2/PostgresIntegrationSuite.scala | 69 + .../apache/spark/sql/jdbc/v2/V2JDBCTest.scala | 123 + external/kafka-0-10-assembly/pom.xml | 8 +- external/kafka-0-10-sql/pom.xml | 4 + .../sql/kafka010/KafkaSourceProvider.scala | 5 +- .../kafka010/KafkaMicroBatchSourceSuite.scala | 2 +- .../spark/sql/kafka010/KafkaTestUtils.scala | 4 +- external/kafka-0-10-token-provider/pom.xml | 5 + .../spark/streaming/kafka010/KafkaRDD.scala | 4 +- external/kinesis-asl-assembly/pom.xml | 8 +- .../kinesis/KinesisBackedBlockRDD.scala | 2 +- hadoop-cloud/pom.xml | 7 +- launcher/pom.xml | 9 +- .../apache/spark/launcher/LauncherServer.java | 2 +- .../launcher/SparkSubmitCommandBuilder.java | 20 +- .../SparkSubmitCommandBuilderSuite.java | 18 + .../org/apache/spark/ml/linalg/Vectors.scala | 2 +- .../distribution/MultivariateGaussian.scala | 32 +- .../MultivariateGaussianSuite.scala | 10 - .../classification/LogisticRegression.scala | 26 +- .../spark/ml/clustering/BisectingKMeans.scala | 33 +- .../spark/ml/clustering/GaussianMixture.scala | 235 +- .../apache/spark/ml/clustering/KMeans.scala | 33 +- .../scala/org/apache/spark/ml/events.scala | 22 +- .../spark/ml/feature/CountVectorizer.scala | 5 +- .../spark/ml/feature/FeatureHasher.scala | 79 +- .../apache/spark/ml/feature/HashingTF.scala | 20 +- .../org/apache/spark/ml/feature/IDF.scala | 6 +- .../org/apache/spark/ml/feature/LSH.scala | 28 +- .../apache/spark/ml/feature/MinHashLSH.scala | 2 +- .../apache/spark/ml/feature/RFormula.scala | 2 +- .../spark/ml/feature/StringIndexer.scala | 5 +- .../spark/ml/feature/VectorIndexer.scala | 2 +- .../apache/spark/ml/feature/Word2Vec.scala | 3 +- .../org/apache/spark/ml/fpm/FPGrowth.scala | 2 +- .../org/apache/spark/ml/fpm/PrefixSpan.scala | 2 +- .../ml/regression/AFTSurvivalRegression.scala | 38 +- .../ml/source/libsvm/LibSVMRelation.scala | 4 +- .../mllib/classification/NaiveBayes.scala | 4 +- .../mllib/clustering/BisectingKMeans.scala | 47 +- .../spark/mllib/clustering/KMeans.scala | 29 +- .../org/apache/spark/mllib/feature/PCA.scala | 6 +- .../apache/spark/mllib/fpm/PrefixSpan.scala | 2 +- .../apache/spark/mllib/linalg/Vectors.scala | 2 +- .../mllib/linalg/distributed/RowMatrix.scala | 6 +- .../MatrixFactorizationModel.scala | 8 +- .../mllib/tree/model/DecisionTreeModel.scala | 2 +- .../org/apache/spark/mllib/util/MLUtils.scala | 7 +- .../org/apache/spark/SharedSparkSession.java | 3 +- .../LogisticRegressionSuite.scala | 3 +- .../ml/clustering/GaussianMixtureSuite.scala | 11 - .../apache/spark/ml/clustering/LDASuite.scala | 4 +- .../BucketedRandomProjectionLSHSuite.scala | 2 +- .../ml/feature/CountVectorizerSuite.scala | 74 +- .../org/apache/spark/ml/feature/LSHTest.scala | 3 +- .../spark/ml/feature/MinHashLSHSuite.scala | 2 +- .../apache/spark/ml/feature/NGramSuite.scala | 2 +- .../ml/feature/StopWordsRemoverSuite.scala | 8 +- .../apache/spark/ml/fpm/FPGrowthSuite.scala | 2 +- .../AFTSurvivalRegressionSuite.scala | 2 +- .../RandomForestRegressorSuite.scala | 2 +- .../source/libsvm/LibSVMRelationSuite.scala | 36 +- .../apache/spark/ml/util/MLTestSuite.scala | 2 +- .../spark/mllib/feature/Word2VecSuite.scala | 9 +- .../linalg/distributed/RowMatrixSuite.scala | 15 + .../mllib/util/MLlibTestSparkContext.scala | 2 +- pom.xml | 168 +- project/MimaBuild.scala | 17 +- project/MimaExcludes.scala | 35 +- project/SparkBuild.scala | 215 +- project/build.properties | 2 +- project/plugins.sbt | 32 +- python/MANIFEST.in | 1 + .../source/_templates/autosummary/class.rst | 38 + .../{ => autosummary}/class_with_docs.rst | 0 python/docs/source/conf.py | 21 +- .../docs/source/development/contributing.rst | 136 + python/docs/source/development/debugging.rst | 280 + python/docs/source/development/index.rst | 7 + .../docs/source/development/setting_ide.rst | 62 + python/docs/source/development/testing.rst | 57 + python/docs/source/getting_started/index.rst | 7 + .../docs/source/getting_started/install.rst | 161 + .../source/getting_started/quickstart.ipynb | 1177 + python/docs/source/index.rst | 36 + python/docs/source/migration_guide/index.rst | 13 + .../pyspark_1.0_1.2_to_1.3.rst | 23 + .../migration_guide/pyspark_1.4_to_1.5.rst | 25 + .../migration_guide/pyspark_2.2_to_2.3.rst | 30 + .../pyspark_2.3.0_to_2.3.1_above.rst | 23 + .../migration_guide/pyspark_2.3_to_2.4.rst | 23 + .../migration_guide/pyspark_2.4_to_3.0.rst | 44 + python/docs/source/reference/pyspark.ml.rst | 29 +- .../docs/source/reference/pyspark.mllib.rst | 26 +- python/docs/source/reference/pyspark.rst | 3 + python/docs/source/reference/pyspark.sql.rst | 46 +- .../docs/source/user_guide/arrow_pandas.rst | 411 + python/docs/source/user_guide/index.rst | 6 + .../source/user_guide/python_packaging.rst | 201 + python/mypy.ini | 39 + python/pyspark/__init__.py | 11 +- python/pyspark/__init__.pyi | 73 + python/pyspark/_typing.pyi | 33 + python/pyspark/accumulators.pyi | 73 + python/pyspark/broadcast.pyi | 48 + python/pyspark/conf.pyi | 44 + python/pyspark/context.py | 105 +- python/pyspark/context.pyi | 177 + python/pyspark/files.pyi | 24 + python/pyspark/find_spark_home.py | 11 +- python/pyspark/install.py | 165 + python/pyspark/ml/__init__.py | 2 +- python/pyspark/ml/_typing.pyi | 76 + python/pyspark/ml/base.py | 26 +- python/pyspark/ml/base.pyi | 103 + python/pyspark/ml/classification.py | 109 +- python/pyspark/ml/classification.pyi | 922 + python/pyspark/ml/clustering.py | 71 +- python/pyspark/ml/clustering.pyi | 437 + python/pyspark/ml/common.pyi | 20 + python/pyspark/ml/evaluation.py | 57 +- python/pyspark/ml/evaluation.pyi | 281 + python/pyspark/ml/feature.py | 336 +- python/pyspark/ml/feature.pyi | 1629 ++ python/pyspark/ml/fpm.py | 24 +- python/pyspark/ml/fpm.pyi | 109 + python/pyspark/ml/functions.pyi | 22 + python/pyspark/ml/image.pyi | 40 + python/pyspark/ml/linalg/__init__.pyi | 255 + python/pyspark/ml/param/__init__.py | 5 +- python/pyspark/ml/param/__init__.pyi | 96 + .../ml/param/_shared_params_code_gen.pyi | 19 + python/pyspark/ml/param/shared.pyi | 187 + python/pyspark/ml/pipeline.py | 16 +- python/pyspark/ml/pipeline.pyi | 97 + python/pyspark/ml/recommendation.py | 36 +- python/pyspark/ml/recommendation.pyi | 152 + python/pyspark/ml/regression.py | 100 +- python/pyspark/ml/regression.pyi | 825 + python/pyspark/ml/stat.py | 4 +- python/pyspark/ml/stat.pyi | 89 + python/pyspark/ml/tests/test_algorithms.py | 6 +- python/pyspark/ml/tests/test_base.py | 4 +- python/pyspark/ml/tests/test_evaluation.py | 4 +- python/pyspark/ml/tests/test_feature.py | 5 +- python/pyspark/ml/tests/test_image.py | 8 +- python/pyspark/ml/tests/test_linalg.py | 4 +- python/pyspark/ml/tests/test_param.py | 5 +- python/pyspark/ml/tests/test_persistence.py | 4 +- python/pyspark/ml/tests/test_pipeline.py | 4 +- python/pyspark/ml/tests/test_stat.py | 4 +- .../pyspark/ml/tests/test_training_summary.py | 5 +- python/pyspark/ml/tests/test_tuning.py | 137 +- python/pyspark/ml/tests/test_wrapper.py | 8 +- python/pyspark/ml/tree.py | 6 +- python/pyspark/ml/tree.pyi | 112 + python/pyspark/ml/tuning.py | 100 +- python/pyspark/ml/tuning.pyi | 185 + python/pyspark/ml/util.py | 1 - python/pyspark/ml/util.pyi | 128 + python/pyspark/ml/wrapper.py | 21 +- python/pyspark/ml/wrapper.pyi | 48 + python/pyspark/mllib/_typing.pyi | 23 + python/pyspark/mllib/classification.py | 5 +- python/pyspark/mllib/classification.pyi | 151 + python/pyspark/mllib/clustering.py | 2 +- python/pyspark/mllib/clustering.pyi | 196 + python/pyspark/mllib/common.pyi | 27 + python/pyspark/mllib/evaluation.pyi | 94 + python/pyspark/mllib/feature.py | 7 +- python/pyspark/mllib/feature.pyi | 167 + python/pyspark/mllib/fpm.pyi | 57 + python/pyspark/mllib/linalg/__init__.pyi | 273 + python/pyspark/mllib/linalg/distributed.pyi | 147 + python/pyspark/mllib/random.pyi | 126 + python/pyspark/mllib/recommendation.pyi | 75 + python/pyspark/mllib/regression.py | 6 +- python/pyspark/mllib/regression.pyi | 155 + python/pyspark/mllib/stat/KernelDensity.pyi | 27 + python/pyspark/mllib/stat/__init__.py | 2 +- python/pyspark/mllib/stat/__init__.pyi | 29 + python/pyspark/mllib/stat/_statistics.pyi | 69 + python/pyspark/mllib/stat/distribution.pyi | 25 + python/pyspark/mllib/stat/test.pyi | 39 + python/pyspark/mllib/tests/test_algorithms.py | 4 +- python/pyspark/mllib/tests/test_feature.py | 8 +- python/pyspark/mllib/tests/test_linalg.py | 9 +- python/pyspark/mllib/tests/test_stat.py | 7 +- .../mllib/tests/test_streaming_algorithms.py | 9 +- python/pyspark/mllib/tests/test_util.py | 6 +- python/pyspark/mllib/tree.pyi | 126 + python/pyspark/mllib/util.pyi | 90 + python/pyspark/profiler.pyi | 56 + python/pyspark/py.typed | 1 + python/pyspark/rdd.py | 6 +- python/pyspark/rdd.pyi | 479 + python/pyspark/resource/information.pyi | 26 + python/pyspark/resource/profile.pyi | 60 + python/pyspark/resource/requests.pyi | 83 + .../pyspark/resource/tests/test_resources.py | 7 +- python/pyspark/resultiterable.pyi | 30 + python/pyspark/serializers.py | 2 +- python/pyspark/shell.py | 4 +- python/pyspark/sql/__init__.pyi | 41 + python/pyspark/sql/_typing.pyi | 57 + python/pyspark/sql/avro/functions.py | 2 +- python/pyspark/sql/avro/functions.pyi | 27 + python/pyspark/sql/catalog.py | 13 +- python/pyspark/sql/catalog.pyi | 63 + python/pyspark/sql/column.py | 82 +- python/pyspark/sql/column.pyi | 113 + python/pyspark/sql/conf.pyi | 27 + python/pyspark/sql/context.py | 11 +- python/pyspark/sql/context.pyi | 139 + python/pyspark/sql/dataframe.py | 36 +- python/pyspark/sql/dataframe.pyi | 324 + python/pyspark/sql/functions.py | 968 +- python/pyspark/sql/functions.pyi | 348 + python/pyspark/sql/group.py | 2 +- python/pyspark/sql/group.pyi | 44 + .../pyspark/sql/pandas/_typing/__init__.pyi | 338 + .../sql/pandas/_typing/protocols/__init__.pyi | 17 + .../sql/pandas/_typing/protocols/frame.pyi | 428 + .../sql/pandas/_typing/protocols/series.pyi | 253 + python/pyspark/sql/pandas/conversion.py | 3 +- python/pyspark/sql/pandas/conversion.pyi | 58 + python/pyspark/sql/pandas/functions.py | 41 +- python/pyspark/sql/pandas/functions.pyi | 176 + python/pyspark/sql/pandas/group_ops.py | 4 +- python/pyspark/sql/pandas/group_ops.pyi | 49 + python/pyspark/sql/pandas/map_ops.pyi | 30 + python/pyspark/sql/pandas/serializers.py | 21 +- python/pyspark/sql/pandas/types.py | 4 +- python/pyspark/sql/pandas/utils.py | 2 +- python/pyspark/sql/readwriter.py | 9 +- python/pyspark/sql/readwriter.pyi | 250 + python/pyspark/sql/session.py | 28 +- python/pyspark/sql/session.pyi | 125 + python/pyspark/sql/streaming.py | 10 +- python/pyspark/sql/streaming.pyi | 179 + python/pyspark/sql/tests/test_arrow.py | 29 +- python/pyspark/sql/tests/test_catalog.py | 21 +- python/pyspark/sql/tests/test_column.py | 44 +- python/pyspark/sql/tests/test_conf.py | 4 +- python/pyspark/sql/tests/test_context.py | 107 +- python/pyspark/sql/tests/test_dataframe.py | 29 +- python/pyspark/sql/tests/test_datasources.py | 6 +- python/pyspark/sql/tests/test_functions.py | 203 +- python/pyspark/sql/tests/test_group.py | 4 +- .../sql/tests/test_pandas_cogrouped_map.py | 13 +- .../sql/tests/test_pandas_grouped_map.py | 28 +- python/pyspark/sql/tests/test_pandas_map.py | 8 +- python/pyspark/sql/tests/test_pandas_udf.py | 8 +- .../sql/tests/test_pandas_udf_grouped_agg.py | 9 +- .../sql/tests/test_pandas_udf_scalar.py | 16 +- .../sql/tests/test_pandas_udf_typehints.py | 6 +- .../sql/tests/test_pandas_udf_window.py | 6 +- python/pyspark/sql/tests/test_readwriter.py | 6 +- python/pyspark/sql/tests/test_serde.py | 6 +- python/pyspark/sql/tests/test_session.py | 4 +- python/pyspark/sql/tests/test_streaming.py | 25 +- python/pyspark/sql/tests/test_types.py | 26 +- python/pyspark/sql/tests/test_udf.py | 23 +- python/pyspark/sql/tests/test_utils.py | 4 +- python/pyspark/sql/types.py | 50 +- python/pyspark/sql/types.pyi | 204 + python/pyspark/sql/udf.pyi | 57 + python/pyspark/sql/window.pyi | 40 + python/pyspark/statcounter.pyi | 44 + python/pyspark/status.pyi | 42 + python/pyspark/storagelevel.py | 2 + python/pyspark/storagelevel.pyi | 43 + python/pyspark/streaming/context.pyi | 75 + python/pyspark/streaming/dstream.py | 1 - python/pyspark/streaming/dstream.pyi | 208 + python/pyspark/streaming/kinesis.pyi | 46 + python/pyspark/streaming/listener.pyi | 35 + .../pyspark/streaming/tests/test_context.py | 4 +- .../pyspark/streaming/tests/test_dstream.py | 6 +- .../pyspark/streaming/tests/test_kinesis.py | 4 +- .../pyspark/streaming/tests/test_listener.py | 4 +- python/pyspark/taskcontext.pyi | 45 + python/pyspark/testing/mlutils.py | 5 +- python/pyspark/testing/sqlutils.py | 2 +- python/pyspark/testing/streamingutils.py | 4 +- python/pyspark/testing/utils.py | 4 +- python/pyspark/tests/test_appsubmit.py | 4 +- python/pyspark/tests/test_broadcast.py | 4 +- python/pyspark/tests/test_conf.py | 4 +- python/pyspark/tests/test_context.py | 18 +- python/pyspark/tests/test_daemon.py | 4 +- python/pyspark/tests/test_install_spark.py | 109 + python/pyspark/tests/test_join.py | 4 +- python/pyspark/tests/test_pin_thread.py | 5 +- python/pyspark/tests/test_profiler.py | 4 +- python/pyspark/tests/test_rdd.py | 7 +- python/pyspark/tests/test_rddbarrier.py | 4 +- python/pyspark/tests/test_readwrite.py | 6 +- python/pyspark/tests/test_serializers.py | 15 +- python/pyspark/tests/test_shuffle.py | 5 +- python/pyspark/tests/test_taskcontext.py | 5 +- python/pyspark/tests/test_util.py | 4 +- python/pyspark/tests/test_worker.py | 7 +- python/pyspark/util.py | 2 +- python/pyspark/version.pyi | 19 + python/run-tests.py | 6 +- python/setup.py | 57 +- .../org/apache/spark/repl/Main.scala | 0 .../org/apache/spark/repl/SparkILoop.scala | 0 .../org/apache/spark/repl/Main.scala | 138 + .../org/apache/spark/repl/SparkILoop.scala | 149 + .../org/apache/spark/repl/Repl2Suite.scala | 58 + .../spark/repl/SingletonRepl2Suite.scala | 171 + .../org/apache/spark/repl/Repl2Suite.scala | 53 + .../spark/repl/SingletonRepl2Suite.scala | 171 + .../org/apache/spark/repl/ReplSuite.scala | 27 - .../spark/repl/SingletonReplSuite.scala | 61 - resource-managers/kubernetes/core/pom.xml | 2 +- .../org/apache/spark/deploy/k8s/Config.scala | 23 +- .../apache/spark/deploy/k8s/Constants.scala | 3 +- .../spark/deploy/k8s/KubernetesConf.scala | 2 +- .../deploy/k8s/KubernetesExecutorSpec.scala | 23 + .../spark/deploy/k8s/KubernetesUtils.scala | 22 +- .../deploy/k8s/KubernetesVolumeSpec.scala | 5 +- .../deploy/k8s/KubernetesVolumeUtils.scala | 18 +- .../features/BasicExecutorFeatureStep.scala | 5 + .../features/MountVolumesFeatureStep.scala | 53 +- .../features/PodTemplateConfigMapStep.scala | 6 +- .../submit/KubernetesClientApplication.scala | 26 +- .../cluster/k8s/ExecutorPodsAllocator.scala | 73 +- .../cluster/k8s/ExecutorPodsSnapshot.scala | 16 +- .../k8s/ExecutorPodsSnapshotsStoreImpl.scala | 3 + .../k8s/KubernetesClusterManager.scala | 3 + .../k8s/KubernetesExecutorBuilder.scala | 14 +- .../spark/deploy/k8s/KubernetesTestConf.scala | 7 +- .../k8s/KubernetesVolumeUtilsSuite.scala | 11 + .../BasicExecutorFeatureStepSuite.scala | 10 + .../DriverCommandFeatureStepSuite.scala | 4 +- .../MountVolumesFeatureStepSuite.scala | 84 + .../PodTemplateConfigMapStepSuite.scala | 8 +- ...erministicExecutorPodsSnapshotsStore.scala | 2 + .../k8s/ExecutorLifecycleTestUtils.scala | 37 +- .../k8s/ExecutorPodsAllocatorSuite.scala | 63 +- ...ecutorPodsPollingSnapshotSourceSuite.scala | 8 +- .../k8s/ExecutorPodsSnapshotSuite.scala | 61 +- .../k8s/ExecutorPodsSnapshotsStoreSuite.scala | 1 + ...ExecutorPodsWatchSnapshotSourceSuite.scala | 10 +- .../k8s/KubernetesExecutorBuilderSuite.scala | 2 +- .../src/main/dockerfiles/spark/Dockerfile | 2 +- .../spark/bindings/python/Dockerfile | 8 +- .../src/main/dockerfiles/spark/decom.sh | 2 +- .../src/main/dockerfiles/spark/entrypoint.sh | 7 +- .../dev/dev-run-integration-tests.sh | 2 +- .../kubernetes/integration-tests/pom.xml | 2 +- .../integrationtest/DecommissionSuite.scala | 73 +- .../k8s/integrationtest/KubernetesSuite.scala | 27 +- .../k8s/integrationtest/ProcessUtils.scala | 4 +- .../integrationtest/PythonTestsSuite.scala | 4 +- .../backend/minikube/Minikube.scala | 7 +- .../integration-tests/tests/autoscale.py | 49 + .../tests/decommissioning.py | 7 +- .../tests/decommissioning_cleanup.py | 59 + .../integration-tests/tests/pyfiles.py | 2 +- .../cluster/mesos/MesosClusterScheduler.scala | 6 +- .../mesos/MesosClusterSchedulerSuite.scala | 4 +- resource-managers/yarn/pom.xml | 65 +- .../spark/deploy/yarn/ApplicationMaster.scala | 14 +- .../org/apache/spark/deploy/yarn/Client.scala | 2 + .../deploy/yarn/BaseYarnClusterSuite.scala | 10 + .../yarn/LocalityPlacementStrategySuite.scala | 2 +- .../spark/deploy/yarn/YarnClusterSuite.scala | 16 +- .../yarn/YarnShuffleServiceMetricsSuite.scala | 2 +- .../yarn/YarnShuffleServiceSuite.scala | 1 + sbin/decommission-worker.sh | 2 +- scalastyle-config.xml | 14 + sql/catalyst/pom.xml | 8 + .../spark/sql/catalyst/parser/SqlBase.g4 | 26 +- .../spark/sql/catalyst/expressions/XXH64.java | 43 +- .../SupportsAtomicPartitionManagement.java | 85 + .../catalog/SupportsPartitionManagement.java | 115 + .../sql/connector/catalog/TableCatalog.java | 2 +- .../catalyst/expressions/AttributeMap.scala | 2 + .../catalyst/expressions/AttributeMap.scala | 2 + .../catalyst/expressions/ExpressionSet.scala | 100 - .../catalyst/util/CaseInsensitiveMap.scala | 2 + .../main/scala/org/apache/spark/sql/Row.scala | 4 +- .../sql/catalyst/CatalystTypeConverters.scala | 2 +- .../sql/catalyst/QueryPlanningTracker.scala | 2 +- .../spark/sql/catalyst/ScalaReflection.scala | 35 +- .../analysis/AlreadyExistException.scala | 33 +- .../sql/catalyst/analysis/Analyzer.scala | 439 +- .../catalyst/analysis/CTESubstitution.scala | 2 +- .../sql/catalyst/analysis/CheckAnalysis.scala | 5 +- .../catalyst/analysis/FunctionRegistry.scala | 4 +- .../analysis/NoSuchItemException.scala | 39 +- .../catalyst/analysis/ResolveCatalogs.scala | 10 - .../sql/catalyst/analysis/ResolveHints.scala | 18 +- .../analysis/ResolveInlineTables.scala | 3 +- .../analysis/ResolveNoopDropTable.scala | 33 + .../sql/catalyst/analysis/ResolveUnion.scala | 187 +- .../SubstituteUnresolvedOrdinals.scala | 3 +- .../sql/catalyst/analysis/TypeCoercion.scala | 95 +- .../UnsupportedOperationChecker.scala | 33 +- .../analysis/higherOrderFunctions.scala | 2 +- .../catalyst/analysis/timeZoneAnalysis.scala | 2 +- .../sql/catalyst/analysis/unresolved.scala | 27 +- .../catalyst/analysis/v2ResolutionPlans.scala | 2 +- .../catalog/InvalidUDFClassException.scala | 28 + .../sql/catalyst/catalog/SessionCatalog.scala | 9 +- .../sql/catalyst/catalog/interface.scala | 47 +- .../spark/sql/catalyst/csv/CSVExprUtils.scala | 11 +- .../spark/sql/catalyst/csv/CSVOptions.scala | 10 +- .../spark/sql/catalyst/dsl/package.scala | 52 +- .../sql/catalyst/encoders/RowEncoder.scala | 4 +- .../catalyst/expressions/AliasHelper.scala | 100 + .../catalyst/expressions/AttributeSet.scala | 16 +- .../expressions/CallMethodViaReflection.scala | 3 +- .../catalyst/expressions/Canonicalize.scala | 7 + .../spark/sql/catalyst/expressions/Cast.scala | 99 +- ...CodeGeneratorWithInterpretedFallback.scala | 5 +- .../sql/catalyst/expressions/Expression.scala | 8 +- .../catalyst/expressions/ExpressionSet.scala | 96 +- .../sql/catalyst/expressions/JoinedRow.scala | 10 + .../MonotonicallyIncreasingID.scala | 8 +- .../sql/catalyst/expressions/SortOrder.scala | 3 - .../expressions/SparkPartitionID.scala | 8 +- .../expressions/SpecificInternalRow.scala | 50 +- .../ApproxCountDistinctForIntervals.scala | 3 +- .../aggregate/ApproximatePercentile.scala | 24 +- .../aggregate/CentralMomentAgg.scala | 60 +- .../catalyst/expressions/aggregate/Corr.scala | 22 +- .../aggregate/CountMinSketchAgg.scala | 7 + .../expressions/aggregate/Covariance.scala | 32 +- .../aggregate/HyperLogLogPlusPlus.scala | 4 +- .../aggregate/bitwiseAggregates.scala | 1 + .../expressions/aggregate/collect.scala | 2 +- .../expressions/aggregate/interfaces.scala | 22 +- .../sql/catalyst/expressions/arithmetic.scala | 38 +- .../expressions/bitwiseExpressions.scala | 12 +- .../expressions/codegen/CodeGenerator.scala | 79 +- .../codegen/GenerateOrdering.scala | 4 +- .../codegen/GeneratePredicate.scala | 13 +- .../expressions/collectionOperations.scala | 48 +- .../expressions/complexTypeCreator.scala | 196 +- .../expressions/complexTypeExtractors.scala | 4 +- .../expressions/conditionalExpressions.scala | 6 +- .../expressions/datetimeExpressions.scala | 59 +- .../sql/catalyst/expressions/generators.scala | 12 +- .../spark/sql/catalyst/expressions/hash.scala | 18 +- .../expressions/higherOrderFunctions.scala | 2 +- .../catalyst/expressions/inputFileBlock.scala | 27 +- .../expressions/jsonExpressions.scala | 12 +- .../expressions/mathExpressions.scala | 120 +- .../spark/sql/catalyst/expressions/misc.scala | 100 +- .../expressions/objects/objects.scala | 65 +- .../sql/catalyst/expressions/predicates.scala | 95 +- .../expressions/regexpExpressions.scala | 101 +- .../expressions/stringExpressions.scala | 7 +- .../expressions/windowExpressions.scala | 258 +- .../sql/catalyst/expressions/xml/xpath.scala | 24 +- .../sql/catalyst/json/JacksonParser.scala | 9 +- .../sql/catalyst/optimizer/ComplexTypes.scala | 24 +- .../optimizer/CostBasedJoinReorder.scala | 43 +- .../optimizer/NormalizeFloatingNumbers.scala | 6 +- .../optimizer/OptimizeJsonExprs.scala | 96 + .../sql/catalyst/optimizer/Optimizer.scala | 165 +- .../optimizer/PropagateEmptyRelation.scala | 4 +- .../optimizer/PushDownLeftSemiAntiJoin.scala | 4 +- .../optimizer/RewriteDistinctAggregates.scala | 16 +- .../UnwrapCastInBinaryComparison.scala | 289 + .../sql/catalyst/optimizer/UpdateFields.scala | 91 + .../sql/catalyst/optimizer/expressions.scala | 138 +- .../catalyst/optimizer/finishAnalysis.scala | 4 +- .../spark/sql/catalyst/optimizer/joins.scala | 27 +- .../sql/catalyst/optimizer/objects.scala | 10 +- .../sql/catalyst/optimizer/subquery.scala | 73 +- .../sql/catalyst/parser/AstBuilder.scala | 35 +- .../sql/catalyst/parser/ParserUtils.scala | 5 + .../sql/catalyst/planning/patterns.scala | 2 +- ...hema.scala => DescribeCommandSchema.scala} | 10 +- .../spark/sql/catalyst/plans/QueryPlan.scala | 110 + .../plans/logical/AnalysisHelper.scala | 15 +- .../catalyst/plans/logical/LogicalPlan.scala | 76 +- .../plans/logical/QueryPlanConstraints.scala | 34 +- .../plans/logical/basicLogicalOperators.scala | 32 +- .../catalyst/plans/logical/statements.scala | 21 - .../SizeInBytesOnlyStatsPlanVisitor.scala | 3 +- .../catalyst/plans/logical/v2Commands.scala | 38 +- .../spark/sql/catalyst/rules/Rule.scala | 3 + .../sql/catalyst/rules/RuleExecutor.scala | 136 +- .../streaming/StreamingRelationV2.scala | 53 + .../spark/sql/catalyst/trees/TreeNode.scala | 42 +- .../sql/catalyst/util/ArrayBasedMapData.scala | 3 +- .../sql/catalyst/util/DateTimeUtils.scala | 5 +- .../sql/catalyst/util/GenericArrayData.scala | 8 +- .../sql/catalyst/util/IntervalUtils.scala | 18 +- .../sql/catalyst/util/QuantileSummaries.scala | 2 +- .../sql/catalyst/util/SQLOrderingUtil.scala | 41 + .../connector/catalog/CatalogManager.scala | 12 +- .../spark/sql/connector/catalog/V1Table.scala | 8 + .../apache/spark/sql/internal/SQLConf.scala | 239 +- .../spark/sql/internal/StaticSQLConf.scala | 9 + .../SupportsStreamingUpdateAsAppend.scala | 33 + .../apache/spark/sql/sources/filters.scala | 2 +- .../org/apache/spark/sql/types/DataType.scala | 8 +- .../org/apache/spark/sql/types/Decimal.scala | 56 +- .../apache/spark/sql/types/DoubleType.scala | 3 +- .../apache/spark/sql/types/FloatType.scala | 3 +- .../org/apache/spark/sql/types/Metadata.scala | 4 +- .../apache/spark/sql/types/StructType.scala | 35 + .../spark/sql/types/UserDefinedType.scala | 9 +- .../org/apache/spark/sql/types/numerics.scala | 5 +- .../apache/spark/sql/util/SchemaUtils.scala | 2 +- .../sql/catalyst/expressions/XXH64Suite.java | 91 +- .../spark/sql/RandomDataGenerator.scala | 6 +- .../sql/catalyst/ScalaReflectionSuite.scala | 15 + .../analysis/AnalysisErrorSuite.scala | 24 +- .../sql/catalyst/analysis/AnalysisSuite.scala | 55 +- .../sql/catalyst/analysis/AnalysisTest.scala | 76 +- .../analysis/DataSourceV2AnalysisSuite.scala | 46 +- .../ResolveGroupingAnalyticsSuite.scala | 2 +- .../analysis/ResolveInlineTablesSuite.scala | 26 +- .../ResolveLambdaVariablesSuite.scala | 2 +- .../ResolvedUuidExpressionsSuite.scala | 2 +- .../SubstituteUnresolvedOrdinalsSuite.scala | 20 +- .../catalyst/analysis/TypeCoercionSuite.scala | 28 +- .../analysis/UnsupportedOperationsSuite.scala | 66 +- .../encoders/ExpressionEncoderSuite.scala | 10 +- .../expressions/CanonicalizeSuite.scala | 47 + .../sql/catalyst/expressions/CastSuite.scala | 108 +- .../expressions/CodeGenerationSuite.scala | 24 +- .../expressions/ComplexTypeSuite.scala | 31 +- .../expressions/ExpressionEvalHelper.scala | 13 +- .../expressions/JsonExpressionsSuite.scala | 8 +- .../expressions/LiteralGenerator.scala | 19 +- .../expressions/MiscExpressionsSuite.scala | 30 +- .../expressions/ObjectExpressionsSuite.scala | 2 +- .../catalyst/expressions/PredicateSuite.scala | 16 + .../expressions/RegexpExpressionsSuite.scala | 14 + .../expressions/SelectedFieldSuite.scala | 2 +- ...CodegenSubexpressionEliminationSuite.scala | 73 + .../optimizer/AggregateOptimizeSuite.scala | 22 +- .../BinaryComparisonSimplificationSuite.scala | 4 +- .../BooleanSimplificationSuite.scala | 8 +- .../optimizer/CombineWithFieldsSuite.scala | 76 - .../EliminateAggregateFilterSuite.scala | 75 + .../optimizer/EliminateSortsSuite.scala | 164 +- .../optimizer/FilterPushdownSuite.scala | 57 +- .../optimizer/FoldablePropagationSuite.scala | 28 +- .../InferFiltersFromGenerateSuite.scala | 75 + .../ObjectSerializerPruningSuite.scala | 33 +- .../optimizer/OptimizeJsonExprsSuite.scala | 290 + .../optimizer/OptimizeWithFieldsSuite.scala | 129 + .../optimizer/OptimizerLoggingSuite.scala | 21 +- .../catalyst/optimizer/OptimizerSuite.scala | 74 + .../PropagateEmptyRelationSuite.scala | 6 + .../PullupCorrelatedPredicatesSuite.scala | 3 +- ...ReplaceNullWithFalseInPredicateSuite.scala | 5 +- .../optimizer/SimplifyCastsSuite.scala | 9 +- .../optimizer/SimplifyConditionalSuite.scala | 59 +- .../UnwrapCastInBinaryComparisonSuite.scala | 265 + .../optimizer/complexTypesSuite.scala | 379 +- .../joinReorder/JoinReorderPlanTestBase.scala | 82 + .../{ => joinReorder}/JoinReorderSuite.scala | 64 +- .../StarJoinCostBasedReorderSuite.scala | 46 +- .../StarJoinReorderSuite.scala | 69 +- .../sql/catalyst/parser/DDLParserSuite.scala | 60 +- .../spark/sql/catalyst/plans/PlanTest.scala | 32 +- .../sql/catalyst/plans/QueryPlanSuite.scala | 26 +- .../logical/LogicalPlanIntegritySuite.scala | 51 + .../statsEstimation/JoinEstimationSuite.scala | 22 + .../StatsEstimationTestBase.scala | 9 +- .../sql/catalyst/trees/TreeNodeSuite.scala | 26 + .../catalyst/util/DateTimeUtilsSuite.scala | 9 +- .../catalyst/util/IntervalUtilsSuite.scala | 21 + .../catalyst/util/SQLOrderingUtilSuite.scala | 75 + .../sql/catalyst/util/UnsafeArraySuite.scala | 3 +- .../InMemoryAtomicPartitionTable.scala | 76 + .../connector/InMemoryPartitionTable.scala | 95 + .../spark/sql/connector/InMemoryTable.scala | 67 +- ...pportsAtomicPartitionManagementSuite.scala | 126 + .../SupportsPartitionManagementSuite.scala | 143 + .../spark/sql/types/DataTypeSuite.scala | 110 +- .../apache/spark/sql/types/DecimalSuite.scala | 30 + .../spark/sql/types/StructTypeSuite.scala | 96 +- .../org/apache/spark/sql/types/TestUDT.scala | 73 + .../spark/sql/util/SchemaUtilsSuite.scala | 2 +- ...mpressionSchemeBenchmark-jdk11-results.txt | 168 +- .../CompressionSchemeBenchmark-results.txt | 168 +- .../DateTimeRebaseBenchmark-jdk11-results.txt | 206 +- .../DateTimeRebaseBenchmark-results.txt | 206 +- ...namicPartitionsBenchmark-jdk11-results.txt | 8 + ...WithDynamicPartitionsBenchmark-results.txt | 8 + .../UpdateFieldsBenchmark-results.txt | 26 + sql/core/pom.xml | 5 - .../datasources/orc/OrcColumnVector.java | 0 .../parquet/VectorizedColumnReader.java | 124 +- .../VectorizedParquetRecordReader.java | 23 +- .../parquet/VectorizedPlainValuesReader.java | 4 +- .../parquet/VectorizedRleValuesReader.java | 2 +- .../vectorized/OffHeapColumnVector.java | 22 +- .../vectorized/OnHeapColumnVector.java | 22 +- .../vectorized/WritableColumnVector.java | 12 + ...ache.spark.sql.jdbc.JdbcConnectionProvider | 6 + .../sql/execution/ui/static/spark-sql-viz.css | 5 + .../scala/org/apache/spark/sql/Column.scala | 105 +- .../apache/spark/sql/DataFrameReader.scala | 50 +- .../apache/spark/sql/DataFrameWriter.scala | 56 +- .../apache/spark/sql/DataFrameWriterV2.scala | 2 +- .../scala/org/apache/spark/sql/Dataset.scala | 18 +- .../spark/sql/RelationalGroupedDataset.scala | 6 +- .../org/apache/spark/sql/SQLContext.scala | 4 +- .../org/apache/spark/sql/SparkSession.scala | 91 +- .../apache/spark/sql/catalog/Catalog.scala | 79 + .../analysis/ResolveSessionCatalog.scala | 37 +- .../sql/execution/AlreadyOptimized.scala} | 31 +- .../BaseScriptTransformationExec.scala | 236 +- .../spark/sql/execution/CacheManager.scala | 29 +- .../apache/spark/sql/execution/Columnar.scala | 19 +- .../sql/execution/DataSourceScanExec.scala | 68 +- .../spark/sql/execution/GenerateExec.scala | 2 +- .../spark/sql/execution/HiveResult.scala | 2 +- .../spark/sql/execution/QueryExecution.scala | 40 +- .../execution/RemoveRedundantProjects.scala | 101 + .../sql/execution/RemoveRedundantSorts.scala | 46 + .../spark/sql/execution/SQLExecution.scala | 6 +- .../spark/sql/execution/SparkPlan.scala | 2 +- .../spark/sql/execution/SparkSqlParser.scala | 28 +- .../spark/sql/execution/SparkStrategies.scala | 14 +- .../sql/execution/SubqueryBroadcastExec.scala | 15 +- .../sql/execution/WholeStageCodegenExec.scala | 13 +- .../sql/execution/adaptive/AQEOptimizer.scala | 63 + .../adaptive/AdaptiveSparkPlanExec.scala | 160 +- .../adaptive/AdaptiveSparkPlanHelper.scala | 16 - .../adaptive/CoalesceShufflePartitions.scala | 6 +- .../adaptive/DemoteBroadcastHashJoin.scala | 2 +- .../EliminateJoinToEmptyRelation.scala | 57 + .../adaptive/InsertAdaptiveSparkPlan.scala | 2 - .../adaptive/OptimizeLocalShuffleReader.scala | 8 +- .../adaptive/OptimizeSkewedJoin.scala | 4 +- .../execution/adaptive/QueryStageExec.scala | 8 +- .../adaptive/ReuseAdaptiveSubquery.scala | 1 - .../aggregate/BaseAggregateExec.scala | 20 +- .../aggregate/HashAggregateExec.scala | 18 +- .../aggregate/ObjectHashAggregateExec.scala | 15 +- .../aggregate/SortAggregateExec.scala | 18 +- .../analysis/DetectAmbiguousSelfJoin.scala | 2 +- .../execution/basicPhysicalOperators.scala | 126 +- .../bucketing/CoalesceBucketsInJoin.scala | 2 +- .../DisableUnnecessaryBucketedScan.scala | 161 + .../compression/compressionSchemes.scala | 33 +- .../sql/execution/command/CommandUtils.scala | 2 +- .../sql/execution/command/SetCommand.scala | 2 +- .../sql/execution/command/commands.scala | 4 +- .../sql/execution/command/functions.scala | 8 +- .../spark/sql/execution/command/tables.scala | 36 +- .../spark/sql/execution/command/views.scala | 12 +- .../datasources/BasicWriteStatsTracker.scala | 16 +- .../execution/datasources/DataSource.scala | 40 +- .../datasources/DataSourceStrategy.scala | 91 +- .../datasources/DataSourceUtils.scala | 78 +- .../execution/datasources/DaysWritable.scala | 7 +- .../datasources/FallBackFileSourceV2.scala | 4 +- .../datasources/FileFormatWriter.scala | 9 +- .../datasources/FileSourceStrategy.scala | 21 +- .../datasources/InMemoryFileIndex.scala | 297 +- .../datasources/csv/CSVDataSource.scala | 2 +- .../spark/sql/execution/datasources/ddl.scala | 11 - .../datasources/jdbc/DriverRegistry.scala | 11 + .../datasources/jdbc/JDBCOptions.scala | 4 +- .../datasources/jdbc/JdbcUtils.scala | 76 +- .../connection/BasicConnectionProvider.scala | 32 +- .../jdbc/connection/ConnectionProvider.scala | 81 +- .../connection/DB2ConnectionProvider.scala | 41 +- .../connection/MSSQLConnectionProvider.scala | 43 +- .../MariaDBConnectionProvider.scala | 20 +- .../connection/OracleConnectionProvider.scala | 41 +- .../PostgresConnectionProvider.scala | 19 +- .../connection/SecureConnectionProvider.scala | 42 +- .../datasources/json/JsonDataSource.scala | 2 +- .../datasources/noop/NoopDataSource.scala | 5 +- .../datasources/orc/OrcFileFormat.scala | 18 +- .../datasources/orc/OrcFilters.scala | 78 +- .../datasources/orc/OrcFiltersBase.scala | 54 +- .../datasources/orc/OrcShimUtils.scala | 0 .../execution/datasources/orc/OrcUtils.scala | 20 +- .../parquet/ParquetFileFormat.scala | 9 +- .../parquet/ParquetReadSupport.scala | 12 +- .../parquet/ParquetRecordMaterializer.scala | 13 +- .../parquet/ParquetRowConverter.scala | 22 +- .../parquet/ParquetSchemaConverter.scala | 4 - .../parquet/ParquetWriteSupport.scala | 17 +- .../sql/execution/datasources/rules.scala | 53 +- .../datasources/v2/DataSourceV2Strategy.scala | 20 +- .../datasources/v2/FileDataSourceV2.scala | 8 + .../execution/datasources/v2/FileScan.scala | 2 +- .../execution/datasources/v2/FileTable.scala | 10 +- .../v2/ShowTablePropertiesExec.scala | 2 +- .../datasources/v2/TableCapabilityCheck.scala | 3 +- .../datasources/v2/V1FallbackWriters.scala | 8 +- .../datasources/v2/V2CommandExec.scala | 2 +- .../datasources/v2/csv/CSVDataSourceV2.scala | 6 +- .../datasources/v2/jdbc/JDBCScan.scala | 50 + .../datasources/v2/jdbc/JDBCScanBuilder.scala | 70 + .../datasources/v2/jdbc/JDBCTable.scala | 36 +- .../v2/jdbc/JDBCTableCatalog.scala | 33 +- .../v2/jdbc/JDBCWriteBuilder.scala | 46 + .../v2/json/JsonDataSourceV2.scala | 6 +- .../datasources/v2/orc/OrcDataSourceV2.scala | 6 +- .../v2/orc/OrcPartitionReaderFactory.scala | 22 +- .../datasources/v2/orc/OrcScan.scala | 2 +- .../datasources/v2/orc/OrcScanBuilder.scala | 12 +- .../v2/parquet/ParquetDataSourceV2.scala | 7 +- .../ParquetPartitionReaderFactory.scala | 24 +- .../v2/text/TextDataSourceV2.scala | 6 +- .../dynamicpruning/PartitionPruning.scala | 4 +- .../PlanDynamicPruningFilters.scala | 15 +- .../exchange/BroadcastExchangeExec.scala | 5 +- .../exchange/EnsureRequirements.scala | 75 +- .../sql/execution/exchange/Exchange.scala | 2 +- .../joins/BroadcastHashJoinExec.scala | 24 +- .../spark/sql/execution/joins/HashJoin.scala | 73 +- .../sql/execution/joins/HashedRelation.scala | 226 +- .../joins/ShuffledHashJoinExec.scala | 246 +- .../sql/execution/joins/ShuffledJoin.scala | 23 +- .../execution/joins/SortMergeJoinExec.scala | 42 +- .../streaming/CompactibleFileStreamLog.scala | 97 +- .../streaming/FileStreamSinkLog.scala | 10 - .../streaming/FileStreamSource.scala | 60 +- .../streaming/FileStreamSourceLog.scala | 6 +- .../execution/streaming/HDFSMetadataLog.scala | 80 +- .../streaming/MicroBatchExecution.scala | 35 +- .../execution/streaming/StreamExecution.scala | 8 +- .../streaming/StreamingRelation.scala | 31 - .../StreamingSymmetricHashJoinExec.scala | 163 +- .../sql/execution/streaming/console.scala | 7 +- .../continuous/ContinuousExecution.scala | 8 +- .../sql/execution/streaming/memory.scala | 7 +- .../sources/ForeachWriterTable.scala | 7 +- .../execution/streaming/sources/memory.scala | 7 +- .../state/SymmetricHashJoinStateManager.scala | 15 +- .../apache/spark/sql/execution/subquery.scala | 30 +- .../sql/execution/ui/ExecutionPage.scala | 2 +- .../execution/ui/SQLAppStatusListener.scala | 9 +- .../sql/execution/window/WindowExec.scala | 8 +- .../sql/execution/window/WindowExecBase.scala | 43 +- .../window/WindowFunctionFrame.scala | 123 +- .../spark/sql/expressions/Aggregator.scala | 2 + .../org/apache/spark/sql/functions.scala | 124 +- .../internal/BaseSessionStateBuilder.scala | 22 +- .../spark/sql/internal/CatalogImpl.scala | 42 +- .../spark/sql/internal/SessionState.scala | 7 +- .../spark/sql/internal/SharedState.scala | 15 +- .../apache/spark/sql/jdbc/DB2Dialect.scala | 21 + .../org/apache/spark/sql/jdbc/H2Dialect.scala | 48 + .../sql/jdbc/JdbcConnectionProvider.scala | 64 + .../apache/spark/sql/jdbc/JdbcDialects.scala | 81 +- .../apache/spark/sql/jdbc/MySQLDialect.scala | 24 +- .../apache/spark/sql/jdbc/OracleDialect.scala | 28 + .../spark/sql/jdbc/PostgresDialect.scala | 18 +- .../scala/org/apache/spark/sql/package.scala | 6 + .../sql/streaming/DataStreamReader.scala | 54 +- .../sql/streaming/DataStreamWriter.scala | 177 +- .../api/v1/sql/ApiSqlRootResource.scala | 9 +- .../sort/RecordBinaryComparatorSuite.java | 48 +- ...ache.spark.sql.jdbc.JdbcConnectionProvider | 1 + .../sql-functions/sql-expression-schema.md | 102 +- .../test/resources/sql-tests/inputs/cast.sql | 5 + .../test/resources/sql-tests/inputs/count.sql | 10 + .../resources/sql-tests/inputs/datetime.sql | 2 + .../resources/sql-tests/inputs/except.sql | 19 + .../sql-tests/inputs/explain-aqe.sql | 1 + .../resources/sql-tests/inputs/explain.sql | 6 + .../sql-tests/inputs/group-by-filter.sql | 8 + .../resources/sql-tests/inputs/group-by.sql | 3 + .../resources/sql-tests/inputs/having.sql | 6 + .../inputs/higher-order-functions.sql | 3 + .../sql-tests/inputs/intersect-all.sql | 15 + .../resources/sql-tests/inputs/interval.sql | 20 + .../sql-tests/inputs/misc-functions.sql | 12 + .../inputs/postgreSQL/window_part1.sql | 4 +- .../inputs/postgreSQL/window_part2.sql | 4 +- .../inputs/postgreSQL/window_part3.sql | 3 +- .../sql-tests/inputs/regexp-functions.sql | 12 + .../test/resources/sql-tests/inputs/union.sql | 14 + .../resources/sql-tests/inputs/window.sql | 124 +- .../sql-tests/results/ansi/datetime.sql.out | 40 +- .../ansi/higher-order-functions.sql.out | 10 +- .../sql-tests/results/ansi/interval.sql.out | 240 +- .../resources/sql-tests/results/cast.sql.out | 44 +- .../resources/sql-tests/results/count.sql.out | 66 +- .../sql-tests/results/datetime-legacy.sql.out | 30 +- .../sql-tests/results/datetime.sql.out | 30 +- .../sql-tests/results/except.sql.out | 58 +- .../sql-tests/results/explain-aqe.sql.out | 309 +- .../sql-tests/results/explain.sql.out | 346 +- .../sql-tests/results/group-by-filter.sql.out | 66 +- .../sql-tests/results/group-by.sql.out | 10 +- .../sql-tests/results/having.sql.out | 32 + .../results/higher-order-functions.sql.out | 10 +- .../sql-tests/results/intersect-all.sql.out | 42 +- .../sql-tests/results/interval.sql.out | 218 +- .../sql-tests/results/json-functions.sql.out | 6 +- .../sql-tests/results/misc-functions.sql.out | 81 +- .../postgreSQL/aggregates_part1.sql.out | 4 +- .../results/postgreSQL/window_part3.sql.out | 11 +- .../results/postgreSQL/window_part4.sql.out | 32 +- .../results/regexp-functions.sql.out | 84 +- .../sql-compatibility-functions.sql.out | 2 +- .../native/promoteStrings.sql.out | 8 +- .../postgreSQL/udf-aggregates_part1.sql.out | 4 +- .../sql-tests/results/udf/udf-window.sql.out | 8 +- .../resources/sql-tests/results/union.sql.out | 43 +- .../sql-tests/results/window.sql.out | 355 +- .../file-sink-log-version-2.1.0/8 | 1 - .../test-data/percentile_approx-input.csv.bz2 | Bin 0 -> 124614 bytes .../src/test/resources/test_script.py | 0 .../q10.sf100/explain.txt | 286 + .../q10.sf100/simplified.txt | 81 + .../approved-plans-modified/q10/explain.txt | 266 + .../q10/simplified.txt | 71 + .../q19.sf100/explain.txt | 221 + .../q19.sf100/simplified.txt | 58 + .../approved-plans-modified/q19/explain.txt | 221 + .../q19/simplified.txt | 58 + .../q27.sf100/explain.txt | 428 + .../q27.sf100/simplified.txt | 113 + .../approved-plans-modified/q27/explain.txt | 428 + .../q27/simplified.txt | 113 + .../q3.sf100/explain.txt | 122 + .../q3.sf100/simplified.txt | 31 + .../approved-plans-modified/q3/explain.txt | 122 + .../approved-plans-modified/q3/simplified.txt | 31 + .../q34.sf100/explain.txt | 218 + .../q34.sf100/simplified.txt | 63 + .../approved-plans-modified/q34/explain.txt | 203 + .../q34/simplified.txt | 54 + .../q42.sf100/explain.txt | 122 + .../q42.sf100/simplified.txt | 31 + .../approved-plans-modified/q42/explain.txt | 122 + .../q42/simplified.txt | 31 + .../q43.sf100/explain.txt | 122 + .../q43.sf100/simplified.txt | 31 + .../approved-plans-modified/q43/explain.txt | 122 + .../q43/simplified.txt | 31 + .../q46.sf100/explain.txt | 281 + .../q46.sf100/simplified.txt | 87 + .../approved-plans-modified/q46/explain.txt | 241 + .../q46/simplified.txt | 63 + .../q52.sf100/explain.txt | 122 + .../q52.sf100/simplified.txt | 31 + .../approved-plans-modified/q52/explain.txt | 122 + .../q52/simplified.txt | 31 + .../q53.sf100/explain.txt | 180 + .../q53.sf100/simplified.txt | 49 + .../approved-plans-modified/q53/explain.txt | 180 + .../q53/simplified.txt | 49 + .../q55.sf100/explain.txt | 122 + .../q55.sf100/simplified.txt | 31 + .../approved-plans-modified/q55/explain.txt | 122 + .../q55/simplified.txt | 31 + .../q59.sf100/explain.txt | 290 + .../q59.sf100/simplified.txt | 76 + .../approved-plans-modified/q59/explain.txt | 290 + .../q59/simplified.txt | 76 + .../q63.sf100/explain.txt | 180 + .../q63.sf100/simplified.txt | 49 + .../approved-plans-modified/q63/explain.txt | 180 + .../q63/simplified.txt | 49 + .../q65.sf100/explain.txt | 245 + .../q65.sf100/simplified.txt | 63 + .../approved-plans-modified/q65/explain.txt | 245 + .../q65/simplified.txt | 63 + .../q68.sf100/explain.txt | 289 + .../q68.sf100/simplified.txt | 86 + .../approved-plans-modified/q68/explain.txt | 241 + .../q68/simplified.txt | 63 + .../q7.sf100/explain.txt | 193 + .../q7.sf100/simplified.txt | 50 + .../approved-plans-modified/q7/explain.txt | 193 + .../approved-plans-modified/q7/simplified.txt | 50 + .../q73.sf100/explain.txt | 203 + .../q73.sf100/simplified.txt | 54 + .../approved-plans-modified/q73/explain.txt | 203 + .../q73/simplified.txt | 54 + .../q79.sf100/explain.txt | 208 + .../q79.sf100/simplified.txt | 59 + .../approved-plans-modified/q79/explain.txt | 193 + .../q79/simplified.txt | 50 + .../q89.sf100/explain.txt | 175 + .../q89.sf100/simplified.txt | 48 + .../approved-plans-modified/q89/explain.txt | 175 + .../q89/simplified.txt | 48 + .../q98.sf100/explain.txt | 162 + .../q98.sf100/simplified.txt | 51 + .../approved-plans-modified/q98/explain.txt | 147 + .../q98/simplified.txt | 42 + .../ss_max.sf100/explain.txt | 56 + .../ss_max.sf100/simplified.txt | 14 + .../ss_max/explain.txt | 56 + .../ss_max/simplified.txt | 14 + .../approved-plans-v1_4/q1.sf100/explain.txt | 270 + .../q1.sf100/simplified.txt | 74 + .../approved-plans-v1_4/q1/explain.txt | 255 + .../approved-plans-v1_4/q1/simplified.txt | 65 + .../approved-plans-v1_4/q10.sf100/explain.txt | 319 + .../q10.sf100/simplified.txt | 95 + .../approved-plans-v1_4/q10/explain.txt | 279 + .../approved-plans-v1_4/q10/simplified.txt | 74 + .../approved-plans-v1_4/q11.sf100/explain.txt | 482 + .../q11.sf100/simplified.txt | 158 + .../approved-plans-v1_4/q11/explain.txt | 415 + .../approved-plans-v1_4/q11/simplified.txt | 108 + .../approved-plans-v1_4/q12.sf100/explain.txt | 152 + .../q12.sf100/simplified.txt | 47 + .../approved-plans-v1_4/q12/explain.txt | 137 + .../approved-plans-v1_4/q12/simplified.txt | 38 + .../approved-plans-v1_4/q13.sf100/explain.txt | 216 + .../q13.sf100/simplified.txt | 57 + .../approved-plans-v1_4/q13/explain.txt | 216 + .../approved-plans-v1_4/q13/simplified.txt | 57 + .../q14a.sf100/explain.txt | 878 + .../q14a.sf100/simplified.txt | 254 + .../approved-plans-v1_4/q14a/explain.txt | 798 + .../approved-plans-v1_4/q14a/simplified.txt | 214 + .../q14b.sf100/explain.txt | 810 + .../q14b.sf100/simplified.txt | 231 + .../approved-plans-v1_4/q14b/explain.txt | 763 + .../approved-plans-v1_4/q14b/simplified.txt | 204 + .../approved-plans-v1_4/q15.sf100/explain.txt | 180 + .../q15.sf100/simplified.txt | 57 + .../approved-plans-v1_4/q15/explain.txt | 150 + .../approved-plans-v1_4/q15/simplified.txt | 39 + .../approved-plans-v1_4/q16.sf100/explain.txt | 250 + .../q16.sf100/simplified.txt | 71 + .../approved-plans-v1_4/q16/explain.txt | 235 + .../approved-plans-v1_4/q16/simplified.txt | 62 + .../approved-plans-v1_4/q17.sf100/explain.txt | 314 + .../q17.sf100/simplified.txt | 98 + .../approved-plans-v1_4/q17/explain.txt | 269 + .../approved-plans-v1_4/q17/simplified.txt | 71 + .../approved-plans-v1_4/q18.sf100/explain.txt | 294 + .../q18.sf100/simplified.txt | 87 + .../approved-plans-v1_4/q18/explain.txt | 264 + .../approved-plans-v1_4/q18/simplified.txt | 69 + .../approved-plans-v1_4/q19.sf100/explain.txt | 251 + .../q19.sf100/simplified.txt | 76 + .../approved-plans-v1_4/q19/explain.txt | 221 + .../approved-plans-v1_4/q19/simplified.txt | 58 + .../approved-plans-v1_4/q2.sf100/explain.txt | 233 + .../q2.sf100/simplified.txt | 70 + .../approved-plans-v1_4/q2/explain.txt | 218 + .../approved-plans-v1_4/q2/simplified.txt | 61 + .../approved-plans-v1_4/q20.sf100/explain.txt | 152 + .../q20.sf100/simplified.txt | 47 + .../approved-plans-v1_4/q20/explain.txt | 137 + .../approved-plans-v1_4/q20/simplified.txt | 38 + .../approved-plans-v1_4/q21.sf100/explain.txt | 155 + .../q21.sf100/simplified.txt | 40 + .../approved-plans-v1_4/q21/explain.txt | 155 + .../approved-plans-v1_4/q21/simplified.txt | 40 + .../approved-plans-v1_4/q22.sf100/explain.txt | 170 + .../q22.sf100/simplified.txt | 49 + .../approved-plans-v1_4/q22/explain.txt | 155 + .../approved-plans-v1_4/q22/simplified.txt | 40 + .../q23a.sf100/explain.txt | 655 + .../q23a.sf100/simplified.txt | 198 + .../approved-plans-v1_4/q23a/explain.txt | 542 + .../approved-plans-v1_4/q23a/simplified.txt | 143 + .../q23b.sf100/explain.txt | 870 + .../q23b.sf100/simplified.txt | 268 + .../approved-plans-v1_4/q23b/explain.txt | 689 + .../approved-plans-v1_4/q23b/simplified.txt | 182 + .../q24a.sf100/explain.txt | 567 + .../q24a.sf100/simplified.txt | 179 + .../approved-plans-v1_4/q24a/explain.txt | 477 + .../approved-plans-v1_4/q24a/simplified.txt | 125 + .../q24b.sf100/explain.txt | 567 + .../q24b.sf100/simplified.txt | 179 + .../approved-plans-v1_4/q24b/explain.txt | 477 + .../approved-plans-v1_4/q24b/simplified.txt | 125 + .../approved-plans-v1_4/q25.sf100/explain.txt | 314 + .../q25.sf100/simplified.txt | 98 + .../approved-plans-v1_4/q25/explain.txt | 269 + .../approved-plans-v1_4/q25/simplified.txt | 71 + .../approved-plans-v1_4/q26.sf100/explain.txt | 193 + .../q26.sf100/simplified.txt | 50 + .../approved-plans-v1_4/q26/explain.txt | 193 + .../approved-plans-v1_4/q26/simplified.txt | 50 + .../approved-plans-v1_4/q27.sf100/explain.txt | 193 + .../q27.sf100/simplified.txt | 50 + .../approved-plans-v1_4/q27/explain.txt | 193 + .../approved-plans-v1_4/q27/simplified.txt | 50 + .../approved-plans-v1_4/q28.sf100/explain.txt | 437 + .../q28.sf100/simplified.txt | 107 + .../approved-plans-v1_4/q28/explain.txt | 437 + .../approved-plans-v1_4/q28/simplified.txt | 107 + .../approved-plans-v1_4/q29.sf100/explain.txt | 337 + .../q29.sf100/simplified.txt | 104 + .../approved-plans-v1_4/q29/explain.txt | 292 + .../approved-plans-v1_4/q29/simplified.txt | 77 + .../approved-plans-v1_4/q3.sf100/explain.txt | 122 + .../q3.sf100/simplified.txt | 31 + .../approved-plans-v1_4/q3/explain.txt | 122 + .../approved-plans-v1_4/q3/simplified.txt | 31 + .../approved-plans-v1_4/q30.sf100/explain.txt | 333 + .../q30.sf100/simplified.txt | 96 + .../approved-plans-v1_4/q30/explain.txt | 303 + .../approved-plans-v1_4/q30/simplified.txt | 78 + .../approved-plans-v1_4/q31.sf100/explain.txt | 663 + .../q31.sf100/simplified.txt | 206 + .../approved-plans-v1_4/q31/explain.txt | 563 + .../approved-plans-v1_4/q31/simplified.txt | 150 + .../approved-plans-v1_4/q32.sf100/explain.txt | 175 + .../q32.sf100/simplified.txt | 45 + .../approved-plans-v1_4/q32/explain.txt | 175 + .../approved-plans-v1_4/q32/simplified.txt | 45 + .../approved-plans-v1_4/q33.sf100/explain.txt | 378 + .../q33.sf100/simplified.txt | 101 + .../approved-plans-v1_4/q33/explain.txt | 378 + .../approved-plans-v1_4/q33/simplified.txt | 101 + .../approved-plans-v1_4/q34.sf100/explain.txt | 218 + .../q34.sf100/simplified.txt | 63 + .../approved-plans-v1_4/q34/explain.txt | 203 + .../approved-plans-v1_4/q34/simplified.txt | 54 + .../approved-plans-v1_4/q35.sf100/explain.txt | 329 + .../q35.sf100/simplified.txt | 103 + .../approved-plans-v1_4/q35/explain.txt | 274 + .../approved-plans-v1_4/q35/simplified.txt | 73 + .../approved-plans-v1_4/q36.sf100/explain.txt | 180 + .../q36.sf100/simplified.txt | 49 + .../approved-plans-v1_4/q36/explain.txt | 180 + .../approved-plans-v1_4/q36/simplified.txt | 49 + .../approved-plans-v1_4/q37.sf100/explain.txt | 175 + .../q37.sf100/simplified.txt | 50 + .../approved-plans-v1_4/q37/explain.txt | 160 + .../approved-plans-v1_4/q37/simplified.txt | 41 + .../approved-plans-v1_4/q38.sf100/explain.txt | 393 + .../q38.sf100/simplified.txt | 118 + .../approved-plans-v1_4/q38/explain.txt | 328 + .../approved-plans-v1_4/q38/simplified.txt | 81 + .../q39a.sf100/explain.txt | 307 + .../q39a.sf100/simplified.txt | 86 + .../approved-plans-v1_4/q39a/explain.txt | 292 + .../approved-plans-v1_4/q39a/simplified.txt | 77 + .../q39b.sf100/explain.txt | 307 + .../q39b.sf100/simplified.txt | 86 + .../approved-plans-v1_4/q39b/explain.txt | 292 + .../approved-plans-v1_4/q39b/simplified.txt | 77 + .../approved-plans-v1_4/q4.sf100/explain.txt | 695 + .../q4.sf100/simplified.txt | 231 + .../approved-plans-v1_4/q4/explain.txt | 606 + .../approved-plans-v1_4/q4/simplified.txt | 158 + .../approved-plans-v1_4/q40.sf100/explain.txt | 198 + .../q40.sf100/simplified.txt | 56 + .../approved-plans-v1_4/q40/explain.txt | 183 + .../approved-plans-v1_4/q40/simplified.txt | 48 + .../approved-plans-v1_4/q41.sf100/explain.txt | 120 + .../q41.sf100/simplified.txt | 29 + .../approved-plans-v1_4/q41/explain.txt | 120 + .../approved-plans-v1_4/q41/simplified.txt | 29 + .../approved-plans-v1_4/q42.sf100/explain.txt | 122 + .../q42.sf100/simplified.txt | 31 + .../approved-plans-v1_4/q42/explain.txt | 122 + .../approved-plans-v1_4/q42/simplified.txt | 31 + .../approved-plans-v1_4/q43.sf100/explain.txt | 122 + .../q43.sf100/simplified.txt | 31 + .../approved-plans-v1_4/q43/explain.txt | 122 + .../approved-plans-v1_4/q43/simplified.txt | 31 + .../approved-plans-v1_4/q44.sf100/explain.txt | 248 + .../q44.sf100/simplified.txt | 68 + .../approved-plans-v1_4/q44/explain.txt | 248 + .../approved-plans-v1_4/q44/simplified.txt | 68 + .../approved-plans-v1_4/q45.sf100/explain.txt | 256 + .../q45.sf100/simplified.txt | 77 + .../approved-plans-v1_4/q45/explain.txt | 226 + .../approved-plans-v1_4/q45/simplified.txt | 59 + .../approved-plans-v1_4/q46.sf100/explain.txt | 281 + .../q46.sf100/simplified.txt | 87 + .../approved-plans-v1_4/q46/explain.txt | 241 + .../approved-plans-v1_4/q46/simplified.txt | 63 + .../approved-plans-v1_4/q47.sf100/explain.txt | 313 + .../q47.sf100/simplified.txt | 107 + .../approved-plans-v1_4/q47/explain.txt | 278 + .../approved-plans-v1_4/q47/simplified.txt | 84 + .../approved-plans-v1_4/q48.sf100/explain.txt | 183 + .../q48.sf100/simplified.txt | 48 + .../approved-plans-v1_4/q48/explain.txt | 183 + .../approved-plans-v1_4/q48/simplified.txt | 48 + .../approved-plans-v1_4/q49.sf100/explain.txt | 478 + .../q49.sf100/simplified.txt | 153 + .../approved-plans-v1_4/q49/explain.txt | 433 + .../approved-plans-v1_4/q49/simplified.txt | 126 + .../approved-plans-v1_4/q5.sf100/explain.txt | 450 + .../q5.sf100/simplified.txt | 132 + .../approved-plans-v1_4/q5/explain.txt | 435 + .../approved-plans-v1_4/q5/simplified.txt | 123 + .../approved-plans-v1_4/q50.sf100/explain.txt | 198 + .../q50.sf100/simplified.txt | 57 + .../approved-plans-v1_4/q50/explain.txt | 183 + .../approved-plans-v1_4/q50/simplified.txt | 48 + .../approved-plans-v1_4/q51.sf100/explain.txt | 228 + .../q51.sf100/simplified.txt | 71 + .../approved-plans-v1_4/q51/explain.txt | 228 + .../approved-plans-v1_4/q51/simplified.txt | 71 + .../approved-plans-v1_4/q52.sf100/explain.txt | 122 + .../q52.sf100/simplified.txt | 31 + .../approved-plans-v1_4/q52/explain.txt | 122 + .../approved-plans-v1_4/q52/simplified.txt | 31 + .../approved-plans-v1_4/q53.sf100/explain.txt | 180 + .../q53.sf100/simplified.txt | 49 + .../approved-plans-v1_4/q53/explain.txt | 180 + .../approved-plans-v1_4/q53/simplified.txt | 49 + .../approved-plans-v1_4/q54.sf100/explain.txt | 494 + .../q54.sf100/simplified.txt | 142 + .../approved-plans-v1_4/q54/explain.txt | 459 + .../approved-plans-v1_4/q54/simplified.txt | 121 + .../approved-plans-v1_4/q55.sf100/explain.txt | 122 + .../q55.sf100/simplified.txt | 31 + .../approved-plans-v1_4/q55/explain.txt | 122 + .../approved-plans-v1_4/q55/simplified.txt | 31 + .../approved-plans-v1_4/q56.sf100/explain.txt | 378 + .../q56.sf100/simplified.txt | 101 + .../approved-plans-v1_4/q56/explain.txt | 378 + .../approved-plans-v1_4/q56/simplified.txt | 101 + .../approved-plans-v1_4/q57.sf100/explain.txt | 313 + .../q57.sf100/simplified.txt | 107 + .../approved-plans-v1_4/q57/explain.txt | 278 + .../approved-plans-v1_4/q57/simplified.txt | 84 + .../approved-plans-v1_4/q58.sf100/explain.txt | 477 + .../q58.sf100/simplified.txt | 125 + .../approved-plans-v1_4/q58/explain.txt | 477 + .../approved-plans-v1_4/q58/simplified.txt | 125 + .../approved-plans-v1_4/q59.sf100/explain.txt | 249 + .../q59.sf100/simplified.txt | 66 + .../approved-plans-v1_4/q59/explain.txt | 249 + .../approved-plans-v1_4/q59/simplified.txt | 66 + .../approved-plans-v1_4/q6.sf100/explain.txt | 331 + .../q6.sf100/simplified.txt | 95 + .../approved-plans-v1_4/q6/explain.txt | 301 + .../approved-plans-v1_4/q6/simplified.txt | 77 + .../approved-plans-v1_4/q60.sf100/explain.txt | 378 + .../q60.sf100/simplified.txt | 101 + .../approved-plans-v1_4/q60/explain.txt | 378 + .../approved-plans-v1_4/q60/simplified.txt | 101 + .../approved-plans-v1_4/q61.sf100/explain.txt | 381 + .../q61.sf100/simplified.txt | 101 + .../approved-plans-v1_4/q61/explain.txt | 396 + .../approved-plans-v1_4/q61/simplified.txt | 105 + .../approved-plans-v1_4/q62.sf100/explain.txt | 183 + .../q62.sf100/simplified.txt | 48 + .../approved-plans-v1_4/q62/explain.txt | 183 + .../approved-plans-v1_4/q62/simplified.txt | 48 + .../approved-plans-v1_4/q63.sf100/explain.txt | 180 + .../q63.sf100/simplified.txt | 49 + .../approved-plans-v1_4/q63/explain.txt | 180 + .../approved-plans-v1_4/q63/simplified.txt | 49 + .../approved-plans-v1_4/q64.sf100/explain.txt | 1110 + .../q64.sf100/simplified.txt | 367 + .../approved-plans-v1_4/q64/explain.txt | 918 + .../approved-plans-v1_4/q64/simplified.txt | 246 + .../approved-plans-v1_4/q65.sf100/explain.txt | 260 + .../q65.sf100/simplified.txt | 72 + .../approved-plans-v1_4/q65/explain.txt | 245 + .../approved-plans-v1_4/q65/simplified.txt | 63 + .../approved-plans-v1_4/q66.sf100/explain.txt | 310 + .../q66.sf100/simplified.txt | 83 + .../approved-plans-v1_4/q66/explain.txt | 310 + .../approved-plans-v1_4/q66/simplified.txt | 83 + .../approved-plans-v1_4/q67.sf100/explain.txt | 190 + .../q67.sf100/simplified.txt | 57 + .../approved-plans-v1_4/q67/explain.txt | 175 + .../approved-plans-v1_4/q67/simplified.txt | 48 + .../approved-plans-v1_4/q68.sf100/explain.txt | 281 + .../q68.sf100/simplified.txt | 87 + .../approved-plans-v1_4/q68/explain.txt | 241 + .../approved-plans-v1_4/q68/simplified.txt | 63 + .../approved-plans-v1_4/q69.sf100/explain.txt | 299 + .../q69.sf100/simplified.txt | 85 + .../approved-plans-v1_4/q69/explain.txt | 274 + .../approved-plans-v1_4/q69/simplified.txt | 73 + .../approved-plans-v1_4/q7.sf100/explain.txt | 193 + .../q7.sf100/simplified.txt | 50 + .../approved-plans-v1_4/q7/explain.txt | 193 + .../approved-plans-v1_4/q7/simplified.txt | 50 + .../approved-plans-v1_4/q70.sf100/explain.txt | 264 + .../q70.sf100/simplified.txt | 74 + .../approved-plans-v1_4/q70/explain.txt | 264 + .../approved-plans-v1_4/q70/simplified.txt | 74 + .../approved-plans-v1_4/q71.sf100/explain.txt | 232 + .../q71.sf100/simplified.txt | 65 + .../approved-plans-v1_4/q71/explain.txt | 232 + .../approved-plans-v1_4/q71/simplified.txt | 65 + .../approved-plans-v1_4/q72.sf100/explain.txt | 436 + .../q72.sf100/simplified.txt | 130 + .../approved-plans-v1_4/q72/explain.txt | 391 + .../approved-plans-v1_4/q72/simplified.txt | 104 + .../approved-plans-v1_4/q73.sf100/explain.txt | 218 + .../q73.sf100/simplified.txt | 63 + .../approved-plans-v1_4/q73/explain.txt | 203 + .../approved-plans-v1_4/q73/simplified.txt | 54 + .../approved-plans-v1_4/q74.sf100/explain.txt | 477 + .../q74.sf100/simplified.txt | 157 + .../approved-plans-v1_4/q74/explain.txt | 410 + .../approved-plans-v1_4/q74/simplified.txt | 107 + .../approved-plans-v1_4/q75.sf100/explain.txt | 752 + .../q75.sf100/simplified.txt | 237 + .../approved-plans-v1_4/q75/explain.txt | 647 + .../approved-plans-v1_4/q75/simplified.txt | 180 + .../approved-plans-v1_4/q76.sf100/explain.txt | 245 + .../q76.sf100/simplified.txt | 68 + .../approved-plans-v1_4/q76/explain.txt | 209 + .../approved-plans-v1_4/q76/simplified.txt | 58 + .../approved-plans-v1_4/q77.sf100/explain.txt | 520 + .../q77.sf100/simplified.txt | 139 + .../approved-plans-v1_4/q77/explain.txt | 520 + .../approved-plans-v1_4/q77/simplified.txt | 139 + .../approved-plans-v1_4/q78.sf100/explain.txt | 391 + .../q78.sf100/simplified.txt | 117 + .../approved-plans-v1_4/q78/explain.txt | 341 + .../approved-plans-v1_4/q78/simplified.txt | 88 + .../approved-plans-v1_4/q79.sf100/explain.txt | 208 + .../q79.sf100/simplified.txt | 59 + .../approved-plans-v1_4/q79/explain.txt | 193 + .../approved-plans-v1_4/q79/simplified.txt | 50 + .../approved-plans-v1_4/q8.sf100/explain.txt | 302 + .../q8.sf100/simplified.txt | 88 + .../approved-plans-v1_4/q8/explain.txt | 272 + .../approved-plans-v1_4/q8/simplified.txt | 70 + .../approved-plans-v1_4/q80.sf100/explain.txt | 598 + .../q80.sf100/simplified.txt | 172 + .../approved-plans-v1_4/q80/explain.txt | 553 + .../approved-plans-v1_4/q80/simplified.txt | 148 + .../approved-plans-v1_4/q81.sf100/explain.txt | 343 + .../q81.sf100/simplified.txt | 104 + .../approved-plans-v1_4/q81/explain.txt | 298 + .../approved-plans-v1_4/q81/simplified.txt | 77 + .../approved-plans-v1_4/q82.sf100/explain.txt | 175 + .../q82.sf100/simplified.txt | 50 + .../approved-plans-v1_4/q82/explain.txt | 160 + .../approved-plans-v1_4/q82/simplified.txt | 41 + .../approved-plans-v1_4/q83.sf100/explain.txt | 344 + .../q83.sf100/simplified.txt | 91 + .../approved-plans-v1_4/q83/explain.txt | 344 + .../approved-plans-v1_4/q83/simplified.txt | 91 + .../approved-plans-v1_4/q84.sf100/explain.txt | 200 + .../q84.sf100/simplified.txt | 53 + .../approved-plans-v1_4/q84/explain.txt | 200 + .../approved-plans-v1_4/q84/simplified.txt | 53 + .../approved-plans-v1_4/q85.sf100/explain.txt | 317 + .../q85.sf100/simplified.txt | 94 + .../approved-plans-v1_4/q85/explain.txt | 287 + .../approved-plans-v1_4/q85/simplified.txt | 76 + .../approved-plans-v1_4/q86.sf100/explain.txt | 142 + .../q86.sf100/simplified.txt | 39 + .../approved-plans-v1_4/q86/explain.txt | 142 + .../approved-plans-v1_4/q86/simplified.txt | 39 + .../approved-plans-v1_4/q87.sf100/explain.txt | 388 + .../q87.sf100/simplified.txt | 117 + .../approved-plans-v1_4/q87/explain.txt | 323 + .../approved-plans-v1_4/q87/simplified.txt | 80 + .../approved-plans-v1_4/q88.sf100/explain.txt | 960 + .../q88.sf100/simplified.txt | 250 + .../approved-plans-v1_4/q88/explain.txt | 960 + .../approved-plans-v1_4/q88/simplified.txt | 250 + .../approved-plans-v1_4/q89.sf100/explain.txt | 175 + .../q89.sf100/simplified.txt | 48 + .../approved-plans-v1_4/q89/explain.txt | 175 + .../approved-plans-v1_4/q89/simplified.txt | 48 + .../approved-plans-v1_4/q9.sf100/explain.txt | 718 + .../q9.sf100/simplified.txt | 186 + .../approved-plans-v1_4/q9/explain.txt | 718 + .../approved-plans-v1_4/q9/simplified.txt | 186 + .../approved-plans-v1_4/q90.sf100/explain.txt | 280 + .../q90.sf100/simplified.txt | 74 + .../approved-plans-v1_4/q90/explain.txt | 280 + .../approved-plans-v1_4/q90/simplified.txt | 74 + .../approved-plans-v1_4/q91.sf100/explain.txt | 264 + .../q91.sf100/simplified.txt | 71 + .../approved-plans-v1_4/q91/explain.txt | 264 + .../approved-plans-v1_4/q91/simplified.txt | 71 + .../approved-plans-v1_4/q92.sf100/explain.txt | 196 + .../q92.sf100/simplified.txt | 50 + .../approved-plans-v1_4/q92/explain.txt | 196 + .../approved-plans-v1_4/q92/simplified.txt | 50 + .../approved-plans-v1_4/q93.sf100/explain.txt | 126 + .../q93.sf100/simplified.txt | 38 + .../approved-plans-v1_4/q93/explain.txt | 111 + .../approved-plans-v1_4/q93/simplified.txt | 29 + .../approved-plans-v1_4/q94.sf100/explain.txt | 265 + .../q94.sf100/simplified.txt | 78 + .../approved-plans-v1_4/q94/explain.txt | 235 + .../approved-plans-v1_4/q94/simplified.txt | 62 + .../approved-plans-v1_4/q95.sf100/explain.txt | 347 + .../q95.sf100/simplified.txt | 111 + .../approved-plans-v1_4/q95/explain.txt | 318 + .../approved-plans-v1_4/q95/simplified.txt | 84 + .../approved-plans-v1_4/q96.sf100/explain.txt | 160 + .../q96.sf100/simplified.txt | 41 + .../approved-plans-v1_4/q96/explain.txt | 160 + .../approved-plans-v1_4/q96/simplified.txt | 41 + .../approved-plans-v1_4/q97.sf100/explain.txt | 179 + .../q97.sf100/simplified.txt | 46 + .../approved-plans-v1_4/q97/explain.txt | 179 + .../approved-plans-v1_4/q97/simplified.txt | 46 + .../approved-plans-v1_4/q98.sf100/explain.txt | 162 + .../q98.sf100/simplified.txt | 51 + .../approved-plans-v1_4/q98/explain.txt | 147 + .../approved-plans-v1_4/q98/simplified.txt | 42 + .../approved-plans-v1_4/q99.sf100/explain.txt | 183 + .../q99.sf100/simplified.txt | 48 + .../approved-plans-v1_4/q99/explain.txt | 183 + .../approved-plans-v1_4/q99/simplified.txt | 48 + .../q10a.sf100/explain.txt | 286 + .../q10a.sf100/simplified.txt | 81 + .../approved-plans-v2_7/q10a/explain.txt | 266 + .../approved-plans-v2_7/q10a/simplified.txt | 71 + .../approved-plans-v2_7/q11.sf100/explain.txt | 477 + .../q11.sf100/simplified.txt | 157 + .../approved-plans-v2_7/q11/explain.txt | 410 + .../approved-plans-v2_7/q11/simplified.txt | 107 + .../approved-plans-v2_7/q12.sf100/explain.txt | 152 + .../q12.sf100/simplified.txt | 47 + .../approved-plans-v2_7/q12/explain.txt | 137 + .../approved-plans-v2_7/q12/simplified.txt | 38 + .../approved-plans-v2_7/q14.sf100/explain.txt | 810 + .../q14.sf100/simplified.txt | 231 + .../approved-plans-v2_7/q14/explain.txt | 763 + .../approved-plans-v2_7/q14/simplified.txt | 204 + .../q14a.sf100/explain.txt | 1460 ++ .../q14a.sf100/simplified.txt | 427 + .../approved-plans-v2_7/q14a/explain.txt | 1380 ++ .../approved-plans-v2_7/q14a/simplified.txt | 387 + .../q18a.sf100/explain.txt | 877 + .../q18a.sf100/simplified.txt | 262 + .../approved-plans-v2_7/q18a/explain.txt | 856 + .../approved-plans-v2_7/q18a/simplified.txt | 227 + .../approved-plans-v2_7/q20.sf100/explain.txt | 152 + .../q20.sf100/simplified.txt | 47 + .../approved-plans-v2_7/q20/explain.txt | 137 + .../approved-plans-v2_7/q20/simplified.txt | 38 + .../approved-plans-v2_7/q22.sf100/explain.txt | 157 + .../q22.sf100/simplified.txt | 48 + .../approved-plans-v2_7/q22/explain.txt | 142 + .../approved-plans-v2_7/q22/simplified.txt | 39 + .../q22a.sf100/explain.txt | 316 + .../q22a.sf100/simplified.txt | 87 + .../approved-plans-v2_7/q22a/explain.txt | 301 + .../approved-plans-v2_7/q22a/simplified.txt | 78 + .../approved-plans-v2_7/q24.sf100/explain.txt | 532 + .../q24.sf100/simplified.txt | 156 + .../approved-plans-v2_7/q24/explain.txt | 487 + .../approved-plans-v2_7/q24/simplified.txt | 129 + .../q27a.sf100/explain.txt | 428 + .../q27a.sf100/simplified.txt | 113 + .../approved-plans-v2_7/q27a/explain.txt | 428 + .../approved-plans-v2_7/q27a/simplified.txt | 113 + .../approved-plans-v2_7/q34.sf100/explain.txt | 218 + .../q34.sf100/simplified.txt | 63 + .../approved-plans-v2_7/q34/explain.txt | 203 + .../approved-plans-v2_7/q34/simplified.txt | 54 + .../approved-plans-v2_7/q35.sf100/explain.txt | 329 + .../q35.sf100/simplified.txt | 103 + .../approved-plans-v2_7/q35/explain.txt | 274 + .../approved-plans-v2_7/q35/simplified.txt | 73 + .../q35a.sf100/explain.txt | 311 + .../q35a.sf100/simplified.txt | 98 + .../approved-plans-v2_7/q35a/explain.txt | 261 + .../approved-plans-v2_7/q35a/simplified.txt | 70 + .../q36a.sf100/explain.txt | 289 + .../q36a.sf100/simplified.txt | 82 + .../approved-plans-v2_7/q36a/explain.txt | 289 + .../approved-plans-v2_7/q36a/simplified.txt | 82 + .../approved-plans-v2_7/q47.sf100/explain.txt | 313 + .../q47.sf100/simplified.txt | 107 + .../approved-plans-v2_7/q47/explain.txt | 278 + .../approved-plans-v2_7/q47/simplified.txt | 84 + .../approved-plans-v2_7/q49.sf100/explain.txt | 478 + .../q49.sf100/simplified.txt | 153 + .../approved-plans-v2_7/q49/explain.txt | 433 + .../approved-plans-v2_7/q49/simplified.txt | 126 + .../q51a.sf100/explain.txt | 441 + .../q51a.sf100/simplified.txt | 139 + .../approved-plans-v2_7/q51a/explain.txt | 426 + .../approved-plans-v2_7/q51a/simplified.txt | 126 + .../approved-plans-v2_7/q57.sf100/explain.txt | 313 + .../q57.sf100/simplified.txt | 107 + .../approved-plans-v2_7/q57/explain.txt | 278 + .../approved-plans-v2_7/q57/simplified.txt | 84 + .../approved-plans-v2_7/q5a.sf100/explain.txt | 559 + .../q5a.sf100/simplified.txt | 165 + .../approved-plans-v2_7/q5a/explain.txt | 544 + .../approved-plans-v2_7/q5a/simplified.txt | 156 + .../approved-plans-v2_7/q6.sf100/explain.txt | 331 + .../q6.sf100/simplified.txt | 95 + .../approved-plans-v2_7/q6/explain.txt | 301 + .../approved-plans-v2_7/q6/simplified.txt | 77 + .../approved-plans-v2_7/q64.sf100/explain.txt | 1110 + .../q64.sf100/simplified.txt | 367 + .../approved-plans-v2_7/q64/explain.txt | 918 + .../approved-plans-v2_7/q64/simplified.txt | 246 + .../q67a.sf100/explain.txt | 452 + .../q67a.sf100/simplified.txt | 129 + .../approved-plans-v2_7/q67a/explain.txt | 437 + .../approved-plans-v2_7/q67a/simplified.txt | 120 + .../q70a.sf100/explain.txt | 373 + .../q70a.sf100/simplified.txt | 107 + .../approved-plans-v2_7/q70a/explain.txt | 373 + .../approved-plans-v2_7/q70a/simplified.txt | 107 + .../approved-plans-v2_7/q72.sf100/explain.txt | 436 + .../q72.sf100/simplified.txt | 130 + .../approved-plans-v2_7/q72/explain.txt | 391 + .../approved-plans-v2_7/q72/simplified.txt | 104 + .../approved-plans-v2_7/q74.sf100/explain.txt | 477 + .../q74.sf100/simplified.txt | 157 + .../approved-plans-v2_7/q74/explain.txt | 410 + .../approved-plans-v2_7/q74/simplified.txt | 107 + .../approved-plans-v2_7/q75.sf100/explain.txt | 752 + .../q75.sf100/simplified.txt | 237 + .../approved-plans-v2_7/q75/explain.txt | 647 + .../approved-plans-v2_7/q75/simplified.txt | 180 + .../q77a.sf100/explain.txt | 629 + .../q77a.sf100/simplified.txt | 172 + .../approved-plans-v2_7/q77a/explain.txt | 629 + .../approved-plans-v2_7/q77a/simplified.txt | 172 + .../approved-plans-v2_7/q78.sf100/explain.txt | 391 + .../q78.sf100/simplified.txt | 117 + .../approved-plans-v2_7/q78/explain.txt | 341 + .../approved-plans-v2_7/q78/simplified.txt | 88 + .../q80a.sf100/explain.txt | 707 + .../q80a.sf100/simplified.txt | 205 + .../approved-plans-v2_7/q80a/explain.txt | 662 + .../approved-plans-v2_7/q80a/simplified.txt | 181 + .../q86a.sf100/explain.txt | 251 + .../q86a.sf100/simplified.txt | 72 + .../approved-plans-v2_7/q86a/explain.txt | 251 + .../approved-plans-v2_7/q86a/simplified.txt | 72 + .../approved-plans-v2_7/q98.sf100/explain.txt | 157 + .../q98.sf100/simplified.txt | 50 + .../approved-plans-v2_7/q98/explain.txt | 142 + .../approved-plans-v2_7/q98/simplified.txt | 41 + .../sql/ApproximatePercentileQuerySuite.scala | 21 +- .../org/apache/spark/sql/CTEHintSuite.scala | 168 + .../apache/spark/sql/CachedTableSuite.scala | 24 + .../spark/sql/ColumnExpressionSuite.scala | 993 +- .../spark/sql/DataFrameAggregateSuite.scala | 70 +- .../apache/spark/sql/DataFrameJoinSuite.scala | 41 +- .../sql/DataFrameSetOperationsSuite.scala | 181 + .../apache/spark/sql/DataFrameStatSuite.scala | 4 +- .../org/apache/spark/sql/DataFrameSuite.scala | 26 + .../sql/DataFrameWindowFunctionsSuite.scala | 313 +- .../spark/sql/DataFrameWriterV2Suite.scala | 87 +- .../spark/sql/DatasetOptimizationSuite.scala | 13 +- .../org/apache/spark/sql/DatasetSuite.scala | 15 +- .../apache/spark/sql/DeprecatedAPISuite.scala | 6 +- .../sql/DynamicPartitionPruningSuite.scala | 103 +- .../org/apache/spark/sql/ExplainSuite.scala | 15 + .../spark/sql/ExpressionsSchemaSuite.scala | 45 +- .../spark/sql/FileBasedDataSourceSuite.scala | 167 +- .../org/apache/spark/sql/JoinSuite.scala | 136 +- .../apache/spark/sql/JsonFunctionsSuite.scala | 33 +- .../apache/spark/sql/LocalSparkSession.scala | 4 +- .../apache/spark/sql/MathFunctionsSuite.scala | 15 + .../apache/spark/sql/PlanStabilitySuite.scala | 334 + .../apache/spark/sql/SQLContextSuite.scala | 2 +- .../org/apache/spark/sql/SQLQuerySuite.scala | 123 +- .../apache/spark/sql/SQLQueryTestSuite.scala | 20 +- .../apache/spark/sql/SessionStateSuite.scala | 2 +- .../spark/sql/SparkSessionBuilderSuite.scala | 91 +- .../sql/SparkSessionExtensionSuite.scala | 2 +- .../org/apache/spark/sql/SubquerySuite.scala | 2 +- .../{TPCDSSchema.scala => TPCDSBase.scala} | 66 +- .../apache/spark/sql/TPCDSQuerySuite.scala | 64 +- .../apache/spark/sql/TPCDSTableStats.scala | 503 + .../scala/org/apache/spark/sql/UDFSuite.scala | 2 +- .../UnwrapCastInComparisonEndToEndSuite.scala | 194 + .../spark/sql/UpdateFieldsBenchmark.scala | 224 + .../spark/sql/UserDefinedTypeSuite.scala | 75 +- .../DataSourceV2DataFrameSuite.scala | 18 + .../sql/connector/DataSourceV2SQLSuite.scala | 73 +- .../sql/connector/DataSourceV2Suite.scala | 31 +- .../SupportsCatalogOptionsSuite.scala | 17 + .../connector/TableCapabilityCheckSuite.scala | 9 +- .../connector/TestV2SessionCatalogBase.scala | 30 +- .../sql/connector/V1WriteFallbackSuite.scala | 42 + .../V2CommandsCaseSensitivitySuite.scala | 2 +- .../sql/execution/AlreadyOptimizedSuite.scala | 85 + .../BaseScriptTransformationSuite.scala | 492 + .../CoalesceShufflePartitionsSuite.scala | 4 +- .../sql/execution/ColumnarRulesSuite.scala | 63 + .../spark/sql/execution/PlannerSuite.scala | 44 +- .../sql/execution/QueryExecutionSuite.scala | 13 + .../RemoveRedundantProjectsSuite.scala | 175 + .../execution/RemoveRedundantSortsSuite.scala | 144 + .../sql/execution/SQLExecutionSuite.scala | 40 +- .../spark/sql/execution/SQLViewSuite.scala | 2 +- .../spark/sql/execution/SortSuite.scala | 13 + .../sql/execution/SparkSqlParserSuite.scala | 94 +- .../TestUncaughtExceptionHandler.scala | 2 +- .../adaptive/AdaptiveQueryExecSuite.scala | 171 +- .../benchmark/DateTimeRebaseBenchmark.scala | 3 +- ...tTableWithDynamicPartitionsBenchmark.scala | 103 + .../benchmark/TPCDSQueryBenchmark.scala | 3 +- .../CoalesceBucketsInJoinSuite.scala | 2 +- .../compression/BooleanBitSetSuite.scala | 26 + .../sql/execution/command/DDLSuite.scala | 116 +- .../command/PlanResolutionSuite.scala | 20 +- ...BasicWriteJobStatsTrackerMetricSuite.scala | 59 + .../CommonFileDataSourceSuite.scala | 62 + .../datasources/DataSourceSuite.scala | 18 +- .../datasources/FileBasedDataSourceTest.scala | 40 +- .../datasources/FileSourceStrategySuite.scala | 81 +- .../execution/datasources/csv/CSVSuite.scala | 61 +- .../jdbc/DriverRegistrySuite.scala | 29 + .../connection/ConnectionProviderSuite.scala | 52 +- .../ConnectionProviderSuiteBase.scala | 19 +- .../DB2ConnectionProviderSuite.scala | 6 +- ...ntentionallyFaultyConnectionProvider.scala | 34 + .../MSSQLConnectionProviderSuite.scala | 42 +- .../MariaDBConnectionProviderSuite.scala | 6 +- .../OracleConnectionProviderSuite.scala | 7 +- .../PostgresConnectionProviderSuite.scala | 16 +- .../jdbc/connection/TestDriver.scala | 33 + .../datasources/json/JsonSuite.scala | 48 +- .../datasources/orc/OrcFilterSuite.scala | 673 + .../datasources/orc/OrcQuerySuite.scala | 7 +- .../datasources/orc/OrcSourceSuite.scala | 13 +- .../execution/datasources/orc/OrcTest.scala | 23 +- .../parquet/ParquetFileFormatSuite.scala | 25 +- .../parquet/ParquetFilterSuite.scala | 35 +- .../datasources/parquet/ParquetIOSuite.scala | 149 +- .../ParquetInteroperabilitySuite.scala | 2 +- .../parquet/ParquetSchemaPruningSuite.scala | 3 + .../parquet/ParquetSchemaSuite.scala | 19 + .../datasources/text/TextSuite.scala | 5 +- .../v2/jdbc/JDBCTableCatalogSuite.scala | 289 +- .../exchange/EnsureRequirementsSuite.scala | 122 + .../execution/joins/BroadcastJoinSuite.scala | 40 +- .../execution/joins/ExistenceJoinSuite.scala | 20 +- .../execution/joins/HashedRelationSuite.scala | 134 + .../sql/execution/joins/InnerJoinSuite.scala | 6 +- .../sql/execution/joins/OuterJoinSuite.scala | 4 +- .../execution/metric/SQLMetricsSuite.scala | 72 +- .../metric/SQLMetricsTestUtils.scala | 20 +- .../CompactibleFileStreamLogSuite.scala | 2 - .../streaming/FileStreamSinkLogSuite.scala | 35 +- .../streaming/MicroBatchExecutionSuite.scala | 123 + .../state/StateStoreCoordinatorSuite.scala | 2 +- .../streaming/state/StateStoreSuite.scala | 2 +- .../SymmetricHashJoinStateManagerSuite.scala | 2 +- .../ui/SQLAppStatusListenerSuite.scala | 132 + .../vectorized/ColumnarBatchSuite.scala | 116 +- .../sql/expressions/ExpressionInfoSuite.scala | 52 +- .../spark/sql/internal/CatalogSuite.scala | 4 + .../spark/sql/internal/SQLConfSuite.scala | 6 +- .../org/apache/spark/sql/jdbc/JDBCSuite.scala | 13 +- .../apache/spark/sql/jdbc/JDBCV2Suite.scala | 224 + .../spark/sql/jdbc/JDBCWriteSuite.scala | 37 +- .../spark/sql/sources/BucketedReadSuite.scala | 64 +- .../sql/sources/DataSourceAnalysisSuite.scala | 40 +- .../DisableUnnecessaryBucketedScanSuite.scala | 244 + .../spark/sql/sources/InsertSuite.scala | 54 +- .../sql/streaming/FileStreamSourceSuite.scala | 159 +- .../FlatMapGroupsWithStateSuite.scala | 52 +- .../spark/sql/streaming/StreamSuite.scala | 9 +- .../spark/sql/streaming/StreamTest.scala | 9 + .../streaming/StreamingAggregationSuite.scala | 45 +- .../StreamingDeduplicationSuite.scala | 42 + .../sql/streaming/StreamingJoinSuite.scala | 502 +- .../sql/streaming/StreamingQuerySuite.scala | 91 +- .../test/DataStreamReaderWriterSuite.scala | 125 +- .../test/DataStreamTableAPISuite.scala | 376 + .../sql/test/DataFrameReaderWriterSuite.scala | 124 +- .../apache/spark/sql/test/SQLTestUtils.scala | 2 +- .../spark/sql/test/SharedSparkSession.scala | 4 +- .../spark/sql/test/TestSQLContext.scala | 11 +- .../status/api/v1/sql/SqlResourceSuite.scala | 5 +- .../SqlResourceWithActualMetricsSuite.scala | 127 + .../datasources/orc/OrcColumnVector.java | 208 - .../datasources/orc/DaysWritable.scala | 79 - .../datasources/orc/OrcFilters.scala | 273 - .../datasources/orc/OrcShimUtils.scala | 66 - .../datasources/orc/OrcFilterSuite.scala | 473 - .../datasources/orc/OrcFilterSuite.scala | 474 - sql/hive-thriftserver/pom.xml | 38 +- .../apache/hive/service/AbstractService.java | 0 .../apache/hive/service/CompositeService.java | 0 .../org/apache/hive/service/CookieSigner.java | 0 .../hive/service/ServiceOperations.java | 0 .../org/apache/hive/service/ServiceUtils.java | 0 .../hive/service/auth/HiveAuthFactory.java | 0 .../hive/service/auth/HttpAuthUtils.java | 0 .../hive/service/auth/KerberosSaslHelper.java | 0 .../hive/service/auth/PlainSaslHelper.java | 0 .../service/auth/TSetIpAddressProcessor.java | 0 .../apache/hive/service/cli/CLIService.java | 0 .../hive/service/cli/ColumnBasedSet.java | 0 .../hive/service/cli/ColumnDescriptor.java | 0 .../apache/hive/service/cli/ColumnValue.java | 0 .../hive/service/cli/FetchOrientation.java | 0 .../apache/hive/service/cli/GetInfoType.java | 0 .../apache/hive/service/cli/GetInfoValue.java | 0 .../org/apache/hive/service/cli/Handle.java | 0 .../hive/service/cli/HandleIdentifier.java | 0 .../hive/service/cli/HiveSQLException.java | 0 .../apache/hive/service/cli/ICLIService.java | 0 .../hive/service/cli/OperationHandle.java | 0 .../hive/service/cli/OperationState.java | 0 .../hive/service/cli/OperationType.java | 0 .../apache/hive/service/cli/RowBasedSet.java | 0 .../org/apache/hive/service/cli/RowSet.java | 0 .../hive/service/cli/RowSetFactory.java | 0 .../hive/service/cli/SessionHandle.java | 0 .../apache/hive/service/cli/TableSchema.java | 0 .../hive/service/cli/TypeDescriptor.java | 0 .../hive/service/cli/TypeQualifiers.java | 0 .../operation/ClassicTableTypeMapping.java | 0 .../operation/ExecuteStatementOperation.java | 0 .../cli/operation/GetCatalogsOperation.java | 0 .../cli/operation/GetColumnsOperation.java | 0 .../operation/GetCrossReferenceOperation.java | 0 .../cli/operation/GetFunctionsOperation.java | 0 .../operation/GetPrimaryKeysOperation.java | 0 .../cli/operation/GetSchemasOperation.java | 0 .../cli/operation/GetTableTypesOperation.java | 0 .../cli/operation/GetTablesOperation.java | 0 .../cli/operation/GetTypeInfoOperation.java | 0 .../cli/operation/HiveCommandOperation.java | 0 .../cli/operation/HiveTableTypeMapping.java | 0 .../cli/operation/MetadataOperation.java | 0 .../hive/service/cli/operation/Operation.java | 0 .../cli/operation/OperationManager.java | 4 +- .../service/cli/operation/SQLOperation.java | 7 +- .../cli/operation/TableTypeMapping.java | 0 .../hive/service/cli/session/HiveSession.java | 0 .../service/cli/session/HiveSessionBase.java | 0 .../cli/session/HiveSessionHookContext.java | 0 .../session/HiveSessionHookContextImpl.java | 0 .../service/cli/session/HiveSessionImpl.java | 0 .../cli/session/HiveSessionImplwithUGI.java | 0 .../service/cli/session/SessionManager.java | 0 .../cli/thrift/ThriftBinaryCLIService.java | 0 .../service/cli/thrift/ThriftCLIService.java | 0 .../cli/thrift/ThriftCLIServiceClient.java | 0 .../cli/thrift/ThriftHttpCLIService.java | 0 .../service/cli/thrift/ThriftHttpServlet.java | 0 .../hive/service/server/HiveServer2.java | 0 .../server/ThreadWithGarbageCleanup.java | 0 .../hive/thriftserver/HiveThriftServer2.scala | 2 +- .../SparkExecuteStatementOperation.scala | 102 +- .../SparkGetColumnsOperation.scala | 29 +- .../SparkGetFunctionsOperation.scala | 5 +- .../SparkGetSchemasOperation.scala | 3 +- .../SparkGetTablesOperation.scala | 6 +- .../SparkGetTypeInfoOperation.scala | 14 +- .../hive/thriftserver/SparkOperation.scala | 41 +- .../hive/thriftserver/SparkSQLCLIDriver.scala | 6 +- .../thriftserver/SparkSQLCLIService.scala | 11 +- .../thriftserver/SparkSQLSessionManager.scala | 3 +- .../server/SparkSQLOperationManager.scala | 5 +- .../ui/HiveThriftServer2AppStatusStore.scala | 1 + .../ui/HiveThriftServer2EventManager.scala | 7 + .../ui/HiveThriftServer2Listener.scala | 10 + ...HiveMetastoreLazyInitializationSuite.scala | 2 + .../sql/hive/thriftserver/CliSuite.scala | 6 +- .../GetCatalogsOperationMock.scala | 0 .../thriftserver/HiveSessionImplSuite.scala | 3 +- .../HiveThriftServer2Suites.scala | 93 +- .../thriftserver/SharedThriftServer.scala | 3 +- .../SparkExecuteStatementOperationSuite.scala | 6 +- .../SparkMetadataOperationSuite.scala | 411 +- ...arkThriftServerProtocolVersionsSuite.scala | 179 +- .../ThriftServerQueryTestSuite.scala | 11 +- .../ThriftServerWithSparkContextSuite.scala | 190 +- .../ui/HiveThriftServer2ListenerSuite.scala | 1 + .../v1.2/if/TCLIService.thrift | 1173 - .../service/cli/thrift/TArrayTypeEntry.java | 383 - .../service/cli/thrift/TBinaryColumn.java | 550 - .../hive/service/cli/thrift/TBoolColumn.java | 548 - .../hive/service/cli/thrift/TBoolValue.java | 386 - .../hive/service/cli/thrift/TByteColumn.java | 548 - .../hive/service/cli/thrift/TByteValue.java | 386 - .../hive/service/cli/thrift/TCLIService.java | 15414 ------------- .../cli/thrift/TCLIServiceConstants.java | 103 - .../cli/thrift/TCancelDelegationTokenReq.java | 491 - .../thrift/TCancelDelegationTokenResp.java | 390 - .../cli/thrift/TCancelOperationReq.java | 390 - .../cli/thrift/TCancelOperationResp.java | 390 - .../cli/thrift/TCloseOperationReq.java | 390 - .../cli/thrift/TCloseOperationResp.java | 390 - .../service/cli/thrift/TCloseSessionReq.java | 390 - .../service/cli/thrift/TCloseSessionResp.java | 390 - .../hive/service/cli/thrift/TColumn.java | 732 - .../hive/service/cli/thrift/TColumnDesc.java | 700 - .../hive/service/cli/thrift/TColumnValue.java | 671 - .../service/cli/thrift/TDoubleColumn.java | 548 - .../hive/service/cli/thrift/TDoubleValue.java | 386 - .../cli/thrift/TExecuteStatementReq.java | 769 - .../cli/thrift/TExecuteStatementResp.java | 505 - .../service/cli/thrift/TFetchOrientation.java | 57 - .../service/cli/thrift/TFetchResultsReq.java | 710 - .../service/cli/thrift/TFetchResultsResp.java | 608 - .../service/cli/thrift/TGetCatalogsReq.java | 390 - .../service/cli/thrift/TGetCatalogsResp.java | 505 - .../service/cli/thrift/TGetColumnsReq.java | 818 - .../service/cli/thrift/TGetColumnsResp.java | 505 - .../cli/thrift/TGetDelegationTokenReq.java | 592 - .../cli/thrift/TGetDelegationTokenResp.java | 500 - .../service/cli/thrift/TGetFunctionsReq.java | 707 - .../service/cli/thrift/TGetFunctionsResp.java | 505 - .../hive/service/cli/thrift/TGetInfoReq.java | 503 - .../hive/service/cli/thrift/TGetInfoResp.java | 493 - .../hive/service/cli/thrift/TGetInfoType.java | 180 - .../service/cli/thrift/TGetInfoValue.java | 593 - .../cli/thrift/TGetOperationStatusReq.java | 390 - .../cli/thrift/TGetOperationStatusResp.java | 827 - .../cli/thrift/TGetResultSetMetadataReq.java | 390 - .../cli/thrift/TGetResultSetMetadataResp.java | 505 - .../service/cli/thrift/TGetSchemasReq.java | 606 - .../service/cli/thrift/TGetSchemasResp.java | 505 - .../service/cli/thrift/TGetTableTypesReq.java | 390 - .../cli/thrift/TGetTableTypesResp.java | 505 - .../service/cli/thrift/TGetTablesReq.java | 870 - .../service/cli/thrift/TGetTablesResp.java | 505 - .../service/cli/thrift/TGetTypeInfoReq.java | 390 - .../service/cli/thrift/TGetTypeInfoResp.java | 505 - .../service/cli/thrift/THandleIdentifier.java | 506 - .../hive/service/cli/thrift/TI16Column.java | 548 - .../hive/service/cli/thrift/TI16Value.java | 386 - .../hive/service/cli/thrift/TI32Column.java | 548 - .../hive/service/cli/thrift/TI32Value.java | 386 - .../hive/service/cli/thrift/TI64Column.java | 548 - .../hive/service/cli/thrift/TI64Value.java | 386 - .../service/cli/thrift/TMapTypeEntry.java | 478 - .../service/cli/thrift/TOpenSessionReq.java | 785 - .../service/cli/thrift/TOpenSessionResp.java | 790 - .../service/cli/thrift/TOperationHandle.java | 705 - .../service/cli/thrift/TOperationState.java | 63 - .../service/cli/thrift/TOperationType.java | 66 - .../cli/thrift/TPrimitiveTypeEntry.java | 512 - .../service/cli/thrift/TProtocolVersion.java | 63 - .../cli/thrift/TRenewDelegationTokenReq.java | 491 - .../cli/thrift/TRenewDelegationTokenResp.java | 390 - .../apache/hive/service/cli/thrift/TRow.java | 439 - .../hive/service/cli/thrift/TRowSet.java | 702 - .../service/cli/thrift/TSessionHandle.java | 390 - .../hive/service/cli/thrift/TStatus.java | 874 - .../hive/service/cli/thrift/TStatusCode.java | 54 - .../service/cli/thrift/TStringColumn.java | 548 - .../hive/service/cli/thrift/TStringValue.java | 389 - .../service/cli/thrift/TStructTypeEntry.java | 448 - .../hive/service/cli/thrift/TTableSchema.java | 439 - .../hive/service/cli/thrift/TTypeDesc.java | 439 - .../hive/service/cli/thrift/TTypeEntry.java | 610 - .../hive/service/cli/thrift/TTypeId.java | 105 - .../cli/thrift/TTypeQualifierValue.java | 361 - .../service/cli/thrift/TTypeQualifiers.java | 450 - .../service/cli/thrift/TUnionTypeEntry.java | 448 - .../cli/thrift/TUserDefinedTypeEntry.java | 385 - .../apache/hive/service/AbstractService.java | 184 - .../apache/hive/service/CompositeService.java | 133 - .../org/apache/hive/service/CookieSigner.java | 108 - .../hive/service/ServiceOperations.java | 141 - .../org/apache/hive/service/ServiceUtils.java | 44 - .../hive/service/auth/HiveAuthFactory.java | 419 - .../hive/service/auth/HttpAuthUtils.java | 189 - .../hive/service/auth/KerberosSaslHelper.java | 111 - .../hive/service/auth/PlainSaslHelper.java | 154 - .../service/auth/TSetIpAddressProcessor.java | 114 - .../apache/hive/service/cli/CLIService.java | 507 - .../org/apache/hive/service/cli/Column.java | 423 - .../hive/service/cli/ColumnBasedSet.java | 149 - .../hive/service/cli/ColumnDescriptor.java | 99 - .../apache/hive/service/cli/ColumnValue.java | 288 - .../service/cli/EmbeddedCLIServiceClient.java | 208 - .../hive/service/cli/FetchOrientation.java | 54 - .../apache/hive/service/cli/GetInfoType.java | 96 - .../apache/hive/service/cli/GetInfoValue.java | 82 - .../org/apache/hive/service/cli/Handle.java | 78 - .../hive/service/cli/HandleIdentifier.java | 113 - .../hive/service/cli/HiveSQLException.java | 249 - .../apache/hive/service/cli/ICLIService.java | 105 - .../hive/service/cli/OperationHandle.java | 102 - .../hive/service/cli/OperationState.java | 108 - .../hive/service/cli/OperationType.java | 58 - .../hive/service/cli/PatternOrIdentifier.java | 47 - .../apache/hive/service/cli/RowBasedSet.java | 140 - .../org/apache/hive/service/cli/RowSet.java | 38 - .../hive/service/cli/RowSetFactory.java | 41 - .../hive/service/cli/SessionHandle.java | 67 - .../apache/hive/service/cli/TableSchema.java | 102 - .../org/apache/hive/service/cli/Type.java | 349 - .../hive/service/cli/TypeDescriptor.java | 159 - .../hive/service/cli/TypeQualifiers.java | 133 - .../operation/ClassicTableTypeMapping.java | 86 - .../operation/ExecuteStatementOperation.java | 83 - .../cli/operation/GetCatalogsOperation.java | 81 - .../cli/operation/GetColumnsOperation.java | 234 - .../cli/operation/GetFunctionsOperation.java | 147 - .../cli/operation/GetSchemasOperation.java | 96 - .../cli/operation/GetTableTypesOperation.java | 93 - .../cli/operation/GetTablesOperation.java | 135 - .../cli/operation/GetTypeInfoOperation.java | 142 - .../cli/operation/HiveCommandOperation.java | 215 - .../cli/operation/HiveTableTypeMapping.java | 51 - .../cli/operation/MetadataOperation.java | 134 - .../hive/service/cli/operation/Operation.java | 328 - .../cli/operation/OperationManager.java | 284 - .../service/cli/operation/SQLOperation.java | 456 - .../cli/operation/TableTypeMapping.java | 44 - .../hive/service/cli/session/HiveSession.java | 156 - .../service/cli/session/HiveSessionBase.java | 90 - .../service/cli/session/HiveSessionImpl.java | 842 - .../cli/session/HiveSessionImplwithUGI.java | 182 - .../service/cli/session/SessionManager.java | 377 - .../cli/thrift/ThriftBinaryCLIService.java | 121 - .../service/cli/thrift/ThriftCLIService.java | 693 - .../cli/thrift/ThriftCLIServiceClient.java | 440 - .../cli/thrift/ThriftHttpCLIService.java | 194 - .../service/cli/thrift/ThriftHttpServlet.java | 545 - .../hive/service/server/HiveServer2.java | 277 - .../server/ThreadWithGarbageCleanup.java | 77 - .../thriftserver/ThriftserverShimUtils.scala | 71 - .../GetCatalogsOperationMock.scala | 50 - .../v2.3/if/TCLIService.thrift | 1269 -- .../service/rpc/thrift/TArrayTypeEntry.java | 387 - .../service/rpc/thrift/TBinaryColumn.java | 548 - .../hive/service/rpc/thrift/TBoolColumn.java | 548 - .../hive/service/rpc/thrift/TBoolValue.java | 390 - .../hive/service/rpc/thrift/TByteColumn.java | 548 - .../hive/service/rpc/thrift/TByteValue.java | 390 - .../hive/service/rpc/thrift/TCLIService.java | 18138 ---------------- .../rpc/thrift/TCLIServiceConstants.java | 106 - .../rpc/thrift/TCancelDelegationTokenReq.java | 495 - .../thrift/TCancelDelegationTokenResp.java | 394 - .../rpc/thrift/TCancelOperationReq.java | 394 - .../rpc/thrift/TCancelOperationResp.java | 394 - .../rpc/thrift/TCloseOperationReq.java | 394 - .../rpc/thrift/TCloseOperationResp.java | 394 - .../service/rpc/thrift/TCloseSessionReq.java | 394 - .../service/rpc/thrift/TCloseSessionResp.java | 394 - .../hive/service/rpc/thrift/TColumn.java | 736 - .../hive/service/rpc/thrift/TColumnDesc.java | 704 - .../hive/service/rpc/thrift/TColumnValue.java | 675 - .../service/rpc/thrift/TDoubleColumn.java | 548 - .../hive/service/rpc/thrift/TDoubleValue.java | 390 - .../rpc/thrift/TExecuteStatementReq.java | 863 - .../rpc/thrift/TExecuteStatementResp.java | 509 - .../service/rpc/thrift/TFetchOrientation.java | 57 - .../service/rpc/thrift/TFetchResultsReq.java | 714 - .../service/rpc/thrift/TFetchResultsResp.java | 612 - .../service/rpc/thrift/TGetCatalogsReq.java | 394 - .../service/rpc/thrift/TGetCatalogsResp.java | 509 - .../service/rpc/thrift/TGetColumnsReq.java | 822 - .../service/rpc/thrift/TGetColumnsResp.java | 509 - .../rpc/thrift/TGetCrossReferenceReq.java | 1034 - .../rpc/thrift/TGetCrossReferenceResp.java | 509 - .../rpc/thrift/TGetDelegationTokenReq.java | 596 - .../rpc/thrift/TGetDelegationTokenResp.java | 504 - .../service/rpc/thrift/TGetFunctionsReq.java | 711 - .../service/rpc/thrift/TGetFunctionsResp.java | 509 - .../hive/service/rpc/thrift/TGetInfoReq.java | 507 - .../hive/service/rpc/thrift/TGetInfoResp.java | 497 - .../hive/service/rpc/thrift/TGetInfoType.java | 180 - .../service/rpc/thrift/TGetInfoValue.java | 597 - .../rpc/thrift/TGetOperationStatusReq.java | 501 - .../rpc/thrift/TGetOperationStatusResp.java | 1342 -- .../rpc/thrift/TGetPrimaryKeysReq.java | 716 - .../rpc/thrift/TGetPrimaryKeysResp.java | 509 - .../rpc/thrift/TGetResultSetMetadataReq.java | 394 - .../rpc/thrift/TGetResultSetMetadataResp.java | 509 - .../service/rpc/thrift/TGetSchemasReq.java | 610 - .../service/rpc/thrift/TGetSchemasResp.java | 509 - .../service/rpc/thrift/TGetTableTypesReq.java | 394 - .../rpc/thrift/TGetTableTypesResp.java | 509 - .../service/rpc/thrift/TGetTablesReq.java | 871 - .../service/rpc/thrift/TGetTablesResp.java | 509 - .../service/rpc/thrift/TGetTypeInfoReq.java | 394 - .../service/rpc/thrift/TGetTypeInfoResp.java | 509 - .../service/rpc/thrift/THandleIdentifier.java | 508 - .../hive/service/rpc/thrift/TI16Column.java | 548 - .../hive/service/rpc/thrift/TI16Value.java | 390 - .../hive/service/rpc/thrift/TI32Column.java | 548 - .../hive/service/rpc/thrift/TI32Value.java | 390 - .../hive/service/rpc/thrift/TI64Column.java | 548 - .../hive/service/rpc/thrift/TI64Value.java | 390 - .../rpc/thrift/TJobExecutionStatus.java | 48 - .../service/rpc/thrift/TMapTypeEntry.java | 482 - .../service/rpc/thrift/TOpenSessionReq.java | 778 - .../service/rpc/thrift/TOpenSessionResp.java | 783 - .../service/rpc/thrift/TOperationHandle.java | 709 - .../service/rpc/thrift/TOperationState.java | 66 - .../service/rpc/thrift/TOperationType.java | 66 - .../rpc/thrift/TPrimitiveTypeEntry.java | 516 - .../rpc/thrift/TProgressUpdateResp.java | 1033 - .../service/rpc/thrift/TProtocolVersion.java | 69 - .../rpc/thrift/TRenewDelegationTokenReq.java | 495 - .../rpc/thrift/TRenewDelegationTokenResp.java | 394 - .../apache/hive/service/rpc/thrift/TRow.java | 443 - .../hive/service/rpc/thrift/TRowSet.java | 920 - .../service/rpc/thrift/TSessionHandle.java | 394 - .../hive/service/rpc/thrift/TStatus.java | 875 - .../hive/service/rpc/thrift/TStatusCode.java | 54 - .../service/rpc/thrift/TStringColumn.java | 548 - .../hive/service/rpc/thrift/TStringValue.java | 393 - .../service/rpc/thrift/TStructTypeEntry.java | 452 - .../hive/service/rpc/thrift/TTableSchema.java | 443 - .../hive/service/rpc/thrift/TTypeDesc.java | 443 - .../hive/service/rpc/thrift/TTypeEntry.java | 614 - .../hive/service/rpc/thrift/TTypeId.java | 105 - .../rpc/thrift/TTypeQualifierValue.java | 365 - .../service/rpc/thrift/TTypeQualifiers.java | 454 - .../service/rpc/thrift/TUnionTypeEntry.java | 452 - .../rpc/thrift/TUserDefinedTypeEntry.java | 389 - .../thriftserver/ThriftserverShimUtils.scala | 74 - ...IntoHiveTableBenchmark-hive1.2-results.txt | 11 - .../execution/HiveCompatibilitySuite.scala | 5 +- sql/hive/pom.xml | 10 +- .../spark/sql/hive/HiveInspectors.scala | 7 +- .../spark/sql/hive/HiveSessionCatalog.scala | 103 +- .../sql/hive/HiveSessionStateBuilder.scala | 36 +- .../org/apache/spark/sql/hive/HiveShim.scala | 56 +- .../spark/sql/hive/HiveStrategies.scala | 33 +- .../org/apache/spark/sql/hive/HiveUtils.scala | 97 +- .../sql/hive/client/HiveClientImpl.scala | 47 +- .../spark/sql/hive/client/HiveShim.scala | 5 +- .../hive/client/IsolatedClientLoader.scala | 19 +- ...ala => HiveScriptTransformationExec.scala} | 247 +- .../hive/execution/InsertIntoHiveTable.scala | 4 +- .../execution/PruneHiveTablePartitions.scala | 11 +- .../org/apache/spark/sql/hive/hiveUDFs.scala | 23 +- .../spark/sql/hive/orc/OrcFileFormat.scala | 8 +- .../spark/sql/hive/orc/OrcFilters.scala | 262 - .../regression-test-SPARK-8489/test-2.13.jar | Bin 0 -> 19579 bytes .../InsertIntoHiveTableBenchmark.scala | 7 +- .../sql/hive/ClasspathDependenciesSuite.scala | 25 +- .../HiveExternalCatalogVersionsSuite.scala | 5 +- .../sql/hive/HiveMetastoreCatalogSuite.scala | 18 +- .../sql/hive/HiveParquetSourceSuite.scala | 16 +- .../sql/hive/HiveSchemaInferenceSuite.scala | 2 +- .../spark/sql/hive/HiveSharedStateSuite.scala | 46 +- .../apache/spark/sql/hive/HiveShimSuite.scala | 12 +- .../spark/sql/hive/HiveSparkSubmitSuite.scala | 2 +- .../apache/spark/sql/hive/InsertSuite.scala | 22 + .../spark/sql/hive/StatisticsSuite.scala | 45 +- .../execution/AggregationQuerySuite.scala | 12 +- .../hive/execution/HiveComparisonTest.scala | 2 +- .../sql/hive/execution/HiveDDLSuite.scala | 4 +- .../sql/hive/execution/HiveSQLViewSuite.scala | 22 +- .../HiveScriptTransformationSuite.scala | 480 +- .../execution/HiveSerDeReadWriteSuite.scala | 2 +- .../hive/execution/HiveTableScanSuite.scala | 84 + .../sql/hive/execution/HiveUDAFSuite.scala | 14 + .../sql/hive/execution/HiveUDFSuite.scala | 30 + .../PruneHiveTablePartitionsSuite.scala | 19 +- .../execution/PrunePartitionSuiteBase.scala | 59 +- .../sql/hive/execution/SQLQuerySuite.scala | 75 +- .../sql/hive/execution/WindowQuerySuite.scala | 9 +- .../sql/hive/orc/HiveOrcFilterSuite.scala | 484 - .../sql/hive/orc/HiveOrcQuerySuite.scala | 23 +- .../sql/hive/orc/HiveOrcSourceSuite.scala | 10 +- .../apache/spark/sql/hive/test/TestHive.scala | 15 +- ...saryBucketedScanWithHiveSupportSuite.scala | 31 + .../sql/sources/HadoopFsRelationTest.scala | 1 + .../apache/spark/streaming/DStreamGraph.scala | 10 +- .../rdd/WriteAheadLogBackedBlockRDD.scala | 2 +- .../receiver/ReceivedBlockHandler.scala | 2 +- .../scheduler/ExecutorAllocationManager.scala | 10 +- .../spark/streaming/util/HdfsUtils.scala | 8 +- .../ExecutorAllocationManagerSuite.scala | 54 +- .../spark/tools/GenerateMIMAIgnore.scala | 3 +- 2271 files changed, 185173 insertions(+), 154179 deletions(-) rename .github/workflows/{master.yml => build_and_test.yml} (50%) create mode 100644 .github/workflows/test_report.yml create mode 100644 .sbtopts mode change 100644 => 100755 bin/spark-class2.cmd create mode 100644 binder/apt.txt create mode 100644 binder/postBuild create mode 100644 common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ErrorHandler.java create mode 100644 common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergedBlockMeta.java create mode 100644 common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergedShuffleFileManager.java create mode 100644 common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/OneForOneBlockPusher.java create mode 100644 common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/FinalizeShuffleMerge.java create mode 100644 common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/MergeStatuses.java create mode 100644 common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/PushBlockStream.java create mode 100644 common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ErrorHandlerSuite.java create mode 100644 common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/OneForOneBlockPusherSuite.java delete mode 100644 core/src/main/resources/org/apache/spark/ui/static/jquery-3.4.1.min.js create mode 100644 core/src/main/resources/org/apache/spark/ui/static/jquery-3.5.1.min.js rename sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdate.scala => core/src/main/scala/org/apache/spark/security/SecurityConfigurationLock.scala (70%) create mode 100644 core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala create mode 100644 core/src/test/scala/org/apache/spark/TempLocalSparkContext.scala create mode 100644 core/src/test/scala/org/apache/spark/deploy/history/HistoryServerMemoryManagerSuite.scala create mode 100644 core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala create mode 100644 core/src/test/scala/org/apache/spark/shuffle/HostLocalShuffleReadingSuite.scala create mode 100644 core/src/test/scala/org/apache/spark/status/api/v1/ExecutorSummarySuite.scala delete mode 100644 dev/deps/spark-deps-hadoop-2.7-hive-1.2 delete mode 100644 docs/api.md delete mode 100644 docs/css/bootstrap-responsive.css delete mode 100644 docs/css/bootstrap-responsive.min.css delete mode 100644 docs/css/bootstrap.css create mode 100644 docs/css/bootstrap.min.css.map create mode 100644 docs/img/pycharm-with-pyspark1.png create mode 100644 docs/img/pycharm-with-pyspark2.png create mode 100644 docs/img/pycharm-with-pyspark3.png create mode 100644 docs/img/pyspark-components.png create mode 100644 docs/img/pyspark-components.pptx create mode 100644 docs/img/pyspark-remote-debug1.png create mode 100644 docs/img/pyspark-remote-debug2.png create mode 100644 docs/js/vendor/bootstrap.bundle.min.js create mode 100644 docs/js/vendor/bootstrap.bundle.min.js.map delete mode 100755 docs/js/vendor/bootstrap.js delete mode 100755 docs/js/vendor/bootstrap.min.js delete mode 100644 docs/js/vendor/jquery-3.4.1.min.js create mode 100644 docs/js/vendor/jquery-3.5.1.min.js create mode 100644 docs/sql-ref-syntax-qry-select-file.md create mode 100644 external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DB2IntegrationSuite.scala create mode 100644 external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala create mode 100644 external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala create mode 100644 external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresIntegrationSuite.scala create mode 100644 external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala create mode 100644 python/docs/source/_templates/autosummary/class.rst rename python/docs/source/_templates/{ => autosummary}/class_with_docs.rst (100%) create mode 100644 python/docs/source/development/contributing.rst create mode 100644 python/docs/source/development/debugging.rst create mode 100644 python/docs/source/development/setting_ide.rst create mode 100644 python/docs/source/development/testing.rst create mode 100644 python/docs/source/getting_started/install.rst create mode 100644 python/docs/source/getting_started/quickstart.ipynb create mode 100644 python/docs/source/migration_guide/pyspark_1.0_1.2_to_1.3.rst create mode 100644 python/docs/source/migration_guide/pyspark_1.4_to_1.5.rst create mode 100644 python/docs/source/migration_guide/pyspark_2.2_to_2.3.rst create mode 100644 python/docs/source/migration_guide/pyspark_2.3.0_to_2.3.1_above.rst create mode 100644 python/docs/source/migration_guide/pyspark_2.3_to_2.4.rst create mode 100644 python/docs/source/migration_guide/pyspark_2.4_to_3.0.rst create mode 100644 python/docs/source/user_guide/arrow_pandas.rst create mode 100644 python/docs/source/user_guide/python_packaging.rst create mode 100644 python/mypy.ini create mode 100644 python/pyspark/__init__.pyi create mode 100644 python/pyspark/_typing.pyi create mode 100644 python/pyspark/accumulators.pyi create mode 100644 python/pyspark/broadcast.pyi create mode 100644 python/pyspark/conf.pyi create mode 100644 python/pyspark/context.pyi create mode 100644 python/pyspark/files.pyi create mode 100644 python/pyspark/install.py create mode 100644 python/pyspark/ml/_typing.pyi create mode 100644 python/pyspark/ml/base.pyi create mode 100644 python/pyspark/ml/classification.pyi create mode 100644 python/pyspark/ml/clustering.pyi create mode 100644 python/pyspark/ml/common.pyi create mode 100644 python/pyspark/ml/evaluation.pyi create mode 100644 python/pyspark/ml/feature.pyi create mode 100644 python/pyspark/ml/fpm.pyi create mode 100644 python/pyspark/ml/functions.pyi create mode 100644 python/pyspark/ml/image.pyi create mode 100644 python/pyspark/ml/linalg/__init__.pyi create mode 100644 python/pyspark/ml/param/__init__.pyi create mode 100644 python/pyspark/ml/param/_shared_params_code_gen.pyi create mode 100644 python/pyspark/ml/param/shared.pyi create mode 100644 python/pyspark/ml/pipeline.pyi create mode 100644 python/pyspark/ml/recommendation.pyi create mode 100644 python/pyspark/ml/regression.pyi create mode 100644 python/pyspark/ml/stat.pyi create mode 100644 python/pyspark/ml/tree.pyi create mode 100644 python/pyspark/ml/tuning.pyi create mode 100644 python/pyspark/ml/util.pyi create mode 100644 python/pyspark/ml/wrapper.pyi create mode 100644 python/pyspark/mllib/_typing.pyi create mode 100644 python/pyspark/mllib/classification.pyi create mode 100644 python/pyspark/mllib/clustering.pyi create mode 100644 python/pyspark/mllib/common.pyi create mode 100644 python/pyspark/mllib/evaluation.pyi create mode 100644 python/pyspark/mllib/feature.pyi create mode 100644 python/pyspark/mllib/fpm.pyi create mode 100644 python/pyspark/mllib/linalg/__init__.pyi create mode 100644 python/pyspark/mllib/linalg/distributed.pyi create mode 100644 python/pyspark/mllib/random.pyi create mode 100644 python/pyspark/mllib/recommendation.pyi create mode 100644 python/pyspark/mllib/regression.pyi create mode 100644 python/pyspark/mllib/stat/KernelDensity.pyi create mode 100644 python/pyspark/mllib/stat/__init__.pyi create mode 100644 python/pyspark/mllib/stat/_statistics.pyi create mode 100644 python/pyspark/mllib/stat/distribution.pyi create mode 100644 python/pyspark/mllib/stat/test.pyi create mode 100644 python/pyspark/mllib/tree.pyi create mode 100644 python/pyspark/mllib/util.pyi create mode 100644 python/pyspark/profiler.pyi create mode 100644 python/pyspark/py.typed create mode 100644 python/pyspark/rdd.pyi create mode 100644 python/pyspark/resource/information.pyi create mode 100644 python/pyspark/resource/profile.pyi create mode 100644 python/pyspark/resource/requests.pyi create mode 100644 python/pyspark/resultiterable.pyi create mode 100644 python/pyspark/sql/__init__.pyi create mode 100644 python/pyspark/sql/_typing.pyi create mode 100644 python/pyspark/sql/avro/functions.pyi create mode 100644 python/pyspark/sql/catalog.pyi create mode 100644 python/pyspark/sql/column.pyi create mode 100644 python/pyspark/sql/conf.pyi create mode 100644 python/pyspark/sql/context.pyi create mode 100644 python/pyspark/sql/dataframe.pyi create mode 100644 python/pyspark/sql/functions.pyi create mode 100644 python/pyspark/sql/group.pyi create mode 100644 python/pyspark/sql/pandas/_typing/__init__.pyi create mode 100644 python/pyspark/sql/pandas/_typing/protocols/__init__.pyi create mode 100644 python/pyspark/sql/pandas/_typing/protocols/frame.pyi create mode 100644 python/pyspark/sql/pandas/_typing/protocols/series.pyi create mode 100644 python/pyspark/sql/pandas/conversion.pyi create mode 100644 python/pyspark/sql/pandas/functions.pyi create mode 100644 python/pyspark/sql/pandas/group_ops.pyi create mode 100644 python/pyspark/sql/pandas/map_ops.pyi create mode 100644 python/pyspark/sql/readwriter.pyi create mode 100644 python/pyspark/sql/session.pyi create mode 100644 python/pyspark/sql/streaming.pyi create mode 100644 python/pyspark/sql/types.pyi create mode 100644 python/pyspark/sql/udf.pyi create mode 100644 python/pyspark/sql/window.pyi create mode 100644 python/pyspark/statcounter.pyi create mode 100644 python/pyspark/status.pyi create mode 100644 python/pyspark/storagelevel.pyi create mode 100644 python/pyspark/streaming/context.pyi create mode 100644 python/pyspark/streaming/dstream.pyi create mode 100644 python/pyspark/streaming/kinesis.pyi create mode 100644 python/pyspark/streaming/listener.pyi create mode 100644 python/pyspark/taskcontext.pyi create mode 100644 python/pyspark/tests/test_install_spark.py create mode 100644 python/pyspark/version.pyi rename repl/src/main/{scala => scala-2.12}/org/apache/spark/repl/Main.scala (100%) rename repl/src/main/{scala => scala-2.12}/org/apache/spark/repl/SparkILoop.scala (100%) create mode 100644 repl/src/main/scala-2.13/org/apache/spark/repl/Main.scala create mode 100644 repl/src/main/scala-2.13/org/apache/spark/repl/SparkILoop.scala create mode 100644 repl/src/test/scala-2.12/org/apache/spark/repl/Repl2Suite.scala create mode 100644 repl/src/test/scala-2.12/org/apache/spark/repl/SingletonRepl2Suite.scala create mode 100644 repl/src/test/scala-2.13/org/apache/spark/repl/Repl2Suite.scala create mode 100644 repl/src/test/scala-2.13/org/apache/spark/repl/SingletonRepl2Suite.scala create mode 100644 resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesExecutorSpec.scala create mode 100644 resource-managers/kubernetes/integration-tests/tests/autoscale.py create mode 100644 resource-managers/kubernetes/integration-tests/tests/decommissioning_cleanup.py create mode 100644 sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagement.java create mode 100644 sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsPartitionManagement.java delete mode 100644 sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveNoopDropTable.scala create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InvalidUDFClassException.scala create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/AliasHelper.scala rename sql/catalyst/src/main/{scala-2.12 => scala}/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala (53%) create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeJsonExprs.scala create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparison.scala create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/UpdateFields.scala rename sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/{DescribeTableSchema.scala => DescribeCommandSchema.scala} (74%) create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/streaming/StreamingRelationV2.scala create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtil.scala create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdateAsAppend.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodegenSubexpressionEliminationSuite.scala delete mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombineWithFieldsSuite.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateAggregateFilterSuite.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/InferFiltersFromGenerateSuite.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeJsonExprsSuite.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeWithFieldsSuite.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerSuite.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/UnwrapCastInBinaryComparisonSuite.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/joinReorder/JoinReorderPlanTestBase.scala rename sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/{ => joinReorder}/JoinReorderSuite.scala (88%) rename sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/{ => joinReorder}/StarJoinCostBasedReorderSuite.scala (92%) rename sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/{ => joinReorder}/StarJoinReorderSuite.scala (91%) create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlanIntegritySuite.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtilSuite.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryAtomicPartitionTable.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryPartitionTable.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagementSuite.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsPartitionManagementSuite.scala create mode 100644 sql/core/benchmarks/InsertTableWithDynamicPartitionsBenchmark-jdk11-results.txt create mode 100644 sql/core/benchmarks/InsertTableWithDynamicPartitionsBenchmark-results.txt create mode 100644 sql/core/benchmarks/UpdateFieldsBenchmark-results.txt rename sql/core/{v2.3 => }/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java (100%) create mode 100644 sql/core/src/main/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider rename sql/{catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/WithFields.scala => core/src/main/scala/org/apache/spark/sql/execution/AlreadyOptimized.scala} (52%) create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantProjects.scala create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantSorts.scala create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AQEOptimizer.scala create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/EliminateJoinToEmptyRelation.scala create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/bucketing/DisableUnnecessaryBucketedScan.scala rename sql/core/{v2.3 => }/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala (83%) rename sql/core/{v2.3 => }/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala (100%) create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScan.scala create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCWriteBuilder.scala create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/jdbc/H2Dialect.scala create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcConnectionProvider.scala create mode 100644 sql/core/src/test/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider create mode 100644 sql/core/src/test/resources/test-data/percentile_approx-input.csv.bz2 rename sql/{hive => core}/src/test/resources/test_script.py (100%) create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q10/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q19/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q27/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q3/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q34/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q42/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q43/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q46/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q52/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q53/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q55/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q59/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q63/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q65/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q68/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q7/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q73/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q79/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q89/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/q98/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-modified/ss_max/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q79/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q8/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q84/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.sf100/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98.sf100/simplified.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt create mode 100644 sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/CTEHintSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/PlanStabilitySuite.scala rename sql/core/src/test/scala/org/apache/spark/sql/{TPCDSSchema.scala => TPCDSBase.scala} (82%) create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/TPCDSTableStats.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/UnwrapCastInComparisonEndToEndSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/UpdateFieldsBenchmark.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyOptimizedSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/ColumnarRulesSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantProjectsSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala rename sql/{hive/src/test/scala/org/apache/spark/sql/hive => core/src/test/scala/org/apache/spark/sql}/execution/TestUncaughtExceptionHandler.scala (96%) create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/InsertTableWithDynamicPartitionsBenchmark.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/BasicWriteJobStatsTrackerMetricSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/CommonFileDataSourceSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/DriverRegistrySuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/IntentionallyFaultyConnectionProvider.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/TestDriver.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/exchange/EnsureRequirementsSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamTableAPISuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala delete mode 100644 sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java delete mode 100644 sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala delete mode 100644 sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala delete mode 100644 sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala delete mode 100644 sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala delete mode 100644 sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/AbstractService.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/CompositeService.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/CookieSigner.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/ServiceOperations.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/ServiceUtils.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/auth/HiveAuthFactory.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/auth/HttpAuthUtils.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/CLIService.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/ColumnDescriptor.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/ColumnValue.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/FetchOrientation.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/GetInfoType.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/GetInfoValue.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/Handle.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/HandleIdentifier.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/HiveSQLException.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/ICLIService.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/OperationHandle.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/OperationState.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/OperationType.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/RowBasedSet.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/RowSet.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/RowSetFactory.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/SessionHandle.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/TableSchema.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/TypeDescriptor.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/TypeQualifiers.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/GetCrossReferenceOperation.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/GetFunctionsOperation.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/GetPrimaryKeysOperation.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/GetTablesOperation.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/MetadataOperation.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/Operation.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java (99%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java (98%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/operation/TableTypeMapping.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/session/HiveSession.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/session/HiveSessionBase.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/session/HiveSessionHookContext.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/session/HiveSessionHookContextImpl.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/session/SessionManager.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/server/HiveServer2.java (100%) rename sql/hive-thriftserver/{v2.3 => }/src/main/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java (100%) rename sql/hive-thriftserver/{v2.3/src/test/scala/ org => src/test/scala/org}/apache/spark/sql/hive/thriftserver/GetCatalogsOperationMock.scala (100%) delete mode 100644 sql/hive-thriftserver/v1.2/if/TCLIService.thrift delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TArrayTypeEntry.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TBinaryColumn.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TBoolColumn.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TBoolValue.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TByteColumn.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TByteValue.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCLIService.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCLIServiceConstants.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelDelegationTokenReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelDelegationTokenResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelOperationReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCancelOperationResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseOperationReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseOperationResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseSessionReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TCloseSessionResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TColumn.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TColumnDesc.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TColumnValue.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TDoubleColumn.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TDoubleValue.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TExecuteStatementReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TExecuteStatementResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TFetchOrientation.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TFetchResultsReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TFetchResultsResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetCatalogsReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetCatalogsResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetColumnsReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetColumnsResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetDelegationTokenReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetDelegationTokenResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetFunctionsReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetFunctionsResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoType.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetInfoValue.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetOperationStatusReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetOperationStatusResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetResultSetMetadataReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetResultSetMetadataResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetSchemasReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetSchemasResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTableTypesReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTableTypesResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTablesReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTablesResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTypeInfoReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TGetTypeInfoResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/THandleIdentifier.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI16Column.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI16Value.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI32Column.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI32Value.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI64Column.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TI64Value.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TMapTypeEntry.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOpenSessionReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOpenSessionResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOperationHandle.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOperationState.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TOperationType.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TPrimitiveTypeEntry.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TProtocolVersion.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRenewDelegationTokenReq.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRenewDelegationTokenResp.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRow.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TRowSet.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TSessionHandle.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStatus.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStatusCode.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStringColumn.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStringValue.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TStructTypeEntry.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTableSchema.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeDesc.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeEntry.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeId.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeQualifierValue.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TTypeQualifiers.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TUnionTypeEntry.java delete mode 100644 sql/hive-thriftserver/v1.2/src/gen/java/org/apache/hive/service/cli/thrift/TUserDefinedTypeEntry.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/AbstractService.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/CompositeService.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/CookieSigner.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/ServiceOperations.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/ServiceUtils.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/HiveAuthFactory.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/HttpAuthUtils.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/CLIService.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/Column.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnBasedSet.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnDescriptor.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/EmbeddedCLIServiceClient.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/FetchOrientation.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/GetInfoType.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/GetInfoValue.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/Handle.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/HandleIdentifier.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/HiveSQLException.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ICLIService.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/OperationHandle.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/OperationState.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/OperationType.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/PatternOrIdentifier.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/RowBasedSet.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/RowSet.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/RowSetFactory.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/SessionHandle.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/TableSchema.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/Type.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/TypeDescriptor.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/TypeQualifiers.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/ClassicTableTypeMapping.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/ExecuteStatementOperation.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetFunctionsOperation.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetTablesOperation.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/HiveTableTypeMapping.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/MetadataOperation.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/Operation.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/OperationManager.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/SQLOperation.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/TableTypeMapping.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSession.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionBase.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/SessionManager.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/server/HiveServer2.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java delete mode 100644 sql/hive-thriftserver/v1.2/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala delete mode 100644 sql/hive-thriftserver/v1.2/src/test/scala/ org/apache/spark/sql/hive/thriftserver/GetCatalogsOperationMock.scala delete mode 100644 sql/hive-thriftserver/v2.3/if/TCLIService.thrift delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TArrayTypeEntry.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TBinaryColumn.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TBoolColumn.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TBoolValue.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TByteColumn.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TByteValue.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCLIService.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCLIServiceConstants.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelDelegationTokenReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelDelegationTokenResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelOperationReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCancelOperationResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseOperationReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseOperationResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseSessionReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TCloseSessionResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TColumn.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TColumnDesc.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TColumnValue.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TDoubleColumn.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TDoubleValue.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TExecuteStatementReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TExecuteStatementResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TFetchOrientation.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TFetchResultsReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TFetchResultsResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCatalogsReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCatalogsResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetColumnsReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetColumnsResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCrossReferenceReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetCrossReferenceResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetDelegationTokenReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetDelegationTokenResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetFunctionsReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetFunctionsResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoType.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetInfoValue.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetOperationStatusReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetOperationStatusResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetPrimaryKeysReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetPrimaryKeysResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetResultSetMetadataReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetResultSetMetadataResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetSchemasReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetSchemasResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTableTypesReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTableTypesResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTablesReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTablesResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTypeInfoReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TGetTypeInfoResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/THandleIdentifier.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI16Column.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI16Value.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI32Column.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI32Value.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI64Column.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TI64Value.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TJobExecutionStatus.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TMapTypeEntry.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOpenSessionReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOpenSessionResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOperationHandle.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOperationState.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TOperationType.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TPrimitiveTypeEntry.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TProgressUpdateResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TProtocolVersion.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRenewDelegationTokenReq.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRenewDelegationTokenResp.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRow.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TRowSet.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TSessionHandle.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStatus.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStatusCode.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStringColumn.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStringValue.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TStructTypeEntry.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTableSchema.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeDesc.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeEntry.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeId.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeQualifierValue.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TTypeQualifiers.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TUnionTypeEntry.java delete mode 100644 sql/hive-thriftserver/v2.3/src/gen/java/org/apache/hive/service/rpc/thrift/TUserDefinedTypeEntry.java delete mode 100644 sql/hive-thriftserver/v2.3/src/main/scala/org/apache/spark/sql/hive/thriftserver/ThriftserverShimUtils.scala delete mode 100644 sql/hive/benchmarks/InsertIntoHiveTableBenchmark-hive1.2-results.txt rename sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/{ScriptTransformationExec.scala => HiveScriptTransformationExec.scala} (58%) delete mode 100644 sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala create mode 100644 sql/hive/src/test/resources/regression-test-SPARK-8489/test-2.13.jar delete mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcFilterSuite.scala create mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/sources/DisableUnnecessaryBucketedScanWithHiveSupportSuite.scala diff --git a/.github/workflows/master.yml b/.github/workflows/build_and_test.yml similarity index 50% rename from .github/workflows/master.yml rename to .github/workflows/build_and_test.yml index 009ebe90ddf51..55c578e15724a 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/build_and_test.yml @@ -1,4 +1,4 @@ -name: master +name: Build and test on: push: @@ -7,13 +7,18 @@ on: pull_request: branches: - master + workflow_dispatch: + inputs: + target: + description: 'Target branch to run' + required: true jobs: - # TODO(SPARK-32248): Recover JDK 11 builds # Build: build Spark and run the tests for specified modules. build: name: "Build modules: ${{ matrix.modules }} ${{ matrix.comment }} (JDK ${{ matrix.java }}, ${{ matrix.hadoop }}, ${{ matrix.hive }})" - runs-on: ubuntu-latest + # Ubuntu 20.04 is the latest LTS. The next LTS is 22.04. + runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: @@ -27,22 +32,16 @@ jobs: # Kinesis tests depends on external Amazon kinesis service. # Note that the modules below are from sparktestsupport/modules.py. modules: - - |- + - >- core, unsafe, kvstore, avro, network-common, network-shuffle, repl, launcher, examples, sketch, graphx - - |- + - >- catalyst, hive-thriftserver - - |- + - >- streaming, sql-kafka-0-10, streaming-kafka-0-10, mllib-local, mllib, yarn, mesos, kubernetes, hadoop-cloud, spark-ganglia-lgpl - - |- - pyspark-sql, pyspark-mllib, pyspark-resource - - |- - pyspark-core, pyspark-streaming, pyspark-ml - - |- - sparkr # Here, we split Hive and SQL tests into some of slow ones and the rest of them. included-tags: [""] excluded-tags: [""] @@ -83,18 +82,26 @@ jobs: # GitHub Actions' default miniconda to use in pip packaging test. CONDA_PREFIX: /usr/share/miniconda GITHUB_PREV_SHA: ${{ github.event.before }} + GITHUB_INPUT_BRANCH: ${{ github.event.inputs.target }} steps: - name: Checkout Spark repository uses: actions/checkout@v2 # In order to fetch changed files with: fetch-depth: 0 + - name: Merge dispatched input branch + if: ${{ github.event.inputs.target != '' }} + run: git merge --progress --ff-only origin/${{ github.event.inputs.target }} # Cache local repositories. Note that GitHub Actions cache has a 2G limit. - name: Cache Scala, SBT, Maven and Zinc - uses: actions/cache@v1 + uses: actions/cache@v2 with: - path: build - key: build-${{ hashFiles('**/pom.xml') }} + path: | + build/apache-maven-* + build/zinc-* + build/scala-* + build/*.jar + key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} restore-keys: | build- - name: Cache Maven local repository @@ -108,80 +115,200 @@ jobs: uses: actions/cache@v2 with: path: ~/.ivy2/cache - key: ${{ matrix.java }}-${{ matrix.hadoop }}-ivy-${{ hashFiles('**/pom.xml') }}-${{ hashFiles('**/plugins.sbt') }} + key: ${{ matrix.java }}-${{ matrix.hadoop }}-ivy-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | ${{ matrix.java }}-${{ matrix.hadoop }}-ivy- - name: Install JDK ${{ matrix.java }} uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - # PySpark - - name: Install PyPy3 - # Note that order of Python installations here matters because default python3 is - # overridden by pypy3. - uses: actions/setup-python@v2 - if: contains(matrix.modules, 'pyspark') - with: - python-version: pypy3 - architecture: x64 - - name: Install Python 3.6 - uses: actions/setup-python@v2 - if: contains(matrix.modules, 'pyspark') - with: - python-version: 3.6 - architecture: x64 - name: Install Python 3.8 uses: actions/setup-python@v2 # We should install one Python that is higher then 3+ for SQL and Yarn because: # - SQL component also has Python related tests, for example, IntegratedUDFTestUtils. # - Yarn has a Python specific test too, for example, YarnClusterSuite. - if: contains(matrix.modules, 'yarn') || contains(matrix.modules, 'pyspark') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-')) + if: contains(matrix.modules, 'yarn') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-')) with: python-version: 3.8 architecture: x64 - - name: Install Python packages (Python 3.6 and PyPy3) - if: contains(matrix.modules, 'pyspark') - # PyArrow is not supported in PyPy yet, see ARROW-2651. - # TODO(SPARK-32247): scipy installation with PyPy fails for an unknown reason. - run: | - python3.6 -m pip install numpy pyarrow pandas scipy - python3.6 -m pip list - pypy3 -m pip install numpy pandas - pypy3 -m pip list - name: Install Python packages (Python 3.8) - if: contains(matrix.modules, 'pyspark') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-')) + if: (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-')) run: | - python3.8 -m pip install numpy pyarrow pandas scipy + python3.8 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner python3.8 -m pip list - # SparkR - - name: Install R 4.0 - if: contains(matrix.modules, 'sparkr') - run: | - sudo sh -c "echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/' >> /etc/apt/sources.list" - curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xE298A3A825C0D65DFD57CBB651716619E084DAB9" | sudo apt-key add - sudo apt-get update - sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev - - name: Install R packages - if: contains(matrix.modules, 'sparkr') - run: | - # qpdf is required to reduce the size of PDFs to make CRAN check pass. See SPARK-32497. - sudo apt-get install -y libcurl4-openssl-dev qpdf - sudo Rscript -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'devtools', 'e1071', 'survival', 'arrow', 'roxygen2'), repos='https://cloud.r-project.org/')" - # Show installed packages in R. - sudo Rscript -e 'pkg_list <- as.data.frame(installed.packages()[, c(1,3:4)]); pkg_list[is.na(pkg_list$Priority), 1:2, drop = FALSE]' # Run the tests. - - name: "Run tests: ${{ matrix.modules }}" + - name: Run tests run: | # Hive tests become flaky when running in parallel as it's too intensive. if [[ "$MODULES_TO_TEST" == "hive" ]]; then export SERIAL_SBT_TESTS=1; fi mkdir -p ~/.m2 ./dev/run-tests --parallelism 2 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS" rm -rf ~/.m2/repository/org/apache/spark + - name: Upload test results to report + if: always() + uses: actions/upload-artifact@v2 + with: + name: test-results-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }} + path: "**/target/test-reports/*.xml" + - name: Upload unit tests log files + if: failure() + uses: actions/upload-artifact@v2 + with: + name: unit-tests-log-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }} + path: "**/target/unit-tests.log" + + pyspark: + name: "Build modules: ${{ matrix.modules }}" + runs-on: ubuntu-20.04 + container: + image: dongjoon/apache-spark-github-action-image:20201015 + strategy: + fail-fast: false + matrix: + modules: + - >- + pyspark-sql, pyspark-mllib, pyspark-resource + - >- + pyspark-core, pyspark-streaming, pyspark-ml + env: + MODULES_TO_TEST: ${{ matrix.modules }} + HADOOP_PROFILE: hadoop3.2 + HIVE_PROFILE: hive2.3 + # GitHub Actions' default miniconda to use in pip packaging test. + CONDA_PREFIX: /usr/share/miniconda + GITHUB_PREV_SHA: ${{ github.event.before }} + GITHUB_INPUT_BRANCH: ${{ github.event.inputs.target }} + steps: + - name: Checkout Spark repository + uses: actions/checkout@v2 + # In order to fetch changed files + with: + fetch-depth: 0 + - name: Merge dispatched input branch + if: ${{ github.event.inputs.target != '' }} + run: git merge --progress --ff-only origin/${{ github.event.inputs.target }} + # Cache local repositories. Note that GitHub Actions cache has a 2G limit. + - name: Cache Scala, SBT, Maven and Zinc + uses: actions/cache@v2 + with: + path: | + build/apache-maven-* + build/zinc-* + build/scala-* + build/*.jar + key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} + restore-keys: | + build- + - name: Cache Maven local repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: pyspark-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + pyspark-maven- + - name: Cache Ivy local repository + uses: actions/cache@v2 + with: + path: ~/.ivy2/cache + key: pyspark-ivy-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + restore-keys: | + pyspark-ivy- + - name: Install Python 3.6 + uses: actions/setup-python@v2 + with: + python-version: 3.6 + architecture: x64 + # This step takes much less time (~30s) than other Python versions so it is not included + # in the Docker image being used. There is also a technical issue to install Python 3.6 on + # Ubuntu 20.04. See also SPARK-33162. + - name: Install Python packages (Python 3.6) + run: | + python3.6 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner + python3.6 -m pip list + # Run the tests. + - name: Run tests + run: | + mkdir -p ~/.m2 + ./dev/run-tests --parallelism 2 --modules "$MODULES_TO_TEST" + rm -rf ~/.m2/repository/org/apache/spark + - name: Upload test results to report + if: always() + uses: actions/upload-artifact@v2 + with: + name: test-results-${{ matrix.modules }}--1.8-hadoop3.2-hive2.3 + path: "**/target/test-reports/*.xml" + - name: Upload unit tests log files + if: failure() + uses: actions/upload-artifact@v2 + with: + name: unit-tests-log-${{ matrix.modules }}--1.8-hadoop3.2-hive2.3 + path: "**/target/unit-tests.log" + + sparkr: + name: Build modules - sparkr + runs-on: ubuntu-20.04 + container: + image: dongjoon/apache-spark-github-action-image:20201025 + env: + HADOOP_PROFILE: hadoop3.2 + HIVE_PROFILE: hive2.3 + GITHUB_PREV_SHA: ${{ github.event.before }} + GITHUB_INPUT_BRANCH: ${{ github.event.inputs.target }} + steps: + - name: Checkout Spark repository + uses: actions/checkout@v2 + # In order to fetch changed files + with: + fetch-depth: 0 + - name: Merge dispatched input branch + if: ${{ github.event.inputs.target != '' }} + run: git merge --progress --ff-only origin/${{ github.event.inputs.target }} + # Cache local repositories. Note that GitHub Actions cache has a 2G limit. + - name: Cache Scala, SBT, Maven and Zinc + uses: actions/cache@v2 + with: + path: | + build/apache-maven-* + build/zinc-* + build/scala-* + build/*.jar + key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} + restore-keys: | + build- + - name: Cache Maven local repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: sparkr-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + sparkr-maven- + - name: Cache Ivy local repository + uses: actions/cache@v2 + with: + path: ~/.ivy2/cache + key: sparkr-ivy-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + restore-keys: | + sparkr-ivy- + - name: Run tests + run: | + mkdir -p ~/.m2 + # The followings are also used by `r-lib/actions/setup-r` to avoid + # R issues at docker environment + export TZ=UTC + export _R_CHECK_SYSTEM_CLOCK_=FALSE + ./dev/run-tests --parallelism 2 --modules sparkr + rm -rf ~/.m2/repository/org/apache/spark + - name: Upload test results to report + if: always() + uses: actions/upload-artifact@v2 + with: + name: test-results-sparkr--1.8-hadoop3.2-hive2.3 + path: "**/target/test-reports/*.xml" # Static analysis, and documentation build lint: name: Linters, licenses, dependencies and documentation generation - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout Spark repository uses: actions/checkout@v2 @@ -205,13 +332,11 @@ jobs: run: | # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes. # See also https://github.com/sphinx-doc/sphinx/issues/7551. - pip3 install flake8 'sphinx<3.1.0' numpy pydata_sphinx_theme + pip3 install flake8 'sphinx<3.1.0' numpy pydata_sphinx_theme ipython nbsphinx mypy numpydoc - name: Install R 4.0 - run: | - sudo sh -c "echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/' >> /etc/apt/sources.list" - curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xE298A3A825C0D65DFD57CBB651716619E084DAB9" | sudo apt-key add - sudo apt-get update - sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev + uses: r-lib/actions/setup-r@v1 + with: + r-version: 4.0 - name: Install R linter dependencies and SparkR run: | sudo apt-get install -y libcurl4-openssl-dev @@ -224,10 +349,11 @@ jobs: ruby-version: 2.7 - name: Install dependencies for documentation generation run: | + # pandoc is required to generate PySpark APIs as well in nbsphinx. sudo apt-get install -y libcurl4-openssl-dev pandoc # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes. # See also https://github.com/sphinx-doc/sphinx/issues/7551. - pip install 'sphinx<3.1.0' mkdocs numpy pydata_sphinx_theme + pip install 'sphinx<3.1.0' mkdocs numpy pydata_sphinx_theme ipython nbsphinx numpydoc gem install jekyll jekyll-redirect-from rouge sudo Rscript -e "install.packages(c('devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2'), repos='https://cloud.r-project.org/')" - name: Scala linter @@ -249,7 +375,7 @@ jobs: java11: name: Java 11 build - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout Spark repository uses: actions/checkout@v2 @@ -271,3 +397,25 @@ jobs: mkdir -p ~/.m2 ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=11 install rm -rf ~/.m2/repository/org/apache/spark + + scala-213: + name: Scala 2.13 build + runs-on: ubuntu-20.04 + steps: + - name: Checkout Spark repository + uses: actions/checkout@v2 + - name: Cache Ivy local repository + uses: actions/cache@v2 + with: + path: ~/.ivy2/cache + key: scala-213-ivy-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + restore-keys: | + scala-213-ivy- + - name: Install Java 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build with SBT + run: | + ./dev/change-scala-version.sh 2.13 + ./build/sbt -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Djava.version=11 -Pscala-2.13 compile test:compile diff --git a/.github/workflows/test_report.yml b/.github/workflows/test_report.yml new file mode 100644 index 0000000000000..93cdb86687261 --- /dev/null +++ b/.github/workflows/test_report.yml @@ -0,0 +1,24 @@ +name: Report test results +on: + workflow_run: + workflows: ["Build and test"] + types: + - completed + +jobs: + test_report: + runs-on: ubuntu-latest + steps: + - name: Download test results to report + uses: dawidd6/action-download-artifact@v2 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + workflow: ${{ github.event.workflow_run.workflow_id }} + commit: ${{ github.event.workflow_run.head_commit.id }} + - name: Publish test report + uses: scacap/action-surefire-report@v1 + with: + check_name: Report test results + github_token: ${{ secrets.GITHUB_TOKEN }} + report_paths: "**/target/test-reports/*.xml" + commit: ${{ github.event.workflow_run.head_commit.id }} diff --git a/.gitignore b/.gitignore index 0d8addeb10e21..9c145fba1bee9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ *.swp *~ .DS_Store +.bsp/ .cache .classpath .ensime @@ -68,6 +69,7 @@ python/docs/source/reference/api/ python/test_coverage/coverage_data python/test_coverage/htmlcov python/pyspark/python +.mypy_cache/ reports/ scalastyle-on-compile.generated.xml scalastyle-output.xml @@ -80,6 +82,7 @@ target/ unit-tests.log work/ docs/.jekyll-metadata +docs/.jekyll-cache # For Hive TempStatsStore/ diff --git a/.sbtopts b/.sbtopts new file mode 100644 index 0000000000000..9afbdca6db1c7 --- /dev/null +++ b/.sbtopts @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +-J-Xmx4G +-J-Xss4m diff --git a/R/pkg/NAMESPACE b/R/pkg/NAMESPACE index 53a0b7856567e..404a6968ea429 100644 --- a/R/pkg/NAMESPACE +++ b/R/pkg/NAMESPACE @@ -228,8 +228,11 @@ exportMethods("%<=>%", "arrays_zip", "arrays_zip_with", "asc", + "asc_nulls_first", + "asc_nulls_last", "ascii", "asin", + "assert_true", "atan", "atan2", "avg", @@ -272,6 +275,9 @@ exportMethods("%<=>%", "degrees", "dense_rank", "desc", + "desc_nulls_first", + "desc_nulls_last", + "dropFields", "element_at", "encode", "endsWith", @@ -348,6 +354,7 @@ exportMethods("%<=>%", "negate", "next_day", "not", + "nth_value", "ntile", "otherwise", "over", @@ -359,6 +366,7 @@ exportMethods("%<=>%", "posexplode_outer", "quarter", "radians", + "raise_error", "rand", "randn", "rank", @@ -405,6 +413,7 @@ exportMethods("%<=>%", "sumDistinct", "tan", "tanh", + "timestamp_seconds", "toDegrees", "toRadians", "to_csv", @@ -425,9 +434,11 @@ exportMethods("%<=>%", "variance", "var_pop", "var_samp", + "vector_to_array", "weekofyear", "when", "window", + "withField", "xxhash64", "year") diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R index 4d38f979c4103..2ce53782d9af0 100644 --- a/R/pkg/R/DataFrame.R +++ b/R/pkg/R/DataFrame.R @@ -1234,13 +1234,9 @@ setMethod("collect", output <- tryCatch({ doServerAuth(conn, authSecret) arrowTable <- arrow::read_ipc_stream(readRaw(conn)) - # Arrow drops `as_tibble` since 0.14.0, see ARROW-5190. - if (exists("as_tibble", envir = asNamespace("arrow"))) { - as.data.frame(arrow::as_tibble(arrowTable), stringsAsFactors = stringsAsFactors) - } else { - as.data.frame(arrowTable, stringsAsFactors = stringsAsFactors) - } - }, finally = { + as.data.frame(arrowTable, stringsAsFactors = stringsAsFactors) + }, + finally = { close(conn) }) return(output) @@ -2867,11 +2863,18 @@ setMethod("unionAll", #' \code{UNION ALL} and \code{UNION DISTINCT} in SQL as column positions are not taken #' into account. Input SparkDataFrames can have different data types in the schema. #' +#' When the parameter allowMissingColumns is `TRUE`, the set of column names +#' in x and y can differ; missing columns will be filled as null. +#' Further, the missing columns of x will be added at the end +#' in the schema of the union result. +#' #' Note: This does not remove duplicate rows across the two SparkDataFrames. #' This function resolves columns by name (not by position). #' #' @param x A SparkDataFrame #' @param y A SparkDataFrame +#' @param allowMissingColumns logical +#' @param ... further arguments to be passed to or from other methods. #' @return A SparkDataFrame containing the result of the union. #' @family SparkDataFrame functions #' @rdname unionByName @@ -2884,12 +2887,15 @@ setMethod("unionAll", #' df1 <- select(createDataFrame(mtcars), "carb", "am", "gear") #' df2 <- select(createDataFrame(mtcars), "am", "gear", "carb") #' head(unionByName(df1, df2)) +#' +#' df3 <- select(createDataFrame(mtcars), "carb") +#' head(unionByName(df1, df3, allowMissingColumns = TRUE)) #' } #' @note unionByName since 2.3.0 setMethod("unionByName", signature(x = "SparkDataFrame", y = "SparkDataFrame"), - function(x, y) { - unioned <- callJMethod(x@sdf, "unionByName", y@sdf) + function(x, y, allowMissingColumns=FALSE) { + unioned <- callJMethod(x@sdf, "unionByName", y@sdf, allowMissingColumns) dataFrame(unioned) }) diff --git a/R/pkg/R/column.R b/R/pkg/R/column.R index 7926a9a2467ee..835178990b485 100644 --- a/R/pkg/R/column.R +++ b/R/pkg/R/column.R @@ -67,7 +67,11 @@ operators <- list( # we can not override `&&` and `||`, so use `&` and `|` instead "&" = "and", "|" = "or", "^" = "pow" ) -column_functions1 <- c("asc", "desc", "isNaN", "isNull", "isNotNull") +column_functions1 <- c( + "asc", "asc_nulls_first", "asc_nulls_last", + "desc", "desc_nulls_first", "desc_nulls_last", + "isNaN", "isNull", "isNotNull" +) column_functions2 <- c("like", "rlike", "getField", "getItem", "contains") createOperator <- function(op) { @@ -356,3 +360,103 @@ setMethod("%<=>%", #' } #' @note ! since 2.3.0 setMethod("!", signature(x = "Column"), function(x) not(x)) + +#' withField +#' +#' Adds/replaces field in a struct \code{Column} by name. +#' +#' @param x a Column +#' @param fieldName a character +#' @param col a Column expression +#' +#' @rdname withField +#' @aliases withField withField,Column-method +#' @examples +#' \dontrun{ +#' df <- withColumn( +#' createDataFrame(iris), +#' "sepal", +#' struct(column("Sepal_Width"), column("Sepal_Length")) +#' ) +#' +#' head(select( +#' df, +#' withField(df$sepal, "product", df$Sepal_Length * df$Sepal_Width) +#' )) +#' } +#' @note withField since 3.1.0 +setMethod("withField", + signature(x = "Column", fieldName = "character", col = "Column"), + function(x, fieldName, col) { + jc <- callJMethod(x@jc, "withField", fieldName, col@jc) + column(jc) + }) + +#' dropFields +#' +#' Drops fields in a struct \code{Column} by name. +#' +#' @param x a Column +#' @param ... names of the fields to be dropped. +#' +#' @rdname dropFields +#' @aliases dropFields dropFields,Column-method +#' @examples +#' \dontrun{ +#' df <- select( +#' createDataFrame(iris), +#' alias( +#' struct( +#' column("Sepal_Width"), column("Sepal_Length"), +#' alias( +#' struct( +#' column("Petal_Width"), column("Petal_Length"), +#' alias( +#' column("Petal_Width") * column("Petal_Length"), +#' "Petal_Product" +#' ) +#' ), +#' "Petal" +#' ) +#' ), +#' "dimensions" +#' ) +#' ) +#' head(withColumn(df, "dimensions", dropFields(df$dimensions, "Petal"))) +#' +#' head( +#' withColumn( +#' df, "dimensions", +#' dropFields(df$dimensions, "Sepal_Width", "Sepal_Length") +#' ) +#' ) +#' +#' # This method supports dropping multiple nested fields directly e.g. +#' head( +#' withColumn( +#' df, "dimensions", +#' dropFields(df$dimensions, "Petal.Petal_Width", "Petal.Petal_Length") +#' ) +#' ) +#' +#' # However, if you are going to add/replace multiple nested fields, +#' # it is preffered to extract out the nested struct before +#' # adding/replacing multiple fields e.g. +#' head( +#' withColumn( +#' df, "dimensions", +#' withField( +#' column("dimensions"), +#' "Petal", +#' dropFields(column("dimensions.Petal"), "Petal_Width", "Petal_Length") +#' ) +#' ) +#' ) +#' } +#' @note dropFields since 3.1.0 +setMethod("dropFields", + signature(x = "Column"), + function(x, ...) { + jc <- callJMethod(x@jc, "dropFields", list(...)) + column(jc) + }) diff --git a/R/pkg/R/deserialize.R b/R/pkg/R/deserialize.R index 3e7c456bd548d..5d22340fb62a0 100644 --- a/R/pkg/R/deserialize.R +++ b/R/pkg/R/deserialize.R @@ -233,24 +233,13 @@ readMultipleObjectsWithKeys <- function(inputCon) { readDeserializeInArrow <- function(inputCon) { if (requireNamespace("arrow", quietly = TRUE)) { - # Arrow drops `as_tibble` since 0.14.0, see ARROW-5190. - useAsTibble <- exists("as_tibble", envir = asNamespace("arrow")) - - # Currently, there looks no way to read batch by batch by socket connection in R side, # See ARROW-4512. Therefore, it reads the whole Arrow streaming-formatted binary at once # for now. dataLen <- readInt(inputCon) arrowData <- readBin(inputCon, raw(), as.integer(dataLen), endian = "big") batches <- arrow::RecordBatchStreamReader$create(arrowData)$batches() - - if (useAsTibble) { - as_tibble <- get("as_tibble", envir = asNamespace("arrow")) - # Read all groupped batches. Tibble -> data.frame is cheap. - lapply(batches, function(batch) as.data.frame(as_tibble(batch))) - } else { - lapply(batches, function(batch) as.data.frame(batch)) - } + lapply(batches, function(batch) as.data.frame(batch)) } else { stop("'arrow' package should be installed.") } diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R index da9ef1d6674bd..bcd798a8c31e2 100644 --- a/R/pkg/R/functions.R +++ b/R/pkg/R/functions.R @@ -338,12 +338,29 @@ NULL #' tmp <- mutate(df, dist = over(cume_dist(), ws), dense_rank = over(dense_rank(), ws), #' lag = over(lag(df$mpg), ws), lead = over(lead(df$mpg, 1), ws), #' percent_rank = over(percent_rank(), ws), -#' rank = over(rank(), ws), row_number = over(row_number(), ws)) +#' rank = over(rank(), ws), row_number = over(row_number(), ws), +#' nth_value = over(nth_value(df$mpg, 3), ws)) #' # Get ntile group id (1-4) for hp #' tmp <- mutate(tmp, ntile = over(ntile(4), ws)) #' head(tmp)} NULL +#' ML functions for Column operations +#' +#' ML functions defined for \code{Column}. +#' +#' @param x Column to compute on. +#' @param ... additional argument(s). +#' @name column_ml_functions +#' @rdname column_ml_functions +#' @family ml functions +#' @examples +#' \dontrun{ +#' df <- read.df("data/mllib/sample_libsvm_data.txt", source = "libsvm") +#' head(select(df, vector_to_array(df$features))) +#' } +NULL + #' @details #' \code{lit}: A new Column is created to represent the literal value. #' If the parameter is a Column, it is returned unchanged. @@ -809,6 +826,57 @@ setMethod("xxhash64", column(jc) }) +#' @details +#' \code{assert_true}: Returns null if the input column is true; throws an exception +#' with the provided error message otherwise. +#' +#' @param errMsg (optional) The error message to be thrown. +#' +#' @rdname column_misc_functions +#' @aliases assert_true assert_true,Column-method +#' @examples +#' \dontrun{ +#' tmp <- mutate(df, v1 = assert_true(df$vs < 2), +#' v2 = assert_true(df$vs < 2, "custom error message"), +#' v3 = assert_true(df$vs < 2, df$vs)) +#' head(tmp)} +#' @note assert_true since 3.1.0 +setMethod("assert_true", + signature(x = "Column"), + function(x, errMsg = NULL) { + jc <- if (is.null(errMsg)) { + callJStatic("org.apache.spark.sql.functions", "assert_true", x@jc) + } else { + if (is.character(errMsg)) { + stopifnot(length(errMsg) == 1) + errMsg <- lit(errMsg) + } + callJStatic("org.apache.spark.sql.functions", "assert_true", x@jc, errMsg@jc) + } + column(jc) + }) + +#' @details +#' \code{raise_error}: Throws an exception with the provided error message. +#' +#' @rdname column_misc_functions +#' @aliases raise_error raise_error,characterOrColumn-method +#' @examples +#' \dontrun{ +#' tmp <- mutate(df, v1 = raise_error("error message")) +#' head(tmp)} +#' @note raise_error since 3.1.0 +setMethod("raise_error", + signature(x = "characterOrColumn"), + function(x) { + if (is.character(x)) { + stopifnot(length(x) == 1) + x <- lit(x) + } + jc <- callJStatic("org.apache.spark.sql.functions", "raise_error", x@jc) + column(jc) + }) + #' @details #' \code{dayofmonth}: Extracts the day of the month as an integer from a #' given date/timestamp/string. @@ -1417,8 +1485,10 @@ setMethod("quarter", }) #' @details -#' \code{percentile_approx} Returns the approximate percentile value of -#' numeric column at the given percentage. +#' \code{percentile_approx} Returns the approximate \code{percentile} of the numeric column +#' \code{col} which is the smallest value in the ordered \code{col} values (sorted from least to +#' greatest) such that no more than \code{percentage} of \code{col} values is less than the value +#' or equal to that value. #' #' @param percentage Numeric percentage at which percentile should be computed #' All values should be between 0 and 1. @@ -2286,7 +2356,7 @@ setMethod("pmod", signature(y = "Column"), column(jc) }) -#' @param rsd maximum estimation error allowed (default = 0.05). +#' @param rsd maximum relative standard deviation allowed (default = 0.05). #' #' @rdname column_aggregate_functions #' @aliases approx_count_distinct,Column-method @@ -3296,6 +3366,37 @@ setMethod("lead", column(jc) }) +#' @details +#' \code{nth_value}: Window function: returns the value that is the \code{offset}th +#' row of the window frame# (counting from 1), and \code{null} if the size of window +#' frame is less than \code{offset} rows. +#' +#' @param offset a numeric indicating number of row to use as the value +#' @param na.rm a logical which indicates that the Nth value should skip null in the +#' determination of which row to use +#' +#' @rdname column_window_functions +#' @aliases nth_value nth_value,characterOrColumn-method +#' @note nth_value since 3.1.0 +setMethod("nth_value", + signature(x = "characterOrColumn", offset = "numeric"), + function(x, offset, na.rm = FALSE) { + x <- if (is.character(x)) { + column(x) + } else { + x + } + offset <- as.integer(offset) + jc <- callJStatic( + "org.apache.spark.sql.functions", + "nth_value", + x@jc, + offset, + na.rm + ) + column(jc) + }) + #' @details #' \code{ntile}: Returns the ntile group id (from 1 to n inclusive) in an ordered window #' partition. For example, if n is 4, the first quarter of the rows will get value 1, the second @@ -4380,7 +4481,8 @@ setMethod("date_trunc", }) #' @details -#' \code{current_date}: Returns the current date as a date column. +#' \code{current_date}: Returns the current date at the start of query evaluation as a date column. +#' All calls of current_date within the same query return the same value. #' #' @rdname column_datetime_functions #' @aliases current_date current_date,missing-method @@ -4396,7 +4498,8 @@ setMethod("current_date", }) #' @details -#' \code{current_timestamp}: Returns the current timestamp as a timestamp column. +#' \code{current_timestamp}: Returns the current timestamp at the start of query evaluation as +#' a timestamp column. All calls of current_timestamp within the same query return the same value. #' #' @rdname column_datetime_functions #' @aliases current_timestamp current_timestamp,missing-method @@ -4407,3 +4510,40 @@ setMethod("current_timestamp", jc <- callJStatic("org.apache.spark.sql.functions", "current_timestamp") column(jc) }) + +#' @details +#' \code{timestamp_seconds}: Creates timestamp from the number of seconds since UTC epoch. +#' +#' @rdname column_datetime_functions +#' @aliases timestamp_seconds timestamp_seconds,Column-method +#' @note timestamp_seconds since 3.1.0 +setMethod("timestamp_seconds", + signature(x = "Column"), + function(x) { + jc <- callJStatic( + "org.apache.spark.sql.functions", "timestamp_seconds", x@jc + ) + column(jc) + }) + +#' @details +#' \code{vector_to_array} Converts a column of MLlib sparse/dense vectors into +#' a column of dense arrays. +#' +#' @param dtype The data type of the output array. Valid values: "float64" or "float32". +#' +#' @rdname column_ml_functions +#' @aliases vector_to_array vector_to_array,Column-method +#' @note vector_to_array since 3.1.0 +setMethod("vector_to_array", + signature(x = "Column"), + function(x, dtype = c("float64", "float32")) { + dtype <- match.arg(dtype) + jc <- callJStatic( + "org.apache.spark.ml.functions", + "vector_to_array", + x@jc, + dtype + ) + column(jc) + }) diff --git a/R/pkg/R/generics.R b/R/pkg/R/generics.R index 839c00cf21aeb..e372ae27e315a 100644 --- a/R/pkg/R/generics.R +++ b/R/pkg/R/generics.R @@ -638,7 +638,7 @@ setGeneric("union", function(x, y) { standardGeneric("union") }) setGeneric("unionAll", function(x, y) { standardGeneric("unionAll") }) #' @rdname unionByName -setGeneric("unionByName", function(x, y) { standardGeneric("unionByName") }) +setGeneric("unionByName", function(x, y, ...) { standardGeneric("unionByName") }) #' @rdname unpersist setGeneric("unpersist", function(x, ...) { standardGeneric("unpersist") }) @@ -675,6 +675,12 @@ setGeneric("broadcast", function(x) { standardGeneric("broadcast") }) #' @rdname columnfunctions setGeneric("asc", function(x) { standardGeneric("asc") }) +#' @rdname columnfunctions +setGeneric("asc_nulls_first", function(x) { standardGeneric("asc_nulls_first") }) + +#' @rdname columnfunctions +setGeneric("asc_nulls_last", function(x) { standardGeneric("asc_nulls_last") }) + #' @rdname between setGeneric("between", function(x, bounds) { standardGeneric("between") }) @@ -689,6 +695,12 @@ setGeneric("contains", function(x, ...) { standardGeneric("contains") }) #' @rdname columnfunctions setGeneric("desc", function(x) { standardGeneric("desc") }) +#' @rdname columnfunctions +setGeneric("desc_nulls_first", function(x) { standardGeneric("desc_nulls_first") }) + +#' @rdname columnfunctions +setGeneric("desc_nulls_last", function(x) { standardGeneric("desc_nulls_last") }) + #' @rdname endsWith setGeneric("endsWith", function(x, suffix) { standardGeneric("endsWith") }) @@ -729,6 +741,12 @@ setGeneric("over", function(x, window) { standardGeneric("over") }) #' @rdname eq_null_safe setGeneric("%<=>%", function(x, value) { standardGeneric("%<=>%") }) +#' @rdname withField +setGeneric("withField", function(x, fieldName, col) { standardGeneric("withField") }) + +#' @rdname dropFields +setGeneric("dropFields", function(x, ...) { standardGeneric("dropFields") }) + ###################### WindowSpec Methods ########################## #' @rdname partitionBy @@ -844,6 +862,10 @@ setGeneric("arrays_zip_with", function(x, y, f) { standardGeneric("arrays_zip_wi #' @name NULL setGeneric("ascii", function(x) { standardGeneric("ascii") }) +#' @rdname column_misc_functions +#' @name NULL +setGeneric("assert_true", function(x, errMsg = NULL) { standardGeneric("assert_true") }) + #' @param x Column to compute on or a GroupedData object. #' @param ... additional argument(s) when \code{x} is a GroupedData object. #' @rdname avg @@ -1161,6 +1183,10 @@ setGeneric("months_between", function(y, x, ...) { standardGeneric("months_betwe #' @rdname count setGeneric("n", function(x) { standardGeneric("n") }) +#' @rdname column_window_functions +#' @name NULL +setGeneric("nth_value", function(x, offset, ...) { standardGeneric("nth_value") }) + #' @rdname column_nonaggregate_functions #' @name NULL setGeneric("nanvl", function(y, x) { standardGeneric("nanvl") }) @@ -1213,6 +1239,10 @@ setGeneric("posexplode_outer", function(x) { standardGeneric("posexplode_outer") #' @name NULL setGeneric("quarter", function(x) { standardGeneric("quarter") }) +#' @rdname column_misc_functions +#' @name NULL +setGeneric("raise_error", function(x) { standardGeneric("raise_error") }) + #' @rdname column_nonaggregate_functions #' @name NULL setGeneric("rand", function(seed) { standardGeneric("rand") }) @@ -1354,6 +1384,10 @@ setGeneric("substring_index", function(x, delim, count) { standardGeneric("subst #' @name NULL setGeneric("sumDistinct", function(x) { standardGeneric("sumDistinct") }) +#' @rdname column_datetime_functions +#' @name timestamp_seconds +setGeneric("timestamp_seconds", function(x) { standardGeneric("timestamp_seconds") }) + #' @rdname column_collection_functions #' @name NULL setGeneric("transform_keys", function(x, f) { standardGeneric("transform_keys") }) @@ -1438,6 +1472,10 @@ setGeneric("var_pop", function(x) { standardGeneric("var_pop") }) #' @name NULL setGeneric("var_samp", function(x) { standardGeneric("var_samp") }) +#' @rdname column_ml_functions +#' @name NULL +setGeneric("vector_to_array", function(x, ...) { standardGeneric("vector_to_array") }) + #' @rdname column_datetime_functions #' @name NULL setGeneric("weekofyear", function(x) { standardGeneric("weekofyear") }) diff --git a/R/pkg/R/utils.R b/R/pkg/R/utils.R index cef2fa9b47440..d6f9f927d5cdc 100644 --- a/R/pkg/R/utils.R +++ b/R/pkg/R/utils.R @@ -376,6 +376,7 @@ varargsToStrEnv <- function(...) { getStorageLevel <- function(newLevel = c("DISK_ONLY", "DISK_ONLY_2", + "DISK_ONLY_3", "MEMORY_AND_DISK", "MEMORY_AND_DISK_2", "MEMORY_AND_DISK_SER", @@ -390,6 +391,7 @@ getStorageLevel <- function(newLevel = c("DISK_ONLY", storageLevel <- switch(newLevel, "DISK_ONLY" = callJStatic(storageLevelClass, "DISK_ONLY"), "DISK_ONLY_2" = callJStatic(storageLevelClass, "DISK_ONLY_2"), + "DISK_ONLY_3" = callJStatic(storageLevelClass, "DISK_ONLY_3"), "MEMORY_AND_DISK" = callJStatic(storageLevelClass, "MEMORY_AND_DISK"), "MEMORY_AND_DISK_2" = callJStatic(storageLevelClass, "MEMORY_AND_DISK_2"), "MEMORY_AND_DISK_SER" = callJStatic(storageLevelClass, @@ -415,6 +417,8 @@ storageLevelToString <- function(levelObj) { "DISK_ONLY" } else if (useDisk && !useMemory && !useOffHeap && !deserialized && replication == 2) { "DISK_ONLY_2" + } else if (useDisk && !useMemory && !useOffHeap && !deserialized && replication == 3) { + "DISK_ONLY_3" } else if (!useDisk && useMemory && !useOffHeap && deserialized && replication == 1) { "MEMORY_ONLY" } else if (!useDisk && useMemory && !useOffHeap && deserialized && replication == 2) { diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R index e008bc5bbd7d9..22bd4133d46a8 100644 --- a/R/pkg/tests/fulltests/test_sparkSQL.R +++ b/R/pkg/tests/fulltests/test_sparkSQL.R @@ -1424,6 +1424,12 @@ test_that("column functions", { date_trunc("quarter", c) + current_date() + current_timestamp() c25 <- overlay(c1, c2, c3, c3) + overlay(c1, c2, c3) + overlay(c1, c2, 1) + overlay(c1, c2, 3, 4) + c26 <- timestamp_seconds(c1) + vector_to_array(c) + + vector_to_array(c, "float32") + vector_to_array(c, "float64") + c27 <- nth_value("x", 1L) + nth_value("y", 2, TRUE) + + nth_value(column("v"), 3) + nth_value(column("z"), 4L, FALSE) + c28 <- asc_nulls_first(c1) + asc_nulls_last(c1) + + desc_nulls_first(c1) + desc_nulls_last(c1) # Test if base::is.nan() is exposed expect_equal(is.nan(c("a", "b")), c(FALSE, FALSE)) @@ -1711,9 +1717,9 @@ test_that("column functions", { df <- as.DataFrame(list(list("col" = "1"))) c <- collect(select(df, schema_of_json('{"name":"Bob"}'))) - expect_equal(c[[1]], "struct") + expect_equal(c[[1]], "STRUCT<`name`: STRING>") c <- collect(select(df, schema_of_json(lit('{"name":"Bob"}')))) - expect_equal(c[[1]], "struct") + expect_equal(c[[1]], "STRUCT<`name`: STRING>") # Test to_json() supports arrays of primitive types and arrays df <- sql("SELECT array(19, 42, 70) as age") @@ -1803,6 +1809,36 @@ test_that("column functions", { ) expect_equal(actual, expected) + + # Test withField + lines <- c("{\"Person\": {\"name\":\"Bob\", \"age\":24, \"height\": 170}}") + jsonPath <- tempfile(pattern = "sparkr-test", fileext = ".tmp") + writeLines(lines, jsonPath) + df <- read.df(jsonPath, "json") + result <- collect( + select( + select(df, alias(withField(df$Person, "dummy", lit(42)), "Person")), + "Person.dummy" + ) + ) + expect_equal(result, data.frame(dummy = 42)) + + # Test dropFields + expect_setequal( + colnames(select( + withColumn(df, "Person", dropFields(df$Person, "age")), + column("Person.*") + )), + c("name", "height") + ) + + expect_equal( + colnames(select( + withColumn(df, "Person", dropFields(df$Person, "height", "name")), + column("Person.*") + )), + "age" + ) }) test_that("column binary mathfunctions", { @@ -2113,7 +2149,7 @@ test_that("group by, agg functions", { df3 <- agg(gd, age = "stddev") expect_is(df3, "SparkDataFrame") df3_local <- collect(df3) - expect_true(is.nan(df3_local[df3_local$name == "Andy", ][1, 2])) + expect_true(is.na(df3_local[df3_local$name == "Andy", ][1, 2])) df4 <- agg(gd, sumAge = sum(df$age)) expect_is(df4, "SparkDataFrame") @@ -2144,7 +2180,7 @@ test_that("group by, agg functions", { df7 <- agg(gd2, value = "stddev") df7_local <- collect(df7) expect_true(abs(df7_local[df7_local$name == "ID1", ][1, 2] - 6.928203) < 1e-6) - expect_true(is.nan(df7_local[df7_local$name == "ID2", ][1, 2])) + expect_true(is.na(df7_local[df7_local$name == "ID2", ][1, 2])) mockLines3 <- c("{\"name\":\"Andy\", \"age\":30}", "{\"name\":\"Andy\", \"age\":30}", @@ -2696,6 +2732,19 @@ test_that("union(), unionByName(), rbind(), except(), and intersect() on a DataF expect_error(rbind(df, df2, df3), "Names of input data frames are different.") + + df4 <- unionByName(df2, select(df2, "age"), TRUE) + + expect_equal( + sum(collect( + select(df4, alias(isNull(df4$name), "missing_name") + ))$missing_name), + 3 + ) + + testthat::expect_error(unionByName(df2, select(df2, "age"), FALSE)) + testthat::expect_error(unionByName(df2, select(df2, "age"))) + excepted <- arrange(except(df, df2), desc(df$age)) expect_is(unioned, "SparkDataFrame") expect_equal(count(excepted), 2) @@ -3898,6 +3947,24 @@ test_that("catalog APIs, listTables, listColumns, listFunctions", { dropTempView("cars") }) +test_that("assert_true, raise_error", { + df <- read.json(jsonPath) + filtered <- filter(df, "age < 20") + + expect_equal(collect(select(filtered, assert_true(filtered$age < 20)))$age, c(NULL)) + expect_equal(collect(select(filtered, assert_true(filtered$age < 20, "error message")))$age, + c(NULL)) + expect_equal(collect(select(filtered, assert_true(filtered$age < 20, filtered$name)))$age, + c(NULL)) + expect_error(collect(select(df, assert_true(df$age < 20))), "is not true!") + expect_error(collect(select(df, assert_true(df$age < 20, "error message"))), + "error message") + expect_error(collect(select(df, assert_true(df$age < 20, df$name))), "Michael") + + expect_error(collect(select(filtered, raise_error("error message"))), "error message") + expect_error(collect(select(filtered, raise_error(filtered$name))), "Justin") +}) + compare_list <- function(list1, list2) { # get testthat to show the diff by first making the 2 lists equal in length expect_equal(length(list1), length(list2)) diff --git a/R/pkg/tests/fulltests/test_sparkSQL_arrow.R b/R/pkg/tests/fulltests/test_sparkSQL_arrow.R index 16d93763ff038..06743488fdf11 100644 --- a/R/pkg/tests/fulltests/test_sparkSQL_arrow.R +++ b/R/pkg/tests/fulltests/test_sparkSQL_arrow.R @@ -19,7 +19,10 @@ library(testthat) context("SparkSQL Arrow optimization") -sparkSession <- sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE) +sparkSession <- sparkR.session( + master = sparkRTestMaster, + enableHiveSupport = FALSE, + sparkConfig = list(spark.sql.execution.arrow.sparkr.enabled = "true")) test_that("createDataFrame/collect Arrow optimization", { skip_if_not_installed("arrow") @@ -35,29 +38,13 @@ test_that("createDataFrame/collect Arrow optimization", { callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) }) - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true") - tryCatch({ - expect_equal(collect(createDataFrame(mtcars)), expected) - }, - finally = { - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) - }) + expect_equal(collect(createDataFrame(mtcars)), expected) }) test_that("createDataFrame/collect Arrow optimization - many partitions (partition order test)", { skip_if_not_installed("arrow") - - conf <- callJMethod(sparkSession, "conf") - arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]] - - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true") - tryCatch({ - expect_equal(collect(createDataFrame(mtcars, numPartitions = 32)), - collect(createDataFrame(mtcars, numPartitions = 1))) - }, - finally = { - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) - }) + expect_equal(collect(createDataFrame(mtcars, numPartitions = 32)), + collect(createDataFrame(mtcars, numPartitions = 1))) }) test_that("createDataFrame/collect Arrow optimization - type specification", { @@ -81,13 +68,7 @@ test_that("createDataFrame/collect Arrow optimization - type specification", { callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) }) - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true") - tryCatch({ - expect_equal(collect(createDataFrame(rdf)), expected) - }, - finally = { - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) - }) + expect_equal(collect(createDataFrame(rdf)), expected) }) test_that("dapply() Arrow optimization", { @@ -98,36 +79,30 @@ test_that("dapply() Arrow optimization", { arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]] callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "false") - tryCatch({ - ret <- dapply(df, - function(rdf) { - stopifnot(is.data.frame(rdf)) - rdf - }, - schema(df)) - expected <- collect(ret) - }, - finally = { - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) - }) - - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true") tryCatch({ ret <- dapply(df, function(rdf) { stopifnot(is.data.frame(rdf)) - # mtcars' hp is more then 50. - stopifnot(all(rdf$hp > 50)) rdf }, schema(df)) - actual <- collect(ret) - expect_equal(actual, expected) - expect_equal(count(ret), nrow(mtcars)) + expected <- collect(ret) }, finally = { callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) }) + + ret <- dapply(df, + function(rdf) { + stopifnot(is.data.frame(rdf)) + # mtcars' hp is more then 50. + stopifnot(all(rdf$hp > 50)) + rdf + }, + schema(df)) + actual <- collect(ret) + expect_equal(actual, expected) + expect_equal(count(ret), nrow(mtcars)) }) test_that("dapply() Arrow optimization - type specification", { @@ -154,15 +129,9 @@ test_that("dapply() Arrow optimization - type specification", { callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) }) - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true") - tryCatch({ - ret <- dapply(df, function(rdf) { rdf }, schema(df)) - actual <- collect(ret) - expect_equal(actual, expected) - }, - finally = { - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) - }) + ret <- dapply(df, function(rdf) { rdf }, schema(df)) + actual <- collect(ret) + expect_equal(actual, expected) }) test_that("dapply() Arrow optimization - type specification (date and timestamp)", { @@ -170,18 +139,8 @@ test_that("dapply() Arrow optimization - type specification (date and timestamp) rdf <- data.frame(list(list(a = as.Date("1990-02-24"), b = as.POSIXct("1990-02-24 12:34:56")))) df <- createDataFrame(rdf) - - conf <- callJMethod(sparkSession, "conf") - arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]] - - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true") - tryCatch({ - ret <- dapply(df, function(rdf) { rdf }, schema(df)) - expect_equal(collect(ret), rdf) - }, - finally = { - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) - }) + ret <- dapply(df, function(rdf) { rdf }, schema(df)) + expect_equal(collect(ret), rdf) }) test_that("gapply() Arrow optimization", { @@ -209,28 +168,22 @@ test_that("gapply() Arrow optimization", { callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) }) - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true") - tryCatch({ - ret <- gapply(df, - "gear", - function(key, grouped) { - if (length(key) > 0) { - stopifnot(is.numeric(key[[1]])) - } - stopifnot(is.data.frame(grouped)) - stopifnot(length(colnames(grouped)) == 11) - # mtcars' hp is more then 50. - stopifnot(all(grouped$hp > 50)) - grouped - }, - schema(df)) - actual <- collect(ret) - expect_equal(actual, expected) - expect_equal(count(ret), nrow(mtcars)) - }, - finally = { - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) - }) + ret <- gapply(df, + "gear", + function(key, grouped) { + if (length(key) > 0) { + stopifnot(is.numeric(key[[1]])) + } + stopifnot(is.data.frame(grouped)) + stopifnot(length(colnames(grouped)) == 11) + # mtcars' hp is more then 50. + stopifnot(all(grouped$hp > 50)) + grouped + }, + schema(df)) + actual <- collect(ret) + expect_equal(actual, expected) + expect_equal(count(ret), nrow(mtcars)) }) test_that("gapply() Arrow optimization - type specification", { @@ -250,26 +203,19 @@ test_that("gapply() Arrow optimization - type specification", { callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "false") tryCatch({ ret <- gapply(df, - "a", - function(key, grouped) { grouped }, schema(df)) + "a", + function(key, grouped) { grouped }, schema(df)) expected <- collect(ret) }, finally = { callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) }) - - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true") - tryCatch({ - ret <- gapply(df, - "a", - function(key, grouped) { grouped }, schema(df)) - actual <- collect(ret) - expect_equal(actual, expected) - }, - finally = { - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) - }) + ret <- gapply(df, + "a", + function(key, grouped) { grouped }, schema(df)) + actual <- collect(ret) + expect_equal(actual, expected) }) test_that("gapply() Arrow optimization - type specification (date and timestamp)", { @@ -277,57 +223,30 @@ test_that("gapply() Arrow optimization - type specification (date and timestamp) rdf <- data.frame(list(list(a = as.Date("1990-02-24"), b = as.POSIXct("1990-02-24 12:34:56")))) df <- createDataFrame(rdf) - - conf <- callJMethod(sparkSession, "conf") - arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]] - - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true") - tryCatch({ - ret <- gapply(df, - "a", - function(key, grouped) { grouped }, schema(df)) - expect_equal(collect(ret), rdf) - }, - finally = { - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) - }) + ret <- gapply(df, + "a", + function(key, grouped) { grouped }, schema(df)) + expect_equal(collect(ret), rdf) }) test_that("Arrow optimization - unsupported types", { skip_if_not_installed("arrow") - conf <- callJMethod(sparkSession, "conf") - arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]] - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true") - tryCatch({ - expect_error(checkSchemaInArrow(structType("a FLOAT")), "not support float type") - expect_error(checkSchemaInArrow(structType("a BINARY")), "not support binary type") - expect_error(checkSchemaInArrow(structType("a ARRAY")), "not support array type") - expect_error(checkSchemaInArrow(structType("a MAP")), "not support map type") - expect_error(checkSchemaInArrow(structType("a STRUCT")), - "not support nested struct type") - }, - finally = { - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) - }) + expect_error(checkSchemaInArrow(structType("a FLOAT")), "not support float type") + expect_error(checkSchemaInArrow(structType("a BINARY")), "not support binary type") + expect_error(checkSchemaInArrow(structType("a ARRAY")), "not support array type") + expect_error(checkSchemaInArrow(structType("a MAP")), "not support map type") + expect_error(checkSchemaInArrow(structType("a STRUCT")), + "not support nested struct type") }) test_that("SPARK-32478: gapply() Arrow optimization - error message for schema mismatch", { skip_if_not_installed("arrow") df <- createDataFrame(list(list(a = 1L, b = "a"))) - conf <- callJMethod(sparkSession, "conf") - arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]] - - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true") - tryCatch({ - expect_error( + expect_error( count(gapply(df, "a", function(key, group) { group }, structType("a int, b int"))), "expected IntegerType, IntegerType, got IntegerType, StringType") - }, - finally = { - callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled) - }) }) sparkR.session.stop() diff --git a/R/pkg/tests/run-all.R b/R/pkg/tests/run-all.R index a46924a5d20e3..3043df0f12075 100644 --- a/R/pkg/tests/run-all.R +++ b/R/pkg/tests/run-all.R @@ -61,15 +61,18 @@ if (identical(Sys.getenv("NOT_CRAN"), "true")) { set.seed(42) # TODO (SPARK-30663) To be removed once testthat 1.x is removed from all builds - if (grepl("^1\\..*", packageVersion("testthat"))) { + if (packageVersion("testthat")$major <= 1) { # testthat 1.x test_runner <- testthat:::run_tests reporter <- "summary" - } else { # testthat >= 2.0.0 test_runner <- testthat:::test_package_dir - reporter <- testthat::default_reporter() + dir.create("target/test-reports", showWarnings = FALSE) + reporter <- MultiReporter$new(list( + SummaryReporter$new(), + JunitReporter$new(file = "target/test-reports/test-results.xml") + )) } test_runner("SparkR", diff --git a/appveyor.yml b/appveyor.yml index 1fd91daae9015..c40b23c8341eb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -41,8 +41,8 @@ cache: install: # Install maven and dependencies - ps: .\dev\appveyor-install-dependencies.ps1 - # Required package for R unit tests - - cmd: Rscript -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow'), repos='https://cloud.r-project.org/')" + # Required package for R unit tests. xml2 is required to use jUnit reporter in testthat. + - cmd: Rscript -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow', 'xml2'), repos='https://cloud.r-project.org/')" - cmd: Rscript -e "pkg_list <- as.data.frame(installed.packages()[,c(1, 3:4)]); pkg_list[is.na(pkg_list$Priority), 1:2, drop = FALSE]" build_script: diff --git a/bin/find-spark-home.cmd b/bin/find-spark-home.cmd index f795d146d49c7..3149d05039ba4 100644 --- a/bin/find-spark-home.cmd +++ b/bin/find-spark-home.cmd @@ -55,6 +55,6 @@ if "x%SPARK_HOME%"=="x" ( set SPARK_HOME=%~dp0.. ) else ( rem We are pip installed, use the Python script to resolve a reasonable SPARK_HOME - for /f "delims=" %%i in ('%PYTHON_RUNNER% %FIND_SPARK_HOME_PYTHON_SCRIPT%') do set SPARK_HOME=%%i + for /f "delims=" %%i in ('%PYTHON_RUNNER% "%FIND_SPARK_HOME_PYTHON_SCRIPT%"') do set SPARK_HOME=%%i ) ) diff --git a/bin/load-spark-env.cmd b/bin/load-spark-env.cmd index fe725a4e1a368..5692af529fb66 100644 --- a/bin/load-spark-env.cmd +++ b/bin/load-spark-env.cmd @@ -24,7 +24,7 @@ rem conf\ subdirectory. if not defined SPARK_ENV_LOADED ( set SPARK_ENV_LOADED=1 - if [%SPARK_CONF_DIR%] == [] ( + if not defined SPARK_CONF_DIR ( set SPARK_CONF_DIR=%~dp0..\conf ) @@ -36,8 +36,8 @@ rem Setting SPARK_SCALA_VERSION if not already set. set SCALA_VERSION_1=2.13 set SCALA_VERSION_2=2.12 -set ASSEMBLY_DIR1=%SPARK_HOME%\assembly\target\scala-%SCALA_VERSION_1% -set ASSEMBLY_DIR2=%SPARK_HOME%\assembly\target\scala-%SCALA_VERSION_2% +set ASSEMBLY_DIR1="%SPARK_HOME%\assembly\target\scala-%SCALA_VERSION_1%" +set ASSEMBLY_DIR2="%SPARK_HOME%\assembly\target\scala-%SCALA_VERSION_2%" set ENV_VARIABLE_DOC=https://spark.apache.org/docs/latest/configuration.html#environment-variables if not defined SPARK_SCALA_VERSION ( diff --git a/bin/spark-class2.cmd b/bin/spark-class2.cmd old mode 100644 new mode 100755 index 34d04c9856d2c..68b271d1d05d9 --- a/bin/spark-class2.cmd +++ b/bin/spark-class2.cmd @@ -30,12 +30,12 @@ if "x%1"=="x" ( rem Find Spark jars. if exist "%SPARK_HOME%\jars" ( - set SPARK_JARS_DIR="%SPARK_HOME%\jars" + set SPARK_JARS_DIR=%SPARK_HOME%\jars ) else ( - set SPARK_JARS_DIR="%SPARK_HOME%\assembly\target\scala-%SPARK_SCALA_VERSION%\jars" + set SPARK_JARS_DIR=%SPARK_HOME%\assembly\target\scala-%SPARK_SCALA_VERSION%\jars ) -if not exist "%SPARK_JARS_DIR%"\ ( +if not exist "%SPARK_JARS_DIR%" ( echo Failed to find Spark jars directory. echo You need to build Spark before running this program. exit /b 1 diff --git a/binder/apt.txt b/binder/apt.txt new file mode 100644 index 0000000000000..385f5b0fba754 --- /dev/null +++ b/binder/apt.txt @@ -0,0 +1 @@ +openjdk-8-jre diff --git a/binder/postBuild b/binder/postBuild new file mode 100644 index 0000000000000..42bb3514c5a2e --- /dev/null +++ b/binder/postBuild @@ -0,0 +1,24 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This file is used for Binder integration to install PySpark available in +# Jupyter notebook. + +VERSION=$(python -c "exec(open('python/pyspark/version.py').read()); print(__version__)") +pip install "pyspark[sql,ml,mllib]<=$VERSION" diff --git a/build/sbt-launch-lib.bash b/build/sbt-launch-lib.bash index 162bfbf2257c7..1d79989f3c3c3 100755 --- a/build/sbt-launch-lib.bash +++ b/build/sbt-launch-lib.bash @@ -39,7 +39,11 @@ dlog () { acquire_sbt_jar () { SBT_VERSION=`awk -F "=" '/sbt\.version/ {print $2}' ./project/build.properties` - URL1=https://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar + # DEFAULT_ARTIFACT_REPOSITORY env variable can be used to only fetch + # artifacts from internal repos only. + # Ex: + # DEFAULT_ARTIFACT_REPOSITORY=https://artifacts.internal.com/libs-release/ + URL1=${DEFAULT_ARTIFACT_REPOSITORY:-https://repo1.maven.org/maven2/}org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar JAR=build/sbt-launch-${SBT_VERSION}.jar sbt_jar=$JAR diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java index 42e090bc83ed1..431c7e42774e4 100644 --- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java +++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java @@ -164,8 +164,9 @@ public void clear() { } /** - * An alias class for the type "ConcurrentHashMap, Boolean>", which is used - * as a concurrent hashset for storing natural keys and the boolean value doesn't matter. + * An alias class for the type "{@literal ConcurrentHashMap, Boolean>}", + * which is used as a concurrent hashset for storing natural keys + * and the boolean value doesn't matter. */ private static class NaturalKeys extends ConcurrentHashMap, Boolean> {} diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 9d5bc9aae0719..d328a7de0a762 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -91,6 +91,10 @@ org.apache.commons commons-crypto + + org.roaringbitmap + RoaringBitmap + diff --git a/common/network-common/src/main/java/org/apache/spark/network/protocol/Encoders.java b/common/network-common/src/main/java/org/apache/spark/network/protocol/Encoders.java index 490915f6de4b3..4fa191b3917e3 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/protocol/Encoders.java +++ b/common/network-common/src/main/java/org/apache/spark/network/protocol/Encoders.java @@ -17,9 +17,11 @@ package org.apache.spark.network.protocol; +import java.io.IOException; import java.nio.charset.StandardCharsets; import io.netty.buffer.ByteBuf; +import org.roaringbitmap.RoaringBitmap; /** Provides a canonical set of Encoders for simple types. */ public class Encoders { @@ -44,6 +46,40 @@ public static String decode(ByteBuf buf) { } } + /** Bitmaps are encoded with their serialization length followed by the serialization bytes. */ + public static class Bitmaps { + public static int encodedLength(RoaringBitmap b) { + // Compress the bitmap before serializing it. Note that since BlockTransferMessage + // needs to invoke encodedLength first to figure out the length for the ByteBuf, it + // guarantees that the bitmap will always be compressed before being serialized. + b.trim(); + b.runOptimize(); + return b.serializedSizeInBytes(); + } + + public static void encode(ByteBuf buf, RoaringBitmap b) { + int encodedLength = b.serializedSizeInBytes(); + // RoaringBitmap requires nio ByteBuffer for serde. We expose the netty ByteBuf as a nio + // ByteBuffer. Here, we need to explicitly manage the index so we can write into the + // ByteBuffer, and the write is reflected in the underneath ByteBuf. + b.serialize(buf.nioBuffer(buf.writerIndex(), encodedLength)); + buf.writerIndex(buf.writerIndex() + encodedLength); + } + + public static RoaringBitmap decode(ByteBuf buf) { + RoaringBitmap bitmap = new RoaringBitmap(); + try { + bitmap.deserialize(buf.nioBuffer()); + // RoaringBitmap deserialize does not advance the reader index of the underlying ByteBuf. + // Manually update the index here. + buf.readerIndex(buf.readerIndex() + bitmap.serializedSizeInBytes()); + } catch (IOException e) { + throw new RuntimeException("Exception while decoding bitmap", e); + } + return bitmap; + } + } + /** Byte arrays are encoded with their length followed by bytes. */ public static class ByteArrays { public static int encodedLength(byte[] arr) { @@ -135,4 +171,31 @@ public static long[] decode(ByteBuf buf) { return longs; } } + + /** Bitmap arrays are encoded with the number of bitmaps followed by per-Bitmap encoding. */ + public static class BitmapArrays { + public static int encodedLength(RoaringBitmap[] bitmaps) { + int totalLength = 4; + for (RoaringBitmap b : bitmaps) { + totalLength += Bitmaps.encodedLength(b); + } + return totalLength; + } + + public static void encode(ByteBuf buf, RoaringBitmap[] bitmaps) { + buf.writeInt(bitmaps.length); + for (RoaringBitmap b : bitmaps) { + Bitmaps.encode(buf, b); + } + } + + public static RoaringBitmap[] decode(ByteBuf buf) { + int numBitmaps = buf.readInt(); + RoaringBitmap[] bitmaps = new RoaringBitmap[numBitmaps]; + for (int i = 0; i < bitmaps.length; i ++) { + bitmaps[i] = Bitmaps.decode(buf); + } + return bitmaps; + } + } } diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java index 6c37f9a382376..646e4278811f4 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java @@ -290,7 +290,7 @@ public boolean sharedByteBufAllocators() { } /** - * If enabled then off-heap byte buffers will be prefered for the shared ByteBuf allocators. + * If enabled then off-heap byte buffers will be preferred for the shared ByteBuf allocators. */ public boolean preferDirectBufsForSharedByteBufAllocators() { return conf.getBoolean("spark.network.io.preferDirectBufs", true); diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index 00f1defbb0093..a4a1ff92ef9a0 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -57,6 +57,10 @@ com.google.guava guava + + org.roaringbitmap + RoaringBitmap + @@ -93,6 +97,11 @@ mockito-core test + + commons-io + commons-io + test + diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/BlockStoreClient.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/BlockStoreClient.java index fbbe8ac0f1f9b..37befcd4b67fa 100644 --- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/BlockStoreClient.java +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/BlockStoreClient.java @@ -18,15 +18,34 @@ package org.apache.spark.network.shuffle; import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CompletableFuture; import com.codahale.metrics.MetricSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.spark.network.buffer.ManagedBuffer; +import org.apache.spark.network.client.RpcResponseCallback; +import org.apache.spark.network.client.TransportClient; +import org.apache.spark.network.client.TransportClientFactory; +import org.apache.spark.network.shuffle.protocol.BlockTransferMessage; +import org.apache.spark.network.shuffle.protocol.GetLocalDirsForExecutors; +import org.apache.spark.network.shuffle.protocol.LocalDirsForExecutors; /** * Provides an interface for reading both shuffle files and RDD blocks, either from an Executor * or external service. */ public abstract class BlockStoreClient implements Closeable { + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + + protected volatile TransportClientFactory clientFactory; + protected String appId; /** * Fetch a sequence of blocks from a remote node asynchronously, @@ -61,4 +80,80 @@ public MetricSet shuffleMetrics() { // Return an empty MetricSet by default. return () -> Collections.emptyMap(); } + + protected void checkInit() { + assert appId != null : "Called before init()"; + } + + /** + * Request the local disk directories for executors which are located at the same host with + * the current BlockStoreClient(it can be ExternalBlockStoreClient or NettyBlockTransferService). + * + * @param host the host of BlockManager or ExternalShuffleService. It should be the same host + * with current BlockStoreClient. + * @param port the port of BlockManager or ExternalShuffleService. + * @param execIds a collection of executor Ids, which specifies the target executors that we + * want to get their local directories. There could be multiple executor Ids if + * BlockStoreClient is implemented by ExternalBlockStoreClient since the request + * handler, ExternalShuffleService, can serve multiple executors on the same node. + * Or, only one executor Id if BlockStoreClient is implemented by + * NettyBlockTransferService. + * @param hostLocalDirsCompletable a CompletableFuture which contains a map from executor Id + * to its local directories if the request handler replies + * successfully. Otherwise, it contains a specific error. + */ + public void getHostLocalDirs( + String host, + int port, + String[] execIds, + CompletableFuture> hostLocalDirsCompletable) { + checkInit(); + GetLocalDirsForExecutors getLocalDirsMessage = new GetLocalDirsForExecutors(appId, execIds); + try { + TransportClient client = clientFactory.createClient(host, port); + client.sendRpc(getLocalDirsMessage.toByteBuffer(), new RpcResponseCallback() { + @Override + public void onSuccess(ByteBuffer response) { + try { + BlockTransferMessage msgObj = BlockTransferMessage.Decoder.fromByteBuffer(response); + hostLocalDirsCompletable.complete( + ((LocalDirsForExecutors) msgObj).getLocalDirsByExec()); + } catch (Throwable t) { + logger.warn("Error while trying to get the host local dirs for " + + Arrays.toString(getLocalDirsMessage.execIds), t.getCause()); + hostLocalDirsCompletable.completeExceptionally(t); + } + } + + @Override + public void onFailure(Throwable t) { + logger.warn("Error while trying to get the host local dirs for " + + Arrays.toString(getLocalDirsMessage.execIds), t.getCause()); + hostLocalDirsCompletable.completeExceptionally(t); + } + }); + } catch (IOException | InterruptedException e) { + hostLocalDirsCompletable.completeExceptionally(e); + } + } + + /** + * Push a sequence of shuffle blocks in a best-effort manner to a remote node asynchronously. + * These shuffle blocks, along with blocks pushed by other clients, will be merged into + * per-shuffle partition merged shuffle files on the destination node. + * + * @param host the host of the remote node. + * @param port the port of the remote node. + * @param blockIds block ids to be pushed + * @param buffers buffers to be pushed + * @param listener the listener to receive block push status. + */ + public void pushBlocks( + String host, + int port, + String[] blockIds, + ManagedBuffer[] buffers, + BlockFetchingListener listener) { + throw new UnsupportedOperationException(); + } } diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ErrorHandler.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ErrorHandler.java new file mode 100644 index 0000000000000..308b0b7a6b33b --- /dev/null +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ErrorHandler.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.network.shuffle; + +import java.net.ConnectException; + +import com.google.common.base.Throwables; + +/** + * Plugs into {@link RetryingBlockFetcher} to further control when an exception should be retried + * and logged. + * Note: {@link RetryingBlockFetcher} will delegate the exception to this handler only when + * - remaining retries < max retries + * - exception is an IOException + */ + +public interface ErrorHandler { + + boolean shouldRetryError(Throwable t); + + default boolean shouldLogError(Throwable t) { + return true; + } + + /** + * A no-op error handler instance. + */ + ErrorHandler NOOP_ERROR_HANDLER = t -> true; + + /** + * The error handler for pushing shuffle blocks to remote shuffle services. + */ + class BlockPushErrorHandler implements ErrorHandler { + /** + * String constant used for generating exception messages indicating a block to be merged + * arrives too late on the server side, and also for later checking such exceptions on the + * client side. When we get a block push failure because of the block arrives too late, we + * will not retry pushing the block nor log the exception on the client side. + */ + public static final String TOO_LATE_MESSAGE_SUFFIX = + "received after merged shuffle is finalized"; + + /** + * String constant used for generating exception messages indicating the server couldn't + * append a block after all available attempts due to collision with other blocks belonging + * to the same shuffle partition, and also for later checking such exceptions on the client + * side. When we get a block push failure because of the block couldn't be written due to + * this reason, we will not log the exception on the client side. + */ + public static final String BLOCK_APPEND_COLLISION_DETECTED_MSG_PREFIX = + "Couldn't find an opportunity to write block"; + + @Override + public boolean shouldRetryError(Throwable t) { + // If it is a connection time out or a connection closed exception, no need to retry. + if (t.getCause() != null && t.getCause() instanceof ConnectException) { + return false; + } + // If the block is too late, there is no need to retry it + return !Throwables.getStackTraceAsString(t).contains(TOO_LATE_MESSAGE_SUFFIX); + } + + @Override + public boolean shouldLogError(Throwable t) { + String errorStackTrace = Throwables.getStackTraceAsString(t); + return !errorStackTrace.contains(BLOCK_APPEND_COLLISION_DETECTED_MSG_PREFIX) && + !errorStackTrace.contains(TOO_LATE_MESSAGE_SUFFIX); + } + } +} diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java index 33865a21ea914..321b25305c504 100644 --- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java @@ -32,6 +32,7 @@ import com.codahale.metrics.Timer; import com.codahale.metrics.Counter; import com.google.common.annotations.VisibleForTesting; +import org.apache.spark.network.client.StreamCallbackWithID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,11 +62,21 @@ public class ExternalBlockHandler extends RpcHandler { final ExternalShuffleBlockResolver blockManager; private final OneForOneStreamManager streamManager; private final ShuffleMetrics metrics; + private final MergedShuffleFileManager mergeManager; public ExternalBlockHandler(TransportConf conf, File registeredExecutorFile) throws IOException { this(new OneForOneStreamManager(), - new ExternalShuffleBlockResolver(conf, registeredExecutorFile)); + new ExternalShuffleBlockResolver(conf, registeredExecutorFile), + new NoOpMergedShuffleFileManager()); + } + + public ExternalBlockHandler( + TransportConf conf, + File registeredExecutorFile, + MergedShuffleFileManager mergeManager) throws IOException { + this(new OneForOneStreamManager(), + new ExternalShuffleBlockResolver(conf, registeredExecutorFile), mergeManager); } @VisibleForTesting @@ -78,9 +89,19 @@ public ExternalShuffleBlockResolver getBlockResolver() { public ExternalBlockHandler( OneForOneStreamManager streamManager, ExternalShuffleBlockResolver blockManager) { + this(streamManager, blockManager, new NoOpMergedShuffleFileManager()); + } + + /** Enables mocking out the StreamManager, BlockManager, and MergeManager. */ + @VisibleForTesting + public ExternalBlockHandler( + OneForOneStreamManager streamManager, + ExternalShuffleBlockResolver blockManager, + MergedShuffleFileManager mergeManager) { this.metrics = new ShuffleMetrics(); this.streamManager = streamManager; this.blockManager = blockManager; + this.mergeManager = mergeManager; } @Override @@ -89,6 +110,21 @@ public void receive(TransportClient client, ByteBuffer message, RpcResponseCallb handleMessage(msgObj, client, callback); } + @Override + public StreamCallbackWithID receiveStream( + TransportClient client, + ByteBuffer messageHeader, + RpcResponseCallback callback) { + BlockTransferMessage msgObj = BlockTransferMessage.Decoder.fromByteBuffer(messageHeader); + if (msgObj instanceof PushBlockStream) { + PushBlockStream message = (PushBlockStream) msgObj; + checkAuth(client, message.appId); + return mergeManager.receiveBlockDataAsStream(message); + } else { + throw new UnsupportedOperationException("Unexpected message with #receiveStream: " + msgObj); + } + } + protected void handleMessage( BlockTransferMessage msgObj, TransportClient client, @@ -139,6 +175,7 @@ protected void handleMessage( RegisterExecutor msg = (RegisterExecutor) msgObj; checkAuth(client, msg.appId); blockManager.registerExecutor(msg.appId, msg.execId, msg.executorInfo); + mergeManager.registerExecutor(msg.appId, msg.executorInfo.localDirs); callback.onSuccess(ByteBuffer.wrap(new byte[0])); } finally { responseDelayContext.stop(); @@ -156,6 +193,20 @@ protected void handleMessage( Map localDirs = blockManager.getLocalDirs(msg.appId, msg.execIds); callback.onSuccess(new LocalDirsForExecutors(localDirs).toByteBuffer()); + } else if (msgObj instanceof FinalizeShuffleMerge) { + final Timer.Context responseDelayContext = + metrics.finalizeShuffleMergeLatencyMillis.time(); + FinalizeShuffleMerge msg = (FinalizeShuffleMerge) msgObj; + try { + checkAuth(client, msg.appId); + MergeStatuses statuses = mergeManager.finalizeShuffleMerge(msg); + callback.onSuccess(statuses.toByteBuffer()); + } catch(IOException e) { + throw new RuntimeException(String.format("Error while finalizing shuffle merge " + + "for application %s shuffle %d", msg.appId, msg.shuffleId), e); + } finally { + responseDelayContext.stop(); + } } else { throw new UnsupportedOperationException("Unexpected message: " + msgObj); } @@ -225,6 +276,8 @@ public class ShuffleMetrics implements MetricSet { private final Timer openBlockRequestLatencyMillis = new Timer(); // Time latency for executor registration latency in ms private final Timer registerExecutorRequestLatencyMillis = new Timer(); + // Time latency for processing finalize shuffle merge request latency in ms + private final Timer finalizeShuffleMergeLatencyMillis = new Timer(); // Block transfer rate in byte per second private final Meter blockTransferRateBytes = new Meter(); // Number of active connections to the shuffle service @@ -236,6 +289,7 @@ public ShuffleMetrics() { allMetrics = new HashMap<>(); allMetrics.put("openBlockRequestLatencyMillis", openBlockRequestLatencyMillis); allMetrics.put("registerExecutorRequestLatencyMillis", registerExecutorRequestLatencyMillis); + allMetrics.put("finalizeShuffleMergeLatencyMillis", finalizeShuffleMergeLatencyMillis); allMetrics.put("blockTransferRateBytes", blockTransferRateBytes); allMetrics.put("registeredExecutorsSize", (Gauge) () -> blockManager.getRegisteredExecutorsSize()); @@ -373,6 +427,54 @@ public ManagedBuffer next() { } } + /** + * Dummy implementation of merged shuffle file manager. Suitable for when push-based shuffle + * is not enabled. + */ + private static class NoOpMergedShuffleFileManager implements MergedShuffleFileManager { + + @Override + public StreamCallbackWithID receiveBlockDataAsStream(PushBlockStream msg) { + throw new UnsupportedOperationException("Cannot handle shuffle block merge"); + } + + @Override + public MergeStatuses finalizeShuffleMerge(FinalizeShuffleMerge msg) throws IOException { + throw new UnsupportedOperationException("Cannot handle shuffle block merge"); + } + + @Override + public void registerApplication(String appId, String user) { + // No-op. Do nothing. + } + + @Override + public void registerExecutor(String appId, String[] localDirs) { + // No-Op. Do nothing. + } + + @Override + public void applicationRemoved(String appId, boolean cleanupLocalDirs) { + throw new UnsupportedOperationException("Cannot handle shuffle block merge"); + } + + @Override + public ManagedBuffer getMergedBlockData( + String appId, int shuffleId, int reduceId, int chunkId) { + throw new UnsupportedOperationException("Cannot handle shuffle block merge"); + } + + @Override + public MergedBlockMeta getMergedBlockMeta(String appId, int shuffleId, int reduceId) { + throw new UnsupportedOperationException("Cannot handle shuffle block merge"); + } + + @Override + public String[] getMergedBlockDirs(String appId) { + throw new UnsupportedOperationException("Cannot handle shuffle block merge"); + } + } + @Override public void channelActive(TransportClient client) { metrics.activeConnections.inc(); diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java index 51dc3337261b9..eca35ed290467 100644 --- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -27,18 +28,16 @@ import com.codahale.metrics.MetricSet; import com.google.common.collect.Lists; + +import org.apache.spark.network.TransportContext; +import org.apache.spark.network.buffer.ManagedBuffer; import org.apache.spark.network.client.RpcResponseCallback; import org.apache.spark.network.client.TransportClient; import org.apache.spark.network.client.TransportClientBootstrap; -import org.apache.spark.network.client.TransportClientFactory; -import org.apache.spark.network.shuffle.protocol.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.spark.network.TransportContext; import org.apache.spark.network.crypto.AuthClientBootstrap; import org.apache.spark.network.sasl.SecretKeyHolder; import org.apache.spark.network.server.NoOpRpcHandler; +import org.apache.spark.network.shuffle.protocol.*; import org.apache.spark.network.util.TransportConf; /** @@ -47,16 +46,13 @@ * (via BlockTransferService), which has the downside of losing the data if we lose the executors. */ public class ExternalBlockStoreClient extends BlockStoreClient { - private static final Logger logger = LoggerFactory.getLogger(ExternalBlockStoreClient.class); + private static final ErrorHandler PUSH_ERROR_HANDLER = new ErrorHandler.BlockPushErrorHandler(); private final TransportConf conf; private final boolean authEnabled; private final SecretKeyHolder secretKeyHolder; private final long registrationTimeoutMs; - protected volatile TransportClientFactory clientFactory; - protected String appId; - /** * Creates an external shuffle client, with SASL optionally enabled. If SASL is not enabled, * then secretKeyHolder may be null. @@ -72,10 +68,6 @@ public ExternalBlockStoreClient( this.registrationTimeoutMs = registrationTimeoutMs; } - protected void checkInit() { - assert appId != null : "Called before init()"; - } - /** * Initializes the BlockStoreClient, specifying this Executor's appId. * Must be called before any other method on the BlockStoreClient. @@ -103,12 +95,12 @@ public void fetchBlocks( try { int maxRetries = conf.maxIORetries(); RetryingBlockFetcher.BlockFetchStarter blockFetchStarter = - (blockIds1, listener1) -> { + (inputBlockId, inputListener) -> { // Unless this client is closed. if (clientFactory != null) { TransportClient client = clientFactory.createClient(host, port, maxRetries > 0); new OneForOneBlockFetcher(client, appId, execId, - blockIds1, listener1, conf, downloadFileManager).start(); + inputBlockId, inputListener, conf, downloadFileManager).start(); } else { logger.info("This clientFactory was closed. Skipping further block fetch retries."); } @@ -129,6 +121,43 @@ public void fetchBlocks( } } + @Override + public void pushBlocks( + String host, + int port, + String[] blockIds, + ManagedBuffer[] buffers, + BlockFetchingListener listener) { + checkInit(); + assert blockIds.length == buffers.length : "Number of block ids and buffers do not match."; + + Map buffersWithId = new HashMap<>(); + for (int i = 0; i < blockIds.length; i++) { + buffersWithId.put(blockIds[i], buffers[i]); + } + logger.debug("Push {} shuffle blocks to {}:{}", blockIds.length, host, port); + try { + RetryingBlockFetcher.BlockFetchStarter blockPushStarter = + (inputBlockId, inputListener) -> { + TransportClient client = clientFactory.createClient(host, port); + new OneForOneBlockPusher(client, appId, inputBlockId, inputListener, buffersWithId) + .start(); + }; + int maxRetries = conf.maxIORetries(); + if (maxRetries > 0) { + new RetryingBlockFetcher( + conf, blockPushStarter, blockIds, listener, PUSH_ERROR_HANDLER).start(); + } else { + blockPushStarter.createAndStart(blockIds, listener); + } + } catch (Exception e) { + logger.error("Exception while beginning pushBlocks", e); + for (String blockId : blockIds) { + listener.onBlockFetchFailure(blockId, e); + } + } + } + @Override public MetricSet shuffleMetrics() { checkInit(); @@ -175,8 +204,6 @@ public void onSuccess(ByteBuffer response) { logger.warn("Error trying to remove RDD blocks " + Arrays.toString(blockIds) + " via external shuffle service from executor: " + execId, t); numRemovedBlocksFuture.complete(0); - } finally { - client.close(); } } @@ -185,52 +212,11 @@ public void onFailure(Throwable e) { logger.warn("Error trying to remove RDD blocks " + Arrays.toString(blockIds) + " via external shuffle service from executor: " + execId, e); numRemovedBlocksFuture.complete(0); - client.close(); } }); return numRemovedBlocksFuture; } - public void getHostLocalDirs( - String host, - int port, - String[] execIds, - CompletableFuture> hostLocalDirsCompletable) { - checkInit(); - GetLocalDirsForExecutors getLocalDirsMessage = new GetLocalDirsForExecutors(appId, execIds); - try { - TransportClient client = clientFactory.createClient(host, port); - client.sendRpc(getLocalDirsMessage.toByteBuffer(), new RpcResponseCallback() { - @Override - public void onSuccess(ByteBuffer response) { - try { - BlockTransferMessage msgObj = BlockTransferMessage.Decoder.fromByteBuffer(response); - hostLocalDirsCompletable.complete( - ((LocalDirsForExecutors) msgObj).getLocalDirsByExec()); - } catch (Throwable t) { - logger.warn("Error trying to get the host local dirs for " + - Arrays.toString(getLocalDirsMessage.execIds) + " via external shuffle service", - t.getCause()); - hostLocalDirsCompletable.completeExceptionally(t); - } finally { - client.close(); - } - } - - @Override - public void onFailure(Throwable t) { - logger.warn("Error trying to get the host local dirs for " + - Arrays.toString(getLocalDirsMessage.execIds) + " via external shuffle service", - t.getCause()); - hostLocalDirsCompletable.completeExceptionally(t); - client.close(); - } - }); - } catch (IOException | InterruptedException e) { - hostLocalDirsCompletable.completeExceptionally(e); - } - } - @Override public void close() { checkInit(); diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java index a6bcbb8850566..a095bf2723418 100644 --- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolver.java @@ -92,10 +92,6 @@ public class ExternalShuffleBlockResolver { @VisibleForTesting final DB db; - private final List knownManagers = Arrays.asList( - "org.apache.spark.shuffle.sort.SortShuffleManager", - "org.apache.spark.shuffle.unsafe.UnsafeShuffleManager"); - public ExternalShuffleBlockResolver(TransportConf conf, File registeredExecutorFile) throws IOException { this(conf, registeredExecutorFile, Executors.newSingleThreadExecutor( @@ -148,10 +144,6 @@ public void registerExecutor( ExecutorShuffleInfo executorInfo) { AppExecId fullId = new AppExecId(appId, execId); logger.info("Registered executor {} with {}", fullId, executorInfo); - if (!knownManagers.contains(executorInfo.shuffleManager)) { - throw new UnsupportedOperationException( - "Unsupported shuffle manager of executor: " + executorInfo); - } try { if (db != null) { byte[] key = dbAppExecKey(fullId); diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergedBlockMeta.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergedBlockMeta.java new file mode 100644 index 0000000000000..e9d9e53495469 --- /dev/null +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergedBlockMeta.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.network.shuffle; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import com.google.common.base.Preconditions; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import org.roaringbitmap.RoaringBitmap; + +import org.apache.spark.network.buffer.ManagedBuffer; +import org.apache.spark.network.protocol.Encoders; + +/** + * Contains meta information for a merged block. Currently this information constitutes: + * 1. Number of chunks in a merged shuffle block. + * 2. Bitmaps for each chunk in the merged block. A chunk bitmap contains all the mapIds that were + * merged to that merged block chunk. + */ +public class MergedBlockMeta { + private final int numChunks; + private final ManagedBuffer chunksBitmapBuffer; + + public MergedBlockMeta(int numChunks, ManagedBuffer chunksBitmapBuffer) { + this.numChunks = numChunks; + this.chunksBitmapBuffer = Preconditions.checkNotNull(chunksBitmapBuffer); + } + + public int getNumChunks() { + return numChunks; + } + + public ManagedBuffer getChunksBitmapBuffer() { + return chunksBitmapBuffer; + } + + public RoaringBitmap[] readChunkBitmaps() throws IOException { + ByteBuf buf = Unpooled.wrappedBuffer(chunksBitmapBuffer.nioByteBuffer()); + List bitmaps = new ArrayList<>(); + while(buf.isReadable()) { + bitmaps.add(Encoders.Bitmaps.decode(buf)); + } + assert (bitmaps.size() == numChunks); + return bitmaps.toArray(new RoaringBitmap[0]); + } +} diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergedShuffleFileManager.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergedShuffleFileManager.java new file mode 100644 index 0000000000000..ef4dbb2bd0059 --- /dev/null +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/MergedShuffleFileManager.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.network.shuffle; + +import java.io.IOException; + +import org.apache.spark.network.buffer.ManagedBuffer; +import org.apache.spark.network.client.StreamCallbackWithID; +import org.apache.spark.network.shuffle.protocol.FinalizeShuffleMerge; +import org.apache.spark.network.shuffle.protocol.MergeStatuses; +import org.apache.spark.network.shuffle.protocol.PushBlockStream; + + +/** + * The MergedShuffleFileManager is used to process push based shuffle when enabled. It works + * along side {@link ExternalBlockHandler} and serves as an RPCHandler for + * {@link org.apache.spark.network.server.RpcHandler#receiveStream}, where it processes the + * remotely pushed streams of shuffle blocks to merge them into merged shuffle files. Right + * now, support for push based shuffle is only implemented for external shuffle service in + * YARN mode. + */ +public interface MergedShuffleFileManager { + /** + * Provides the stream callback used to process a remotely pushed block. The callback is + * used by the {@link org.apache.spark.network.client.StreamInterceptor} installed on the + * channel to process the block data in the channel outside of the message frame. + * + * @param msg metadata of the remotely pushed blocks. This is processed inside the message frame + * @return A stream callback to process the block data in streaming fashion as it arrives + */ + StreamCallbackWithID receiveBlockDataAsStream(PushBlockStream msg); + + /** + * Handles the request to finalize shuffle merge for a given shuffle. + * + * @param msg contains appId and shuffleId to uniquely identify a shuffle to be finalized + * @return The statuses of the merged shuffle partitions for the given shuffle on this + * shuffle service + * @throws IOException + */ + MergeStatuses finalizeShuffleMerge(FinalizeShuffleMerge msg) throws IOException; + + /** + * Registers an application when it starts. It also stores the username which is necessary + * for generating the host local directories for merged shuffle files. + * Right now, this is invoked by YarnShuffleService. + * + * @param appId application ID + * @param user username + */ + void registerApplication(String appId, String user); + + /** + * Registers an executor with its local dir list when it starts. This provides the specific path + * so MergedShuffleFileManager knows where to store and look for shuffle data for a + * given application. It is invoked by the RPC call when executor tries to register with the + * local shuffle service. + * + * @param appId application ID + * @param localDirs The list of local dirs that this executor gets granted from NodeManager + */ + void registerExecutor(String appId, String[] localDirs); + + /** + * Invoked when an application finishes. This cleans up any remaining metadata associated with + * this application, and optionally deletes the application specific directory path. + * + * @param appId application ID + * @param cleanupLocalDirs flag indicating whether MergedShuffleFileManager should handle + * deletion of local dirs itself. + */ + void applicationRemoved(String appId, boolean cleanupLocalDirs); + + /** + * Get the buffer for a given merged shuffle chunk when serving merged shuffle to reducers + * + * @param appId application ID + * @param shuffleId shuffle ID + * @param reduceId reducer ID + * @param chunkId merged shuffle file chunk ID + * @return The {@link ManagedBuffer} for the given merged shuffle chunk + */ + ManagedBuffer getMergedBlockData(String appId, int shuffleId, int reduceId, int chunkId); + + /** + * Get the meta information of a merged block. + * + * @param appId application ID + * @param shuffleId shuffle ID + * @param reduceId reducer ID + * @return meta information of a merged block + */ + MergedBlockMeta getMergedBlockMeta(String appId, int shuffleId, int reduceId); + + /** + * Get the local directories which stores the merged shuffle files. + * + * @param appId application ID + */ + String[] getMergedBlockDirs(String appId); +} diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/OneForOneBlockPusher.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/OneForOneBlockPusher.java new file mode 100644 index 0000000000000..407b248170a46 --- /dev/null +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/OneForOneBlockPusher.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.network.shuffle; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.spark.network.buffer.ManagedBuffer; +import org.apache.spark.network.buffer.NioManagedBuffer; +import org.apache.spark.network.client.RpcResponseCallback; +import org.apache.spark.network.client.TransportClient; +import org.apache.spark.network.shuffle.protocol.PushBlockStream; + +/** + * Similar to {@link OneForOneBlockFetcher}, but for pushing blocks to remote shuffle service to + * be merged instead of for fetching them from remote shuffle services. This is used by + * ShuffleWriter when the block push process is initiated. The supplied BlockFetchingListener + * is used to handle the success or failure in pushing each blocks. + */ +public class OneForOneBlockPusher { + private static final Logger logger = LoggerFactory.getLogger(OneForOneBlockPusher.class); + private static final ErrorHandler PUSH_ERROR_HANDLER = new ErrorHandler.BlockPushErrorHandler(); + + private final TransportClient client; + private final String appId; + private final String[] blockIds; + private final BlockFetchingListener listener; + private final Map buffers; + + public OneForOneBlockPusher( + TransportClient client, + String appId, + String[] blockIds, + BlockFetchingListener listener, + Map buffers) { + this.client = client; + this.appId = appId; + this.blockIds = blockIds; + this.listener = listener; + this.buffers = buffers; + } + + private class BlockPushCallback implements RpcResponseCallback { + + private int index; + private String blockId; + + BlockPushCallback(int index, String blockId) { + this.index = index; + this.blockId = blockId; + } + + @Override + public void onSuccess(ByteBuffer response) { + // On receipt of a successful block push + listener.onBlockFetchSuccess(blockId, new NioManagedBuffer(ByteBuffer.allocate(0))); + } + + @Override + public void onFailure(Throwable e) { + // Since block push is best effort, i.e., if we encountered a block push failure that's not + // retriable or exceeding the max retires, we should not fail all remaining block pushes. + // The best effort nature makes block push tolerable of a partial completion. Thus, we only + // fail the block that's actually failed. Not that, on the RetryingBlockFetcher side, once + // retry is initiated, it would still invalidate the previous active retry listener, and + // retry all outstanding blocks. We are preventing forwarding unnecessary block push failures + // to the parent listener of the retry listener. The only exceptions would be if the block + // push failure is due to block arriving on the server side after merge finalization, or the + // client fails to establish connection to the server side. In both cases, we would fail all + // remaining blocks. + if (PUSH_ERROR_HANDLER.shouldRetryError(e)) { + String[] targetBlockId = Arrays.copyOfRange(blockIds, index, index + 1); + failRemainingBlocks(targetBlockId, e); + } else { + String[] targetBlockId = Arrays.copyOfRange(blockIds, index, blockIds.length); + failRemainingBlocks(targetBlockId, e); + } + } + } + + private void failRemainingBlocks(String[] failedBlockIds, Throwable e) { + for (String blockId : failedBlockIds) { + try { + listener.onBlockFetchFailure(blockId, e); + } catch (Exception e2) { + logger.error("Error in block push failure callback", e2); + } + } + } + + /** + * Begins the block pushing process, calling the listener with every block pushed. + */ + public void start() { + logger.debug("Start pushing {} blocks", blockIds.length); + for (int i = 0; i < blockIds.length; i++) { + assert buffers.containsKey(blockIds[i]) : "Could not find the block buffer for block " + + blockIds[i]; + ByteBuffer header = new PushBlockStream(appId, blockIds[i], i).toByteBuffer(); + client.uploadStream(new NioManagedBuffer(header), buffers.get(blockIds[i]), + new BlockPushCallback(i, blockIds[i])); + } + } +} diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RetryingBlockFetcher.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RetryingBlockFetcher.java index 6bf3da94030d4..43bde1610e41e 100644 --- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RetryingBlockFetcher.java +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RetryingBlockFetcher.java @@ -99,11 +99,14 @@ void createAndStart(String[] blockIds, BlockFetchingListener listener) */ private RetryingBlockFetchListener currentListener; + private final ErrorHandler errorHandler; + public RetryingBlockFetcher( TransportConf conf, RetryingBlockFetcher.BlockFetchStarter fetchStarter, String[] blockIds, - BlockFetchingListener listener) { + BlockFetchingListener listener, + ErrorHandler errorHandler) { this.fetchStarter = fetchStarter; this.listener = listener; this.maxRetries = conf.maxIORetries(); @@ -111,6 +114,15 @@ public RetryingBlockFetcher( this.outstandingBlocksIds = Sets.newLinkedHashSet(); Collections.addAll(outstandingBlocksIds, blockIds); this.currentListener = new RetryingBlockFetchListener(); + this.errorHandler = errorHandler; + } + + public RetryingBlockFetcher( + TransportConf conf, + BlockFetchStarter fetchStarter, + String[] blockIds, + BlockFetchingListener listener) { + this(conf, fetchStarter, blockIds, listener, ErrorHandler.NOOP_ERROR_HANDLER); } /** @@ -178,7 +190,7 @@ private synchronized boolean shouldRetry(Throwable e) { boolean isIOException = e instanceof IOException || (e.getCause() != null && e.getCause() instanceof IOException); boolean hasRemainingRetries = retryCount < maxRetries; - return isIOException && hasRemainingRetries; + return isIOException && hasRemainingRetries && errorHandler.shouldRetryError(e); } /** @@ -215,8 +227,15 @@ public void onBlockFetchFailure(String blockId, Throwable exception) { if (shouldRetry(exception)) { initiateRetry(); } else { - logger.error(String.format("Failed to fetch block %s, and will not retry (%s retries)", - blockId, retryCount), exception); + if (errorHandler.shouldLogError(exception)) { + logger.error( + String.format("Failed to fetch block %s, and will not retry (%s retries)", + blockId, retryCount), exception); + } else { + logger.debug( + String.format("Failed to fetch block %s, and will not retry (%s retries)", + blockId, retryCount), exception); + } outstandingBlocksIds.remove(blockId); shouldForwardFailure = true; } diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/BlockTransferMessage.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/BlockTransferMessage.java index 89d8dfe8716b8..7f5058124988f 100644 --- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/BlockTransferMessage.java +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/BlockTransferMessage.java @@ -47,7 +47,8 @@ public abstract class BlockTransferMessage implements Encodable { public enum Type { OPEN_BLOCKS(0), UPLOAD_BLOCK(1), REGISTER_EXECUTOR(2), STREAM_HANDLE(3), REGISTER_DRIVER(4), HEARTBEAT(5), UPLOAD_BLOCK_STREAM(6), REMOVE_BLOCKS(7), BLOCKS_REMOVED(8), - FETCH_SHUFFLE_BLOCKS(9), GET_LOCAL_DIRS_FOR_EXECUTORS(10), LOCAL_DIRS_FOR_EXECUTORS(11); + FETCH_SHUFFLE_BLOCKS(9), GET_LOCAL_DIRS_FOR_EXECUTORS(10), LOCAL_DIRS_FOR_EXECUTORS(11), + PUSH_BLOCK_STREAM(12), FINALIZE_SHUFFLE_MERGE(13), MERGE_STATUSES(14); private final byte id; @@ -78,6 +79,9 @@ public static BlockTransferMessage fromByteBuffer(ByteBuffer msg) { case 9: return FetchShuffleBlocks.decode(buf); case 10: return GetLocalDirsForExecutors.decode(buf); case 11: return LocalDirsForExecutors.decode(buf); + case 12: return PushBlockStream.decode(buf); + case 13: return FinalizeShuffleMerge.decode(buf); + case 14: return MergeStatuses.decode(buf); default: throw new IllegalArgumentException("Unknown message type: " + type); } } diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/FinalizeShuffleMerge.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/FinalizeShuffleMerge.java new file mode 100644 index 0000000000000..9058575df57ef --- /dev/null +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/FinalizeShuffleMerge.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.network.shuffle.protocol; + +import com.google.common.base.Objects; +import io.netty.buffer.ByteBuf; + +import org.apache.spark.network.protocol.Encoders; + +/** + * Request to finalize merge for a given shuffle. + * Returns {@link MergeStatuses} + */ +public class FinalizeShuffleMerge extends BlockTransferMessage { + public final String appId; + public final int shuffleId; + + public FinalizeShuffleMerge( + String appId, + int shuffleId) { + this.appId = appId; + this.shuffleId = shuffleId; + } + + @Override + protected BlockTransferMessage.Type type() { + return Type.FINALIZE_SHUFFLE_MERGE; + } + + @Override + public int hashCode() { + return Objects.hashCode(appId, shuffleId); + } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("appId", appId) + .add("shuffleId", shuffleId) + .toString(); + } + + @Override + public boolean equals(Object other) { + if (other != null && other instanceof FinalizeShuffleMerge) { + FinalizeShuffleMerge o = (FinalizeShuffleMerge) other; + return Objects.equal(appId, o.appId) + && shuffleId == o.shuffleId; + } + return false; + } + + @Override + public int encodedLength() { + return Encoders.Strings.encodedLength(appId) + 4; + } + + @Override + public void encode(ByteBuf buf) { + Encoders.Strings.encode(buf, appId); + buf.writeInt(shuffleId); + } + + public static FinalizeShuffleMerge decode(ByteBuf buf) { + String appId = Encoders.Strings.decode(buf); + int shuffleId = buf.readInt(); + return new FinalizeShuffleMerge(appId, shuffleId); + } +} diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/MergeStatuses.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/MergeStatuses.java new file mode 100644 index 0000000000000..f57e8b326e5e2 --- /dev/null +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/MergeStatuses.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.network.shuffle.protocol; + +import java.util.Arrays; + +import com.google.common.base.Objects; +import io.netty.buffer.ByteBuf; +import org.roaringbitmap.RoaringBitmap; + +import org.apache.spark.network.protocol.Encoders; + +/** + * Result returned by an ExternalShuffleService to the DAGScheduler. This represents the result + * of all the remote shuffle block merge operations performed by an ExternalShuffleService + * for a given shuffle ID. It includes the shuffle ID, an array of bitmaps each representing + * the set of mapper partition blocks that are merged for a given reducer partition, an array + * of reducer IDs, and an array of merged shuffle partition sizes. The 3 arrays list information + * about all the reducer partitions merged by the ExternalShuffleService in the same order. + */ +public class MergeStatuses extends BlockTransferMessage { + /** Shuffle ID **/ + public final int shuffleId; + /** + * Array of bitmaps tracking the set of mapper partition blocks merged for each + * reducer partition + */ + public final RoaringBitmap[] bitmaps; + /** Array of reducer IDs **/ + public final int[] reduceIds; + /** + * Array of merged shuffle partition block size. Each represents the total size of all + * merged shuffle partition blocks for one reducer partition. + * **/ + public final long[] sizes; + + public MergeStatuses( + int shuffleId, + RoaringBitmap[] bitmaps, + int[] reduceIds, + long[] sizes) { + this.shuffleId = shuffleId; + this.bitmaps = bitmaps; + this.reduceIds = reduceIds; + this.sizes = sizes; + } + + @Override + protected Type type() { + return Type.MERGE_STATUSES; + } + + @Override + public int hashCode() { + int objectHashCode = Objects.hashCode(shuffleId); + return (objectHashCode * 41 + Arrays.hashCode(reduceIds) * 41 + + Arrays.hashCode(bitmaps) * 41 + Arrays.hashCode(sizes)); + } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("shuffleId", shuffleId) + .add("reduceId size", reduceIds.length) + .toString(); + } + + @Override + public boolean equals(Object other) { + if (other != null && other instanceof MergeStatuses) { + MergeStatuses o = (MergeStatuses) other; + return Objects.equal(shuffleId, o.shuffleId) + && Arrays.equals(bitmaps, o.bitmaps) + && Arrays.equals(reduceIds, o.reduceIds) + && Arrays.equals(sizes, o.sizes); + } + return false; + } + + @Override + public int encodedLength() { + return 4 // int + + Encoders.BitmapArrays.encodedLength(bitmaps) + + Encoders.IntArrays.encodedLength(reduceIds) + + Encoders.LongArrays.encodedLength(sizes); + } + + @Override + public void encode(ByteBuf buf) { + buf.writeInt(shuffleId); + Encoders.BitmapArrays.encode(buf, bitmaps); + Encoders.IntArrays.encode(buf, reduceIds); + Encoders.LongArrays.encode(buf, sizes); + } + + public static MergeStatuses decode(ByteBuf buf) { + int shuffleId = buf.readInt(); + RoaringBitmap[] bitmaps = Encoders.BitmapArrays.decode(buf); + int[] reduceIds = Encoders.IntArrays.decode(buf); + long[] sizes = Encoders.LongArrays.decode(buf); + return new MergeStatuses(shuffleId, bitmaps, reduceIds, sizes); + } +} diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/PushBlockStream.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/PushBlockStream.java new file mode 100644 index 0000000000000..7eab5a644783c --- /dev/null +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/PushBlockStream.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.network.shuffle.protocol; + +import com.google.common.base.Objects; +import io.netty.buffer.ByteBuf; + +import org.apache.spark.network.protocol.Encoders; + +// Needed by ScalaDoc. See SPARK-7726 +import static org.apache.spark.network.shuffle.protocol.BlockTransferMessage.Type; + + +/** + * Request to push a block to a remote shuffle service to be merged in push based shuffle. + * The remote shuffle service will also include this message when responding the push requests. + */ +public class PushBlockStream extends BlockTransferMessage { + public final String appId; + public final String blockId; + // Similar to the chunkIndex in StreamChunkId, indicating the index of a block in a batch of + // blocks to be pushed. + public final int index; + + public PushBlockStream(String appId, String blockId, int index) { + this.appId = appId; + this.blockId = blockId; + this.index = index; + } + + @Override + protected Type type() { + return Type.PUSH_BLOCK_STREAM; + } + + @Override + public int hashCode() { + return Objects.hashCode(appId, blockId, index); + } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("appId", appId) + .add("blockId", blockId) + .add("index", index) + .toString(); + } + + @Override + public boolean equals(Object other) { + if (other != null && other instanceof PushBlockStream) { + PushBlockStream o = (PushBlockStream) other; + return Objects.equal(appId, o.appId) + && Objects.equal(blockId, o.blockId) + && index == o.index; + } + return false; + } + + @Override + public int encodedLength() { + return Encoders.Strings.encodedLength(appId) + + Encoders.Strings.encodedLength(blockId) + 4; + } + + @Override + public void encode(ByteBuf buf) { + Encoders.Strings.encode(buf, appId); + Encoders.Strings.encode(buf, blockId); + buf.writeInt(index); + } + + public static PushBlockStream decode(ByteBuf buf) { + String appId = Encoders.Strings.decode(buf); + String blockId = Encoders.Strings.decode(buf); + int index = buf.readInt(); + return new PushBlockStream(appId, blockId, index); + } +} diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ErrorHandlerSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ErrorHandlerSuite.java new file mode 100644 index 0000000000000..992e7762c5a54 --- /dev/null +++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ErrorHandlerSuite.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.network.shuffle; + +import java.net.ConnectException; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Test suite for {@link ErrorHandler} + */ +public class ErrorHandlerSuite { + + @Test + public void testPushErrorRetry() { + ErrorHandler.BlockPushErrorHandler handler = new ErrorHandler.BlockPushErrorHandler(); + assertFalse(handler.shouldRetryError(new RuntimeException(new IllegalArgumentException( + ErrorHandler.BlockPushErrorHandler.TOO_LATE_MESSAGE_SUFFIX)))); + assertFalse(handler.shouldRetryError(new RuntimeException(new ConnectException()))); + assertTrue(handler.shouldRetryError(new RuntimeException(new IllegalArgumentException( + ErrorHandler.BlockPushErrorHandler.BLOCK_APPEND_COLLISION_DETECTED_MSG_PREFIX)))); + assertTrue(handler.shouldRetryError(new Throwable())); + } + + @Test + public void testPushErrorLogging() { + ErrorHandler.BlockPushErrorHandler handler = new ErrorHandler.BlockPushErrorHandler(); + assertFalse(handler.shouldLogError(new RuntimeException(new IllegalArgumentException( + ErrorHandler.BlockPushErrorHandler.TOO_LATE_MESSAGE_SUFFIX)))); + assertFalse(handler.shouldLogError(new RuntimeException(new IllegalArgumentException( + ErrorHandler.BlockPushErrorHandler.BLOCK_APPEND_COLLISION_DETECTED_MSG_PREFIX)))); + assertTrue(handler.shouldLogError(new Throwable())); + } +} diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalBlockHandlerSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalBlockHandlerSuite.java index 455351fcf767c..680b8d74a2eea 100644 --- a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalBlockHandlerSuite.java +++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalBlockHandlerSuite.java @@ -17,6 +17,7 @@ package org.apache.spark.network.shuffle; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.Iterator; @@ -25,6 +26,7 @@ import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; +import org.roaringbitmap.RoaringBitmap; import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.any; @@ -39,6 +41,8 @@ import org.apache.spark.network.shuffle.protocol.BlockTransferMessage; import org.apache.spark.network.shuffle.protocol.ExecutorShuffleInfo; import org.apache.spark.network.shuffle.protocol.FetchShuffleBlocks; +import org.apache.spark.network.shuffle.protocol.FinalizeShuffleMerge; +import org.apache.spark.network.shuffle.protocol.MergeStatuses; import org.apache.spark.network.shuffle.protocol.OpenBlocks; import org.apache.spark.network.shuffle.protocol.RegisterExecutor; import org.apache.spark.network.shuffle.protocol.StreamHandle; @@ -50,6 +54,7 @@ public class ExternalBlockHandlerSuite { OneForOneStreamManager streamManager; ExternalShuffleBlockResolver blockResolver; RpcHandler handler; + MergedShuffleFileManager mergedShuffleManager; ManagedBuffer[] blockMarkers = { new NioManagedBuffer(ByteBuffer.wrap(new byte[3])), new NioManagedBuffer(ByteBuffer.wrap(new byte[7])) @@ -59,17 +64,20 @@ public class ExternalBlockHandlerSuite { public void beforeEach() { streamManager = mock(OneForOneStreamManager.class); blockResolver = mock(ExternalShuffleBlockResolver.class); - handler = new ExternalBlockHandler(streamManager, blockResolver); + mergedShuffleManager = mock(MergedShuffleFileManager.class); + handler = new ExternalBlockHandler(streamManager, blockResolver, mergedShuffleManager); } @Test public void testRegisterExecutor() { RpcResponseCallback callback = mock(RpcResponseCallback.class); - ExecutorShuffleInfo config = new ExecutorShuffleInfo(new String[] {"/a", "/b"}, 16, "sort"); + String[] localDirs = new String[] {"/a", "/b"}; + ExecutorShuffleInfo config = new ExecutorShuffleInfo(localDirs, 16, "sort"); ByteBuffer registerMessage = new RegisterExecutor("app0", "exec1", config).toByteBuffer(); handler.receive(client, registerMessage, callback); verify(blockResolver, times(1)).registerExecutor("app0", "exec1", config); + verify(mergedShuffleManager, times(1)).registerExecutor("app0", localDirs); verify(callback, times(1)).onSuccess(any(ByteBuffer.class)); verify(callback, never()).onFailure(any(Throwable.class)); @@ -222,4 +230,32 @@ public void testBadMessages() { verify(callback, never()).onSuccess(any(ByteBuffer.class)); verify(callback, never()).onFailure(any(Throwable.class)); } + + @Test + public void testFinalizeShuffleMerge() throws IOException { + RpcResponseCallback callback = mock(RpcResponseCallback.class); + + FinalizeShuffleMerge req = new FinalizeShuffleMerge("app0", 0); + RoaringBitmap bitmap = RoaringBitmap.bitmapOf(0, 1, 2); + MergeStatuses statuses = new MergeStatuses(0, new RoaringBitmap[]{bitmap}, + new int[]{3}, new long[]{30}); + when(mergedShuffleManager.finalizeShuffleMerge(req)).thenReturn(statuses); + + ByteBuffer reqBuf = req.toByteBuffer(); + handler.receive(client, reqBuf, callback); + verify(mergedShuffleManager, times(1)).finalizeShuffleMerge(req); + ArgumentCaptor response = ArgumentCaptor.forClass(ByteBuffer.class); + verify(callback, times(1)).onSuccess(response.capture()); + verify(callback, never()).onFailure(any()); + + MergeStatuses mergeStatuses = + (MergeStatuses) BlockTransferMessage.Decoder.fromByteBuffer(response.getValue()); + assertEquals(mergeStatuses, statuses); + + Timer finalizeShuffleMergeLatencyMillis = (Timer) ((ExternalBlockHandler) handler) + .getAllMetrics() + .getMetrics() + .get("finalizeShuffleMergeLatencyMillis"); + assertEquals(1, finalizeShuffleMergeLatencyMillis.getCount()); + } } diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java index 88bcf43c2371f..04d4bdf92bae7 100644 --- a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java +++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java @@ -71,15 +71,6 @@ public void testBadRequests() throws IOException { assertTrue("Bad error message: " + e, e.getMessage().contains("not registered")); } - // Invalid shuffle manager - try { - resolver.registerExecutor("app0", "exec2", dataContext.createExecutorInfo("foobar")); - resolver.getBlockData("app0", "exec2", 1, 1, 0); - fail("Should have failed"); - } catch (UnsupportedOperationException e) { - // pass - } - // Nonexistent shuffle block resolver.registerExecutor("app0", "exec3", dataContext.createExecutorInfo(SORT_MANAGER)); diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleIntegrationSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleIntegrationSuite.java index 9d398e372056b..49d02e5dc6fb4 100644 --- a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleIntegrationSuite.java +++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleIntegrationSuite.java @@ -233,9 +233,9 @@ public void testFetchThreeSort() throws Exception { exec0Fetch.releaseBuffers(); } - @Test (expected = RuntimeException.class) - public void testRegisterInvalidExecutor() throws Exception { - registerExecutor("exec-1", dataContext0.createExecutorInfo("unknown sort manager")); + @Test + public void testRegisterWithCustomShuffleManager() throws Exception { + registerExecutor("exec-1", dataContext0.createExecutorInfo("custom shuffle manager")); } @Test diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/OneForOneBlockPusherSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/OneForOneBlockPusherSuite.java new file mode 100644 index 0000000000000..ebcdba72aa1a8 --- /dev/null +++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/OneForOneBlockPusherSuite.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.network.shuffle; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; + +import com.google.common.collect.Maps; +import io.netty.buffer.Unpooled; +import org.junit.Test; + +import static org.junit.Assert.*; +import static org.mockito.AdditionalMatchers.*; +import static org.mockito.Mockito.*; + +import org.apache.spark.network.buffer.ManagedBuffer; +import org.apache.spark.network.buffer.NettyManagedBuffer; +import org.apache.spark.network.buffer.NioManagedBuffer; +import org.apache.spark.network.client.RpcResponseCallback; +import org.apache.spark.network.client.TransportClient; +import org.apache.spark.network.shuffle.protocol.BlockTransferMessage; +import org.apache.spark.network.shuffle.protocol.PushBlockStream; + + +public class OneForOneBlockPusherSuite { + + @Test + public void testPushOne() { + LinkedHashMap blocks = Maps.newLinkedHashMap(); + blocks.put("shuffle_0_0_0", new NioManagedBuffer(ByteBuffer.wrap(new byte[1]))); + String[] blockIds = blocks.keySet().toArray(new String[blocks.size()]); + + BlockFetchingListener listener = pushBlocks( + blocks, + blockIds, + Arrays.asList(new PushBlockStream("app-id", "shuffle_0_0_0", 0))); + + verify(listener).onBlockFetchSuccess(eq("shuffle_0_0_0"), any()); + } + + @Test + public void testPushThree() { + LinkedHashMap blocks = Maps.newLinkedHashMap(); + blocks.put("b0", new NioManagedBuffer(ByteBuffer.wrap(new byte[12]))); + blocks.put("b1", new NioManagedBuffer(ByteBuffer.wrap(new byte[23]))); + blocks.put("b2", new NettyManagedBuffer(Unpooled.wrappedBuffer(new byte[23]))); + String[] blockIds = blocks.keySet().toArray(new String[blocks.size()]); + + BlockFetchingListener listener = pushBlocks( + blocks, + blockIds, + Arrays.asList(new PushBlockStream("app-id", "b0", 0), + new PushBlockStream("app-id", "b1", 1), + new PushBlockStream("app-id", "b2", 2))); + + for (int i = 0; i < 3; i ++) { + verify(listener, times(1)).onBlockFetchSuccess(eq("b" + i), any()); + } + } + + @Test + public void testServerFailures() { + LinkedHashMap blocks = Maps.newLinkedHashMap(); + blocks.put("b0", new NioManagedBuffer(ByteBuffer.wrap(new byte[12]))); + blocks.put("b1", new NioManagedBuffer(ByteBuffer.wrap(new byte[0]))); + blocks.put("b2", new NioManagedBuffer(ByteBuffer.wrap(new byte[0]))); + String[] blockIds = blocks.keySet().toArray(new String[blocks.size()]); + + BlockFetchingListener listener = pushBlocks( + blocks, + blockIds, + Arrays.asList(new PushBlockStream("app-id", "b0", 0), + new PushBlockStream("app-id", "b1", 1), + new PushBlockStream("app-id", "b2", 2))); + + verify(listener, times(1)).onBlockFetchSuccess(eq("b0"), any()); + verify(listener, times(1)).onBlockFetchFailure(eq("b1"), any()); + verify(listener, times(1)).onBlockFetchFailure(eq("b2"), any()); + } + + @Test + public void testHandlingRetriableFailures() { + LinkedHashMap blocks = Maps.newLinkedHashMap(); + blocks.put("b0", new NioManagedBuffer(ByteBuffer.wrap(new byte[12]))); + blocks.put("b1", null); + blocks.put("b2", new NioManagedBuffer(ByteBuffer.wrap(new byte[0]))); + String[] blockIds = blocks.keySet().toArray(new String[blocks.size()]); + + BlockFetchingListener listener = pushBlocks( + blocks, + blockIds, + Arrays.asList(new PushBlockStream("app-id", "b0", 0), + new PushBlockStream("app-id", "b1", 1), + new PushBlockStream("app-id", "b2", 2))); + + verify(listener, times(1)).onBlockFetchSuccess(eq("b0"), any()); + verify(listener, times(0)).onBlockFetchSuccess(not(eq("b0")), any()); + verify(listener, times(0)).onBlockFetchFailure(eq("b0"), any()); + verify(listener, times(1)).onBlockFetchFailure(eq("b1"), any()); + verify(listener, times(2)).onBlockFetchFailure(eq("b2"), any()); + } + + /** + * Begins a push on the given set of blocks by mocking the response from server side. + * If a block is an empty byte, a server side retriable exception will be thrown. + * If a block is null, a non-retriable exception will be thrown. + */ + private static BlockFetchingListener pushBlocks( + LinkedHashMap blocks, + String[] blockIds, + Iterable expectMessages) { + TransportClient client = mock(TransportClient.class); + BlockFetchingListener listener = mock(BlockFetchingListener.class); + OneForOneBlockPusher pusher = + new OneForOneBlockPusher(client, "app-id", blockIds, listener, blocks); + + Iterator> blockIterator = blocks.entrySet().iterator(); + Iterator msgIterator = expectMessages.iterator(); + doAnswer(invocation -> { + ByteBuffer header = ((ManagedBuffer) invocation.getArguments()[0]).nioByteBuffer(); + BlockTransferMessage message = BlockTransferMessage.Decoder.fromByteBuffer(header); + RpcResponseCallback callback = (RpcResponseCallback) invocation.getArguments()[2]; + Map.Entry entry = blockIterator.next(); + ManagedBuffer block = entry.getValue(); + if (block != null && block.nioByteBuffer().capacity() > 0) { + callback.onSuccess(header); + } else if (block != null) { + callback.onFailure(new RuntimeException("Failed " + entry.getKey() + + ErrorHandler.BlockPushErrorHandler.BLOCK_APPEND_COLLISION_DETECTED_MSG_PREFIX)); + } else { + callback.onFailure(new RuntimeException("Quick fail " + entry.getKey() + + ErrorHandler.BlockPushErrorHandler.TOO_LATE_MESSAGE_SUFFIX)); + } + assertEquals(msgIterator.next(), message); + return null; + }).when(client).uploadStream(any(ManagedBuffer.class), any(), any(RpcResponseCallback.class)); + + pusher.start(); + return listener; + } +} diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index 0225db81925c5..9938e5d769e12 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -65,7 +65,13 @@ org.apache.hadoop - hadoop-client + ${hadoop-client-api.artifact} + ${hadoop.version} + + + org.apache.hadoop + ${hadoop-client-runtime.artifact} + ${hadoop.version} org.slf4j diff --git a/common/sketch/src/main/java/org/apache/spark/util/sketch/Murmur3_x86_32.java b/common/sketch/src/main/java/org/apache/spark/util/sketch/Murmur3_x86_32.java index e83b331391e39..61cd2cec1a34b 100644 --- a/common/sketch/src/main/java/org/apache/spark/util/sketch/Murmur3_x86_32.java +++ b/common/sketch/src/main/java/org/apache/spark/util/sketch/Murmur3_x86_32.java @@ -17,12 +17,16 @@ package org.apache.spark.util.sketch; +import java.nio.ByteOrder; + /** * 32-bit Murmur3 hasher. This is based on Guava's Murmur3_32HashFunction. */ // This class is duplicated from `org.apache.spark.unsafe.hash.Murmur3_x86_32` to make sure // spark-sketch has no external dependencies. final class Murmur3_x86_32 { + private static final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); + private static final int C1 = 0xcc9e2d51; private static final int C2 = 0x1b873593; @@ -92,8 +96,10 @@ private static int hashBytesByInt(Object base, long offset, int lengthInBytes, i int h1 = seed; for (int i = 0; i < lengthInBytes; i += 4) { int halfWord = Platform.getInt(base, offset + i); - int k1 = mixK1(halfWord); - h1 = mixH1(h1, k1); + if (isBigEndian) { + halfWord = Integer.reverseBytes(halfWord); + } + h1 = mixH1(h1, mixK1(halfWord)); } return h1; } diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/hash/Murmur3_x86_32.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/hash/Murmur3_x86_32.java index d239de6083ad0..0b9d9ced312a1 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/hash/Murmur3_x86_32.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/hash/Murmur3_x86_32.java @@ -17,12 +17,16 @@ package org.apache.spark.unsafe.hash; +import java.nio.ByteOrder; + import org.apache.spark.unsafe.Platform; /** * 32-bit Murmur3 hasher. This is based on Guava's Murmur3_32HashFunction. */ public final class Murmur3_x86_32 { + private static final boolean isBigEndian = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); + private static final int C1 = 0xcc9e2d51; private static final int C2 = 0x1b873593; @@ -92,8 +96,10 @@ private static int hashBytesByInt(Object base, long offset, int lengthInBytes, i int h1 = seed; for (int i = 0; i < lengthInBytes; i += 4) { int halfWord = Platform.getInt(base, offset + i); - int k1 = mixK1(halfWord); - h1 = mixH1(h1, k1); + if (isBigEndian) { + halfWord = Integer.reverseBytes(halfWord); + } + h1 = mixH1(h1, mixK1(halfWord)); } return h1; } diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java index 7205293aa48c5..b8dda22240042 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java @@ -563,7 +563,7 @@ public UTF8String trim() { } /** - * Trims whitespaces (<= ASCII 32) from both ends of this string. + * Trims whitespaces ({@literal <=} ASCII 32) from both ends of this string. * * Note that, this method is the same as java's {@link String#trim}, and different from * {@link UTF8String#trim()} which remove only spaces(= ASCII 32) from both ends. @@ -575,14 +575,14 @@ public UTF8String trim() { public UTF8String trimAll() { int s = 0; // skip all of the whitespaces (<=0x20) in the left side - while (s < this.numBytes && getByte(s) <= ' ') s++; + while (s < this.numBytes && Character.isWhitespace(getByte(s))) s++; if (s == this.numBytes) { // Everything trimmed return EMPTY_UTF8; } // skip all of the whitespaces (<=0x20) in the right side int e = this.numBytes - 1; - while (e > s && getByte(e) <= ' ') e--; + while (e > s && Character.isWhitespace(getByte(e))) e--; if (s == 0 && e == numBytes - 1) { // Nothing trimmed return this; @@ -1119,11 +1119,11 @@ public boolean toLong(LongWrapper toLongResult) { private boolean toLong(LongWrapper toLongResult, boolean allowDecimal) { int offset = 0; - while (offset < this.numBytes && getByte(offset) <= ' ') offset++; + while (offset < this.numBytes && Character.isWhitespace(getByte(offset))) offset++; if (offset == this.numBytes) return false; int end = this.numBytes - 1; - while (end > offset && getByte(end) <= ' ') end--; + while (end > offset && Character.isWhitespace(getByte(end))) end--; byte b = getByte(offset); final boolean negative = b == '-'; @@ -1216,11 +1216,11 @@ public boolean toInt(IntWrapper intWrapper) { private boolean toInt(IntWrapper intWrapper, boolean allowDecimal) { int offset = 0; - while (offset < this.numBytes && getByte(offset) <= ' ') offset++; + while (offset < this.numBytes && Character.isWhitespace(getByte(offset))) offset++; if (offset == this.numBytes) return false; int end = this.numBytes - 1; - while (end > offset && getByte(end) <= ' ') end--; + while (end > offset && Character.isWhitespace(getByte(end))) end--; byte b = getByte(offset); final boolean negative = b == '-'; diff --git a/core/pom.xml b/core/pom.xml index 14b217d7fb22e..7a56c4ca3c638 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -66,7 +66,13 @@ org.apache.hadoop - hadoop-client + ${hadoop-client-api.artifact} + ${hadoop.version} + + + org.apache.hadoop + ${hadoop-client-runtime.artifact} + ${hadoop.version} org.apache.spark @@ -177,6 +183,14 @@ org.apache.commons commons-text + + commons-io + commons-io + + + commons-collections + commons-collections + com.google.code.findbugs jsr305 diff --git a/core/src/main/java/org/apache/spark/api/java/StorageLevels.java b/core/src/main/java/org/apache/spark/api/java/StorageLevels.java index 3fcb52f615834..b51cde48e632b 100644 --- a/core/src/main/java/org/apache/spark/api/java/StorageLevels.java +++ b/core/src/main/java/org/apache/spark/api/java/StorageLevels.java @@ -26,6 +26,7 @@ public class StorageLevels { public static final StorageLevel NONE = create(false, false, false, false, 1); public static final StorageLevel DISK_ONLY = create(true, false, false, false, 1); public static final StorageLevel DISK_ONLY_2 = create(true, false, false, false, 2); + public static final StorageLevel DISK_ONLY_3 = create(true, false, false, false, 3); public static final StorageLevel MEMORY_ONLY = create(false, true, false, true, 1); public static final StorageLevel MEMORY_ONLY_2 = create(false, true, false, true, 2); public static final StorageLevel MEMORY_ONLY_SER = create(false, true, false, false, 1); diff --git a/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java b/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java index 0c0d0df8ae682..1d676ff781c70 100644 --- a/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java +++ b/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java @@ -41,7 +41,7 @@ public interface DriverPlugin { * initialization. *

* It's recommended that plugins be careful about what operations are performed in this call, - * preferrably performing expensive operations in a separate thread, or postponing them until + * preferably performing expensive operations in a separate thread, or postponing them until * the application has fully started. * * @param sc The SparkContext loading the plugin. diff --git a/core/src/main/java/org/apache/spark/api/plugin/ExecutorPlugin.java b/core/src/main/java/org/apache/spark/api/plugin/ExecutorPlugin.java index 4961308035163..481bf985f1c6c 100644 --- a/core/src/main/java/org/apache/spark/api/plugin/ExecutorPlugin.java +++ b/core/src/main/java/org/apache/spark/api/plugin/ExecutorPlugin.java @@ -19,6 +19,7 @@ import java.util.Map; +import org.apache.spark.TaskFailedReason; import org.apache.spark.annotation.DeveloperApi; /** @@ -54,4 +55,45 @@ default void init(PluginContext ctx, Map extraConf) {} */ default void shutdown() {} + /** + * Perform any action before the task is run. + *

+ * This method is invoked from the same thread the task will be executed. + * Task-specific information can be accessed via {@link org.apache.spark.TaskContext#get}. + *

+ * Plugin authors should avoid expensive operations here, as this method will be called + * on every task, and doing something expensive can significantly slow down a job. + * It is not recommended for a user to call a remote service, for example. + *

+ * Exceptions thrown from this method do not propagate - they're caught, + * logged, and suppressed. Therefore exceptions when executing this method won't + * make the job fail. + * + * @since 3.1.0 + */ + default void onTaskStart() {} + + /** + * Perform an action after tasks completes without exceptions. + *

+ * As {@link #onTaskStart() onTaskStart} exceptions are suppressed, this method + * will still be invoked even if the corresponding {@link #onTaskStart} call for this + * task failed. + *

+ * Same warnings of {@link #onTaskStart() onTaskStart} apply here. + * + * @since 3.1.0 + */ + default void onTaskSucceeded() {} + + /** + * Perform an action after tasks completes with exceptions. + *

+ * Same warnings of {@link #onTaskStart() onTaskStart} apply here. + * + * @param failureReason the exception thrown from the failed task. + * + * @since 3.1.0 + */ + default void onTaskFailed(TaskFailedReason failureReason) {} } diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java index 5515a85295d78..79e38a824fea4 100644 --- a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java +++ b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java @@ -18,6 +18,7 @@ package org.apache.spark.shuffle.sort; import java.nio.channels.Channels; +import java.util.Arrays; import java.util.Optional; import javax.annotation.Nullable; import java.io.*; @@ -274,6 +275,8 @@ private long[] mergeSpills(SpillInfo[] spills) throws IOException { // Here, we don't need to perform any metrics updates because the bytes written to this // output file would have already been counted as shuffle bytes written. partitionLengths = spills[0].partitionLengths; + logger.debug("Merge shuffle spills for mapId {} with length {}", mapId, + partitionLengths.length); maybeSingleFileWriter.get().transferMapSpillFile(spills[0].file, partitionLengths); } else { partitionLengths = mergeSpillsUsingStandardWriter(spills); @@ -360,6 +363,7 @@ private void mergeSpillsWithFileStream( SpillInfo[] spills, ShuffleMapOutputWriter mapWriter, @Nullable CompressionCodec compressionCodec) throws IOException { + logger.debug("Merge shuffle spills with FileStream for mapId {}", mapId); final int numPartitions = partitioner.numPartitions(); final InputStream[] spillInputStreams = new InputStream[spills.length]; @@ -369,6 +373,11 @@ private void mergeSpillsWithFileStream( spillInputStreams[i] = new NioBufferedFileInputStream( spills[i].file, inputBufferSizeInBytes); + // Only convert the partitionLengths when debug level is enabled. + if (logger.isDebugEnabled()) { + logger.debug("Partition lengths for mapId {} in Spill {}: {}", mapId, i, + Arrays.toString(spills[i].partitionLengths)); + } } for (int partition = 0; partition < numPartitions; partition++) { boolean copyThrewException = true; @@ -431,6 +440,7 @@ private void mergeSpillsWithFileStream( private void mergeSpillsWithTransferTo( SpillInfo[] spills, ShuffleMapOutputWriter mapWriter) throws IOException { + logger.debug("Merge shuffle spills with TransferTo for mapId {}", mapId); final int numPartitions = partitioner.numPartitions(); final FileChannel[] spillInputChannels = new FileChannel[spills.length]; final long[] spillInputChannelPositions = new long[spills.length]; @@ -439,6 +449,11 @@ private void mergeSpillsWithTransferTo( try { for (int i = 0; i < spills.length; i++) { spillInputChannels[i] = new FileInputStream(spills[i].file).getChannel(); + // Only convert the partitionLengths when debug level is enabled. + if (logger.isDebugEnabled()) { + logger.debug("Partition lengths for mapId {} in Spill {}: {}", mapId, i, + Arrays.toString(spills[i].partitionLengths)); + } } for (int partition = 0; partition < numPartitions; partition++) { boolean copyThrewException = true; diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java index eea6c762f5c63..0b286264be43d 100644 --- a/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java +++ b/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java @@ -113,6 +113,8 @@ public MapOutputCommitMessage commitAllPartitions() throws IOException { } cleanUp(); File resolvedTmp = outputTempFile != null && outputTempFile.isFile() ? outputTempFile : null; + log.debug("Writing shuffle index file for mapId {} with length {}", mapId, + partitionLengths.length); blockResolver.writeIndexFileAndCommit(shuffleId, mapId, partitionLengths, resolvedTmp); return MapOutputCommitMessage.of(partitionLengths); } @@ -211,14 +213,14 @@ public long getNumBytesWritten() { private class PartitionWriterStream extends OutputStream { private final int partitionId; - private int count = 0; + private long count = 0; private boolean isClosed = false; PartitionWriterStream(int partitionId) { this.partitionId = partitionId; } - public int getCount() { + public long getCount() { return count; } diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java index 6e028886f2318..d7940fc08e1a5 100644 --- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java +++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java @@ -428,6 +428,68 @@ public MapIterator destructiveIterator() { return new MapIterator(numValues, new Location(), true); } + /** + * Iterator for the entries of this map. This is to first iterate over key indices in + * `longArray` then accessing values in `dataPages`. NOTE: this is different from `MapIterator` + * in the sense that key index is preserved here + * (See `UnsafeHashedRelation` for example of usage). + */ + public final class MapIteratorWithKeyIndex implements Iterator { + + /** + * The index in `longArray` where the key is stored. + */ + private int keyIndex = 0; + + private int numRecords; + private final Location loc; + + private MapIteratorWithKeyIndex() { + this.numRecords = numValues; + this.loc = new Location(); + } + + @Override + public boolean hasNext() { + return numRecords > 0; + } + + @Override + public Location next() { + if (!loc.isDefined() || !loc.nextValue()) { + while (longArray.get(keyIndex * 2) == 0) { + keyIndex++; + } + loc.with(keyIndex, 0, true); + keyIndex++; + } + numRecords--; + return loc; + } + } + + /** + * Returns an iterator for iterating over the entries of this map, + * by first iterating over the key index inside hash map's `longArray`. + * + * For efficiency, all calls to `next()` will return the same {@link Location} object. + * + * The returned iterator is NOT thread-safe. If the map is modified while iterating over it, + * the behavior of the returned iterator is undefined. + */ + public MapIteratorWithKeyIndex iteratorWithKeyIndex() { + return new MapIteratorWithKeyIndex(); + } + + /** + * The maximum number of allowed keys index. + * + * The value of allowed keys index is in the range of [0, maxNumKeysIndex - 1]. + */ + public int maxNumKeysIndex() { + return (int) (longArray.size() / 2); + } + /** * Looks up a key, and return a {@link Location} handle that can be used to test existence * and read/write values. @@ -601,6 +663,14 @@ public boolean isDefined() { return isDefined; } + /** + * Returns index for key. + */ + public int getKeyIndex() { + assert (isDefined); + return pos; + } + /** * Returns the base object for key. */ @@ -738,12 +808,21 @@ public boolean append(Object kbase, long koff, int klen, Object vbase, long voff longArray.set(pos * 2 + 1, keyHashcode); isDefined = true; - // We use two array entries per key, so the array size is twice the capacity. - // We should compare the current capacity of the array, instead of its size. - if (numKeys >= growthThreshold && longArray.size() / 2 < MAX_CAPACITY) { - try { - growAndRehash(); - } catch (SparkOutOfMemoryError oom) { + // If the map has reached its growth threshold, try to grow it. + if (numKeys >= growthThreshold) { + // We use two array entries per key, so the array size is twice the capacity. + // We should compare the current capacity of the array, instead of its size. + if (longArray.size() / 2 < MAX_CAPACITY) { + try { + growAndRehash(); + } catch (SparkOutOfMemoryError oom) { + canGrowArray = false; + } + } else { + // The map is already at MAX_CAPACITY and cannot grow. Instead, we prevent it from + // accepting any more new elements to make sure we don't exceed the load factor. If we + // need to spill later, this allows UnsafeKVExternalSorter to reuse the array for + // sorting. canGrowArray = false; } } diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java index 55e4e609c3c7b..dda8ed4c239ae 100644 --- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java +++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java @@ -203,6 +203,10 @@ public long spill(long size, MemoryConsumer trigger) throws IOException { } if (inMemSorter == null || inMemSorter.numRecords() <= 0) { + // There could still be some memory allocated when there are no records in the in-memory + // sorter. We will not spill it however, to ensure that we can always process at least one + // record before spilling. See the comments in `allocateMemoryForRecordIfNecessary` for why + // this is necessary. return 0L; } @@ -224,7 +228,7 @@ public long spill(long size, MemoryConsumer trigger) throws IOException { // Note that this is more-or-less going to be a multiple of the page size, so wasted space in // pages will currently be counted as memory spilled even though that space isn't actually // written to disk. This also counts the space needed to store the sorter's pointer array. - inMemSorter.reset(); + inMemSorter.freeMemory(); // Reset the in-memory sorter's pointer array only after freeing up the memory pages holding the // records. Otherwise, if the task is over allocated memory, then without freeing the memory // pages, we might not be able to get memory for the pointer array. @@ -325,7 +329,7 @@ public void cleanupResources() { deleteSpillFiles(); freeMemory(); if (inMemSorter != null) { - inMemSorter.free(); + inMemSorter.freeMemory(); inMemSorter = null; } } @@ -339,40 +343,53 @@ public void cleanupResources() { private void growPointerArrayIfNecessary() throws IOException { assert(inMemSorter != null); if (!inMemSorter.hasSpaceForAnotherRecord()) { + if (inMemSorter.numRecords() <= 0) { + // Spilling was triggered just before this method was called. The pointer array was freed + // during the spill, so a new pointer array needs to be allocated here. + LongArray array = allocateArray(inMemSorter.getInitialSize()); + inMemSorter.expandPointerArray(array); + return; + } + long used = inMemSorter.getMemoryUsage(); - LongArray array; + LongArray array = null; try { // could trigger spilling array = allocateArray(used / 8 * 2); } catch (TooLargePageException e) { // The pointer array is too big to fix in a single page, spill. spill(); - return; } catch (SparkOutOfMemoryError e) { - // should have trigger spilling - if (!inMemSorter.hasSpaceForAnotherRecord()) { + if (inMemSorter.numRecords() > 0) { logger.error("Unable to grow the pointer array"); throw e; } - return; + // The new array could not be allocated, but that is not an issue as it is longer needed, + // as all records were spilled. } - // check if spilling is triggered or not - if (inMemSorter.hasSpaceForAnotherRecord()) { - freeArray(array); - } else { - inMemSorter.expandPointerArray(array); + + if (inMemSorter.numRecords() <= 0) { + // Spilling was triggered while trying to allocate the new array. + if (array != null) { + // We succeeded in allocating the new array, but, since all records were spilled, a + // smaller array would also suffice. + freeArray(array); + } + // The pointer array was freed during the spill, so a new pointer array needs to be + // allocated here. + array = allocateArray(inMemSorter.getInitialSize()); } + inMemSorter.expandPointerArray(array); } } /** - * Allocates more memory in order to insert an additional record. This will request additional - * memory from the memory manager and spill if the requested memory can not be obtained. + * Allocates an additional page in order to insert an additional record. This will request + * additional memory from the memory manager and spill if the requested memory can not be + * obtained. * * @param required the required space in the data page, in bytes, including space for storing - * the record size. This must be less than or equal to the page size (records - * that exceed the page size are handled via a different code path which uses - * special overflow pages). + * the record size. */ private void acquireNewPageIfNecessary(int required) { if (currentPage == null || @@ -384,6 +401,37 @@ private void acquireNewPageIfNecessary(int required) { } } + /** + * Allocates more memory in order to insert an additional record. This will request additional + * memory from the memory manager and spill if the requested memory can not be obtained. + * + * @param required the required space in the data page, in bytes, including space for storing + * the record size. + */ + private void allocateMemoryForRecordIfNecessary(int required) throws IOException { + // Step 1: + // Ensure that the pointer array has space for another record. This may cause a spill. + growPointerArrayIfNecessary(); + // Step 2: + // Ensure that the last page has space for another record. This may cause a spill. + acquireNewPageIfNecessary(required); + // Step 3: + // The allocation in step 2 could have caused a spill, which would have freed the pointer + // array allocated in step 1. Therefore we need to check again whether we have to allocate + // a new pointer array. + // + // If the allocation in this step causes a spill event then it will not cause the page + // allocated in the previous step to be freed. The function `spill` only frees memory if at + // least one record has been inserted in the in-memory sorter. This will not be the case if + // we have spilled in the previous step. + // + // If we did not spill in the previous step then `growPointerArrayIfNecessary` will be a + // no-op that does not allocate any memory, and therefore can't cause a spill event. + // + // Thus there is no need to call `acquireNewPageIfNecessary` again after this step. + growPointerArrayIfNecessary(); + } + /** * Write a record to the sorter. */ @@ -398,11 +446,10 @@ public void insertRecord( spill(); } - growPointerArrayIfNecessary(); - int uaoSize = UnsafeAlignedOffset.getUaoSize(); + final int uaoSize = UnsafeAlignedOffset.getUaoSize(); // Need 4 or 8 bytes to store the record length. final int required = length + uaoSize; - acquireNewPageIfNecessary(required); + allocateMemoryForRecordIfNecessary(required); final Object base = currentPage.getBaseObject(); final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor); @@ -425,10 +472,9 @@ public void insertKVRecord(Object keyBase, long keyOffset, int keyLen, Object valueBase, long valueOffset, int valueLen, long prefix, boolean prefixIsNull) throws IOException { - growPointerArrayIfNecessary(); - int uaoSize = UnsafeAlignedOffset.getUaoSize(); + final int uaoSize = UnsafeAlignedOffset.getUaoSize(); final int required = keyLen + valueLen + (2 * uaoSize); - acquireNewPageIfNecessary(required); + allocateMemoryForRecordIfNecessary(required); final Object base = currentPage.getBaseObject(); final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor); @@ -501,10 +547,14 @@ private static void spillIterator(UnsafeSorterIterator inMemIterator, */ class SpillableIterator extends UnsafeSorterIterator { private UnsafeSorterIterator upstream; - private UnsafeSorterIterator nextUpstream = null; private MemoryBlock lastPage = null; private boolean loaded = false; - private int numRecords = 0; + private int numRecords; + + private Object currentBaseObject; + private long currentBaseOffset; + private int currentRecordLength; + private long currentKeyPrefix; SpillableIterator(UnsafeSorterIterator inMemIterator) { this.upstream = inMemIterator; @@ -516,23 +566,32 @@ public int getNumRecords() { return numRecords; } + @Override + public long getCurrentPageNumber() { + throw new UnsupportedOperationException(); + } + public long spill() throws IOException { synchronized (this) { - if (!(upstream instanceof UnsafeInMemorySorter.SortedIterator && nextUpstream == null - && numRecords > 0)) { + if (inMemSorter == null) { return 0L; } - UnsafeInMemorySorter.SortedIterator inMemIterator = - ((UnsafeInMemorySorter.SortedIterator) upstream).clone(); + long currentPageNumber = upstream.getCurrentPageNumber(); - ShuffleWriteMetrics writeMetrics = new ShuffleWriteMetrics(); - // Iterate over the records that have not been returned and spill them. - final UnsafeSorterSpillWriter spillWriter = - new UnsafeSorterSpillWriter(blockManager, fileBufferSizeBytes, writeMetrics, numRecords); - spillIterator(inMemIterator, spillWriter); - spillWriters.add(spillWriter); - nextUpstream = spillWriter.getReader(serializerManager); + ShuffleWriteMetrics writeMetrics = new ShuffleWriteMetrics(); + if (numRecords > 0) { + // Iterate over the records that have not been returned and spill them. + final UnsafeSorterSpillWriter spillWriter = new UnsafeSorterSpillWriter( + blockManager, fileBufferSizeBytes, writeMetrics, numRecords); + spillIterator(upstream, spillWriter); + spillWriters.add(spillWriter); + upstream = spillWriter.getReader(serializerManager); + } else { + // Nothing to spill as all records have been read already, but do not return yet, as the + // memory still has to be freed. + upstream = null; + } long released = 0L; synchronized (UnsafeExternalSorter.this) { @@ -540,8 +599,7 @@ public long spill() throws IOException { // is accessing the current record. We free this page in that caller's next loadNext() // call. for (MemoryBlock page : allocatedPages) { - if (!loaded || page.pageNumber != - ((UnsafeInMemorySorter.SortedIterator)upstream).getCurrentPageNumber()) { + if (!loaded || page.pageNumber != currentPageNumber) { released += page.size(); freePage(page); } else { @@ -549,13 +607,18 @@ public long spill() throws IOException { } } allocatedPages.clear(); + if (lastPage != null) { + // Add the last page back to the list of allocated pages to make sure it gets freed in + // case loadNext() never gets called again. + allocatedPages.add(lastPage); + } } // in-memory sorter will not be used after spilling assert(inMemSorter != null); released += inMemSorter.getMemoryUsage(); totalSortTimeNanos += inMemSorter.getSortTimeNanos(); - inMemSorter.free(); + inMemSorter.freeMemory(); inMemSorter = null; taskContext.taskMetrics().incMemoryBytesSpilled(released); taskContext.taskMetrics().incDiskBytesSpilled(writeMetrics.bytesWritten()); @@ -571,26 +634,32 @@ public boolean hasNext() { @Override public void loadNext() throws IOException { + assert upstream != null; MemoryBlock pageToFree = null; try { synchronized (this) { loaded = true; - if (nextUpstream != null) { - // Just consumed the last record from in memory iterator - if(lastPage != null) { - // Do not free the page here, while we are locking `SpillableIterator`. The `freePage` - // method locks the `TaskMemoryManager`, and it's a bad idea to lock 2 objects in - // sequence. We may hit dead lock if another thread locks `TaskMemoryManager` and - // `SpillableIterator` in sequence, which may happen in - // `TaskMemoryManager.acquireExecutionMemory`. - pageToFree = lastPage; - lastPage = null; - } - upstream = nextUpstream; - nextUpstream = null; + // Just consumed the last record from the in-memory iterator. + if (lastPage != null) { + // Do not free the page here, while we are locking `SpillableIterator`. The `freePage` + // method locks the `TaskMemoryManager`, and it's a bad idea to lock 2 objects in + // sequence. We may hit dead lock if another thread locks `TaskMemoryManager` and + // `SpillableIterator` in sequence, which may happen in + // `TaskMemoryManager.acquireExecutionMemory`. + pageToFree = lastPage; + allocatedPages.clear(); + lastPage = null; } numRecords--; upstream.loadNext(); + + // Keep track of the current base object, base offset, record length, and key prefix, + // so that the current record can still be read in case a spill is triggered and we + // switch to the spill writer's iterator. + currentBaseObject = upstream.getBaseObject(); + currentBaseOffset = upstream.getBaseOffset(); + currentRecordLength = upstream.getRecordLength(); + currentKeyPrefix = upstream.getKeyPrefix(); } } finally { if (pageToFree != null) { @@ -601,22 +670,22 @@ public void loadNext() throws IOException { @Override public Object getBaseObject() { - return upstream.getBaseObject(); + return currentBaseObject; } @Override public long getBaseOffset() { - return upstream.getBaseOffset(); + return currentBaseOffset; } @Override public int getRecordLength() { - return upstream.getRecordLength(); + return currentRecordLength; } @Override public long getKeyPrefix() { - return upstream.getKeyPrefix(); + return currentKeyPrefix; } } @@ -646,7 +715,7 @@ public UnsafeSorterIterator getIterator(int startIndex) throws IOException { } i += spillWriter.recordsSpilled(); } - if (inMemSorter != null) { + if (inMemSorter != null && inMemSorter.numRecords() > 0) { UnsafeSorterIterator iter = inMemSorter.getSortedIterator(); moveOver(iter, startIndex - i); queue.add(iter); @@ -693,6 +762,11 @@ public int getNumRecords() { return numRecords; } + @Override + public long getCurrentPageNumber() { + return current.getCurrentPageNumber(); + } + @Override public boolean hasNext() { while (!current.hasNext() && !iterators.isEmpty()) { diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java index 660eb790a550b..33be899b6b438 100644 --- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java +++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java @@ -159,32 +159,26 @@ private int getUsableCapacity() { return (int) (array.size() / (radixSortSupport != null ? 2 : 1.5)); } + public long getInitialSize() { + return initialSize; + } + /** * Free the memory used by pointer array. */ - public void free() { + public void freeMemory() { if (consumer != null) { if (array != null) { consumer.freeArray(array); } - array = null; - } - } - public void reset() { - if (consumer != null) { - consumer.freeArray(array); - // the call to consumer.allocateArray may trigger a spill which in turn access this instance - // and eventually re-enter this method and try to free the array again. by setting the array - // to null and its length to 0 we effectively make the spill code-path a no-op. setting the - // array to null also indicates that it has already been de-allocated which prevents a double - // de-allocation in free(). + // Set the array to null instead of allocating a new array. Allocating an array could have + // triggered another spill and this method already is called from UnsafeExternalSorter when + // spilling. Attempting to allocate while spilling is dangerous, as we could be holding onto + // a large partially complete allocation, which may prevent other memory from being allocated. + // Instead we will allocate the new array when it is necessary. array = null; usableCapacity = 0; - pos = 0; - nullBoundaryPos = 0; - array = consumer.allocateArray(initialSize); - usableCapacity = getUsableCapacity(); } pos = 0; nullBoundaryPos = 0; @@ -217,18 +211,20 @@ public boolean hasSpaceForAnotherRecord() { } public void expandPointerArray(LongArray newArray) { - if (newArray.size() < array.size()) { - // checkstyle.off: RegexpSinglelineJava - throw new SparkOutOfMemoryError("Not enough memory to grow pointer array"); - // checkstyle.on: RegexpSinglelineJava + if (array != null) { + if (newArray.size() < array.size()) { + // checkstyle.off: RegexpSinglelineJava + throw new SparkOutOfMemoryError("Not enough memory to grow pointer array"); + // checkstyle.on: RegexpSinglelineJava + } + Platform.copyMemory( + array.getBaseObject(), + array.getBaseOffset(), + newArray.getBaseObject(), + newArray.getBaseOffset(), + pos * 8L); + consumer.freeArray(array); } - Platform.copyMemory( - array.getBaseObject(), - array.getBaseOffset(), - newArray.getBaseObject(), - newArray.getBaseOffset(), - pos * 8L); - consumer.freeArray(array); array = newArray; usableCapacity = getUsableCapacity(); } @@ -330,6 +326,7 @@ public void loadNext() { @Override public long getBaseOffset() { return baseOffset; } + @Override public long getCurrentPageNumber() { return currentPageNumber; } @@ -346,6 +343,11 @@ public long getCurrentPageNumber() { * {@code next()} will return the same mutable object. */ public UnsafeSorterIterator getSortedIterator() { + if (numRecords() == 0) { + // `array` might be null, so make sure that it is not accessed by returning early. + return new SortedIterator(0, 0); + } + int offset = 0; long start = System.nanoTime(); if (sortComparator != null) { diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java index 1b3167fcc250c..d9f22311d07c2 100644 --- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java +++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java @@ -34,4 +34,6 @@ public abstract class UnsafeSorterIterator { public abstract long getKeyPrefix(); public abstract int getNumRecords(); + + public abstract long getCurrentPageNumber(); } diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java index ab800288dcb43..f8603c5799e9b 100644 --- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java +++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java @@ -70,6 +70,11 @@ public int getNumRecords() { return numRecords; } + @Override + public long getCurrentPageNumber() { + throw new UnsupportedOperationException(); + } + @Override public boolean hasNext() { return !priorityQueue.isEmpty() || (spillReader != null && spillReader.hasNext()); diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java index a524c4790407d..db79efd008530 100644 --- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java +++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java @@ -89,6 +89,11 @@ public int getNumRecords() { return numRecords; } + @Override + public long getCurrentPageNumber() { + throw new UnsupportedOperationException(); + } + @Override public boolean hasNext() { return (numRecordsRemaining > 0); diff --git a/core/src/main/resources/org/apache/spark/ui/static/jquery-3.4.1.min.js b/core/src/main/resources/org/apache/spark/ui/static/jquery-3.4.1.min.js deleted file mode 100644 index 07c00cd227da0..0000000000000 --- a/core/src/main/resources/org/apache/spark/ui/static/jquery-3.4.1.min.js +++ /dev/null @@ -1,2 +0,0 @@ -/*! jQuery v3.4.1 | (c) JS Foundation and other contributors | jquery.org/license */ -!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],E=C.document,r=Object.getPrototypeOf,s=t.slice,g=t.concat,u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType},x=function(e){return null!=e&&e===e.window},c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.4.1",k=function(e,t){return new k.fn.init(e,t)},p=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;function d(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp($),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+$),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\([\\da-f]{1,6}"+M+"?|("+M+")|.)","ig"),ne=function(e,t,n){var r="0x"+t-65536;return r!=r||n?t:r<0?String.fromCharCode(r+65536):String.fromCharCode(r>>10|55296,1023&r|56320)},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(m.childNodes),m.childNodes),t[m.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&((e?e.ownerDocument||e:m)!==C&&T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!A[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&U.test(t)){(s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=k),o=(l=h(t)).length;while(o--)l[o]="#"+s+" "+xe(l[o]);c=l.join(","),f=ee.test(t)&&ye(e.parentNode)||e}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){A(t,!0)}finally{s===k&&e.removeAttribute("id")}}}return g(t.replace(B,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[k]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:m;return r!==C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),m!==C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=k,!C.getElementsByName||!C.getElementsByName(k).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+k+"-]").length||v.push("~="),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+k+"+*").length||v.push(".#.+[+~]")}),ce(function(e){e.innerHTML="";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",$)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)===(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e===C||e.ownerDocument===m&&y(m,e)?-1:t===C||t.ownerDocument===m&&y(m,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e===C?-1:t===C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]===m?-1:s[r]===m?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if((e.ownerDocument||e)!==C&&T(e),d.matchesSelector&&E&&!A[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){A(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=p[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&p(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?k.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?k.grep(e,function(e){return e===n!==r}):"string"!=typeof n?k.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(k.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||q,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:L.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof k?t[0]:t,k.merge(this,k.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),D.test(r[1])&&k.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(k):k.makeArray(e,this)}).prototype=k.fn,q=k(E);var H=/^(?:parents|prev(?:Until|All))/,O={children:!0,contents:!0,next:!0,prev:!0};function P(e,t){while((e=e[t])&&1!==e.nodeType);return e}k.fn.extend({has:function(e){var t=k(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i,ge={option:[1,""],thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?k.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;nx",y.noCloneChecked=!!me.cloneNode(!0).lastChild.defaultValue;var Te=/^key/,Ce=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,Ee=/^([^.]*)(?:\.(.+)|)/;function ke(){return!0}function Se(){return!1}function Ne(e,t){return e===function(){try{return E.activeElement}catch(e){}}()==("focus"===t)}function Ae(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)Ae(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=Se;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return k().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=k.guid++)),e.each(function(){k.event.add(this,t,i,r,n)})}function De(e,i,o){o?(Q.set(e,i,!1),k.event.add(e,i,{namespace:!1,handler:function(e){var t,n,r=Q.get(this,i);if(1&e.isTrigger&&this[i]){if(r.length)(k.event.special[i]||{}).delegateType&&e.stopPropagation();else if(r=s.call(arguments),Q.set(this,i,r),t=o(this,i),this[i](),r!==(n=Q.get(this,i))||t?Q.set(this,i,!1):n={},r!==n)return e.stopImmediatePropagation(),e.preventDefault(),n.value}else r.length&&(Q.set(this,i,{value:k.event.trigger(k.extend(r[0],k.Event.prototype),r.slice(1),this)}),e.stopImmediatePropagation())}})):void 0===Q.get(e,i)&&k.event.add(e,i,ke)}k.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Q.get(t);if(v){n.handler&&(n=(o=n).handler,i=o.selector),i&&k.find.matchesSelector(ie,i),n.guid||(n.guid=k.guid++),(u=v.events)||(u=v.events={}),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof k&&k.event.triggered!==e.type?k.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(R)||[""]).length;while(l--)d=g=(s=Ee.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=k.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=k.event.special[d]||{},c=k.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&k.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),k.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Q.hasData(e)&&Q.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(R)||[""]).length;while(l--)if(d=g=(s=Ee.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=k.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||k.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)k.event.remove(e,d+t[l],n,r,!0);k.isEmptyObject(u)&&Q.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=k.event.fix(e),u=new Array(arguments.length),l=(Q.get(this,"events")||{})[s.type]||[],c=k.event.special[s.type]||{};for(u[0]=s,t=1;t\x20\t\r\n\f]*)[^>]*)\/>/gi,qe=/\s*$/g;function Oe(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&k(e).children("tbody")[0]||e}function Pe(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function Re(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Me(e,t){var n,r,i,o,a,s,u,l;if(1===t.nodeType){if(Q.hasData(e)&&(o=Q.access(e),a=Q.set(t,o),l=o.events))for(i in delete a.handle,a.events={},l)for(n=0,r=l[i].length;n")},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=oe(e);if(!(y.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||k.isXMLDoc(e)))for(a=ve(c),r=0,i=(o=ve(e)).length;r").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var Vt,Gt=[],Yt=/(=)\?(?=&|$)|\?\?/;k.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Gt.pop()||k.expando+"_"+kt++;return this[e]=!0,e}}),k.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Yt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Yt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Yt,"$1"+r):!1!==e.jsonp&&(e.url+=(St.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||k.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?k(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Gt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((Vt=E.implementation.createHTMLDocument("").body).innerHTML="

",2===Vt.childNodes.length),k.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=D.exec(e))?[t.createElement(i[1])]:(i=we([e],t,o),o&&o.length&&k(o).remove(),k.merge([],i.childNodes)));var r,i,o},k.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(k.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},k.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){k.fn[t]=function(e){return this.on(t,e)}}),k.expr.pseudos.animated=function(t){return k.grep(k.timers,function(e){return t===e.elem}).length},k.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=k.css(e,"position"),c=k(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=k.css(e,"top"),u=k.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,k.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},k.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){k.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===k.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===k.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=k(e).offset()).top+=k.css(e,"borderTopWidth",!0),i.left+=k.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-k.css(r,"marginTop",!0),left:t.left-i.left-k.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===k.css(e,"position"))e=e.offsetParent;return e||ie})}}),k.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;k.fn[t]=function(e){return _(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),k.each(["top","left"],function(e,n){k.cssHooks[n]=ze(y.pixelPosition,function(e,t){if(t)return t=_e(e,n),$e.test(t)?k(e).position()[n]+"px":t})}),k.each({Height:"height",Width:"width"},function(a,s){k.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){k.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return _(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?k.css(e,t,i):k.style(e,t,n,i)},s,n?e:void 0,n)}})}),k.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){k.fn[n]=function(e,t){return 0+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce(function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function D(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||j,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,j=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="",y.option=!!ce.lastChild;var ge={thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n",""]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d\s*$/g;function qe(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function Le(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function He(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Oe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var Ut,Xt=[],Vt=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Xt.pop()||S.expando+"_"+Ct.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Vt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Vt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Vt,"$1"+r):!1!==e.jsonp&&(e.url+=(Et.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Xt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((Ut=E.implementation.createHTMLDocument("").body).innerHTML="
",2===Ut.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):("number"==typeof f.top&&(f.top+="px"),"number"==typeof f.left&&(f.left+="px"),c.css(f))}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=$e(y.pixelPosition,function(e,t){if(t)return t=Be(e,n),Me.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0 0!") } if (!conf.get(config.SHUFFLE_SERVICE_ENABLED)) { - if (conf.get(config.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED)) { + // If dynamic allocation shuffle tracking or worker decommissioning along with + // storage shuffle decommissioning is enabled we have *experimental* support for + // decommissioning without a shuffle service. + if (conf.get(config.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED) || + (decommissionEnabled && + conf.get(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED))) { logWarning("Dynamic allocation without a shuffle service is an experimental feature.") } else if (!testing) { throw new SparkException("Dynamic allocation of executors requires the external " + @@ -239,7 +243,10 @@ private[spark] class ExecutorAllocationManager( } } } - executor.scheduleWithFixedDelay(scheduleTask, 0, intervalMillis, TimeUnit.MILLISECONDS) + + if (!testing || conf.get(TEST_DYNAMIC_ALLOCATION_SCHEDULE_ENABLED)) { + executor.scheduleWithFixedDelay(scheduleTask, 0, intervalMillis, TimeUnit.MILLISECONDS) + } // copy the maps inside synchonize to ensure not being modified val (numExecutorsTarget, numLocalityAware) = synchronized { @@ -271,6 +278,9 @@ private[spark] class ExecutorAllocationManager( numExecutorsTargetPerResourceProfileId.keys.foreach { rpId => numExecutorsTargetPerResourceProfileId(rpId) = initialNumExecutors } + numExecutorsToAddPerResourceProfileId.keys.foreach { rpId => + numExecutorsToAddPerResourceProfileId(rpId) = 1 + } executorMonitor.reset() } @@ -539,7 +549,9 @@ private[spark] class ExecutorAllocationManager( // get the running total as we remove or initialize it to the count - pendingRemoval val newExecutorTotal = numExecutorsTotalPerRpId.getOrElseUpdate(rpId, (executorMonitor.executorCountWithResourceProfile(rpId) - - executorMonitor.pendingRemovalCountPerResourceProfileId(rpId))) + executorMonitor.pendingRemovalCountPerResourceProfileId(rpId) - + executorMonitor.decommissioningPerResourceProfileId(rpId) + )) if (newExecutorTotal - 1 < minNumExecutors) { logDebug(s"Not removing idle executor $executorIdToBeRemoved because there " + s"are only $newExecutorTotal executor(s) left (minimum number of executor limit " + @@ -565,8 +577,17 @@ private[spark] class ExecutorAllocationManager( } else { // We don't want to change our target number of executors, because we already did that // when the task backlog decreased. - client.killExecutors(executorIdsToBeRemoved.toSeq, adjustTargetNumExecutors = false, - countFailures = false, force = false) + if (decommissionEnabled) { + val executorIdsWithoutHostLoss = executorIdsToBeRemoved.toSeq.map( + id => (id, ExecutorDecommissionInfo("spark scale down"))).toArray + client.decommissionExecutors( + executorIdsWithoutHostLoss, + adjustTargetNumExecutors = false, + triggeredByExecutor = false) + } else { + client.killExecutors(executorIdsToBeRemoved.toSeq, adjustTargetNumExecutors = false, + countFailures = false, force = false) + } } // [SPARK-21834] killExecutors api reduces the target number of executors. @@ -578,7 +599,11 @@ private[spark] class ExecutorAllocationManager( // reset the newExecutorTotal to the existing number of executors if (testing || executorsRemoved.nonEmpty) { - executorMonitor.executorsKilled(executorsRemoved.toSeq) + if (decommissionEnabled) { + executorMonitor.executorsDecommissioned(executorsRemoved.toSeq) + } else { + executorMonitor.executorsKilled(executorsRemoved.toSeq) + } logInfo(s"Executors ${executorsRemoved.mkString(",")} removed due to idle timeout.") executorsRemoved.toSeq } else { diff --git a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala index 64102ccc05882..c3152d9225107 100644 --- a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala +++ b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala @@ -125,14 +125,19 @@ private class ShuffleStatus(numPartitions: Int) extends Logging { * Update the map output location (e.g. during migration). */ def updateMapOutput(mapId: Long, bmAddress: BlockManagerId): Unit = withWriteLock { - val mapStatusOpt = mapStatuses.find(_.mapId == mapId) - mapStatusOpt match { - case Some(mapStatus) => - logInfo(s"Updating map output for ${mapId} to ${bmAddress}") - mapStatus.updateLocation(bmAddress) - invalidateSerializedMapOutputStatusCache() - case None => - logError(s"Asked to update map output ${mapId} for untracked map status.") + try { + val mapStatusOpt = mapStatuses.find(_.mapId == mapId) + mapStatusOpt match { + case Some(mapStatus) => + logInfo(s"Updating map output for ${mapId} to ${bmAddress}") + mapStatus.updateLocation(bmAddress) + invalidateSerializedMapOutputStatusCache() + case None => + logWarning(s"Asked to update map output ${mapId} for untracked map status.") + } + } catch { + case e: java.lang.NullPointerException => + logWarning(s"Unable to update map output for ${mapId}, status removed in-flight") } } diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala index dbd89d646ae54..427e98e616515 100644 --- a/core/src/main/scala/org/apache/spark/SparkConf.scala +++ b/core/src/main/scala/org/apache/spark/SparkConf.scala @@ -568,7 +568,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria // If spark.executor.heartbeatInterval bigger than spark.network.timeout, // it will almost always cause ExecutorLostFailure. See SPARK-22754. require(executorTimeoutThresholdMs > executorHeartbeatIntervalMs, "The value of " + - s"${networkTimeout}=${executorTimeoutThresholdMs}ms must be no less than the value of " + + s"${networkTimeout}=${executorTimeoutThresholdMs}ms must be greater than the value of " + s"${EXECUTOR_HEARTBEAT_INTERVAL.key}=${executorHeartbeatIntervalMs}ms.") } diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 78f509c670839..501e865c4105a 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -384,6 +384,7 @@ class SparkContext(config: SparkConf) extends Logging { try { _conf = config.clone() _conf.validateSettings() + _conf.set("spark.app.startTime", startTime.toString) if (!_conf.contains("spark.master")) { throw new SparkException("A master URL must be set in your configuration") @@ -492,11 +493,17 @@ class SparkContext(config: SparkConf) extends Logging { // Add each JAR given through the constructor if (jars != null) { - jars.foreach(addJar) + jars.foreach(jar => addJar(jar, true)) + if (addedJars.nonEmpty) { + _conf.set("spark.app.initial.jar.urls", addedJars.keys.toSeq.mkString(",")) + } } if (files != null) { - files.foreach(addFile) + files.foreach(file => addFile(file, false, true)) + if (addedFiles.nonEmpty) { + _conf.set("spark.app.initial.file.urls", addedFiles.keys.toSeq.mkString(",")) + } } _executorMemory = _conf.getOption(EXECUTOR_MEMORY.key) @@ -1500,7 +1507,7 @@ class SparkContext(config: SparkConf) extends Logging { * @note A path can be added only once. Subsequent additions of the same path are ignored. */ def addFile(path: String): Unit = { - addFile(path, false) + addFile(path, false, false) } /** @@ -1522,6 +1529,10 @@ class SparkContext(config: SparkConf) extends Logging { * @note A path can be added only once. Subsequent additions of the same path are ignored. */ def addFile(path: String, recursive: Boolean): Unit = { + addFile(path, recursive, false) + } + + private def addFile(path: String, recursive: Boolean, addedOnSubmit: Boolean): Unit = { val uri = new Path(path).toUri val schemeCorrectedURI = uri.getScheme match { case null => new File(path).getCanonicalFile.toURI @@ -1559,7 +1570,7 @@ class SparkContext(config: SparkConf) extends Logging { path } } - val timestamp = System.currentTimeMillis + val timestamp = if (addedOnSubmit) startTime else System.currentTimeMillis if (addedFiles.putIfAbsent(key, timestamp).isEmpty) { logInfo(s"Added file $path at $key with timestamp $timestamp") // Fetch the file locally so that closures which are run on the driver can still use the @@ -1569,7 +1580,7 @@ class SparkContext(config: SparkConf) extends Logging { postEnvironmentUpdate() } else { logWarning(s"The path $path has been added already. Overwriting of added paths " + - "is not supported in the current version.") + "is not supported in the current version.") } } @@ -1603,7 +1614,7 @@ class SparkContext(config: SparkConf) extends Logging { /** * Get the max number of tasks that can be concurrent launched based on the ResourceProfile - * being used. + * could be used, even if some of them are being used at the moment. * Note that please don't cache the value returned by this method, because the number can change * due to add/remove executors. * @@ -1840,6 +1851,10 @@ class SparkContext(config: SparkConf) extends Logging { * @note A path can be added only once. Subsequent additions of the same path are ignored. */ def addJar(path: String): Unit = { + addJar(path, false) + } + + private def addJar(path: String, addedOnSubmit: Boolean): Unit = { def addLocalJarFile(file: File): String = { try { if (!file.exists()) { @@ -1866,7 +1881,7 @@ class SparkContext(config: SparkConf) extends Logging { if (!fs.exists(hadoopPath)) { throw new FileNotFoundException(s"Jar ${path} not found") } - if (fs.isDirectory(hadoopPath)) { + if (fs.getFileStatus(hadoopPath).isDirectory) { throw new IllegalArgumentException( s"Directory ${path} is not allowed for addJar") } @@ -1884,7 +1899,7 @@ class SparkContext(config: SparkConf) extends Logging { if (path == null || path.isEmpty) { logWarning("null or empty path specified as parameter to addJar") } else { - val key = if (path.contains("\\")) { + val key = if (path.contains("\\") && Utils.isWindows) { // For local paths with backslashes on Windows, URI throws an exception addLocalJarFile(new File(path)) } else { @@ -1904,7 +1919,7 @@ class SparkContext(config: SparkConf) extends Logging { } } if (key != null) { - val timestamp = System.currentTimeMillis + val timestamp = if (addedOnSubmit) startTime else System.currentTimeMillis if (addedJars.putIfAbsent(key, timestamp).isEmpty) { logInfo(s"Added JAR $path at $key with timestamp $timestamp") postEnvironmentUpdate() diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala b/core/src/main/scala/org/apache/spark/TestUtils.scala index 6947d1c72f12b..9632d6c691085 100644 --- a/core/src/main/scala/org/apache/spark/TestUtils.scala +++ b/core/src/main/scala/org/apache/spark/TestUtils.scala @@ -20,13 +20,14 @@ package org.apache.spark import java.io.{ByteArrayInputStream, File, FileInputStream, FileOutputStream} import java.net.{HttpURLConnection, URI, URL} import java.nio.charset.StandardCharsets -import java.nio.file.{Files => JavaFiles} +import java.nio.file.{Files => JavaFiles, Paths} import java.nio.file.attribute.PosixFilePermission.{OWNER_EXECUTE, OWNER_READ, OWNER_WRITE} import java.security.SecureRandom import java.security.cert.X509Certificate import java.util.{Arrays, EnumSet, Locale, Properties} import java.util.concurrent.{TimeoutException, TimeUnit} import java.util.jar.{JarEntry, JarOutputStream, Manifest} +import java.util.regex.Pattern import javax.net.ssl._ import javax.tools.{JavaFileObject, SimpleJavaFileObject, ToolProvider} @@ -37,6 +38,7 @@ import scala.sys.process.{Process, ProcessLogger} import scala.util.Try import com.google.common.io.{ByteStreams, Files} +import org.apache.commons.lang3.StringUtils import org.apache.log4j.PropertyConfigurator import org.json4s.JsonAST.JValue import org.json4s.jackson.JsonMethods.{compact, render} @@ -255,6 +257,37 @@ private[spark] object TestUtils { attempt.isSuccess && attempt.get == 0 } + def isPythonVersionAtLeast38(): Boolean = { + val attempt = if (Utils.isWindows) { + Try(Process(Seq("cmd.exe", "/C", "python3 --version")) + .run(ProcessLogger(s => s.startsWith("Python 3.8") || s.startsWith("Python 3.9"))) + .exitValue()) + } else { + Try(Process(Seq("sh", "-c", "python3 --version")) + .run(ProcessLogger(s => s.startsWith("Python 3.8") || s.startsWith("Python 3.9"))) + .exitValue()) + } + attempt.isSuccess && attempt.get == 0 + } + + /** + * Get the absolute path from the executable. This implementation was borrowed from + * `spark/dev/sparktestsupport/shellutils.py`. + */ + def getAbsolutePathFromExecutable(executable: String): Option[String] = { + val command = if (Utils.isWindows) s"$executable.exe" else executable + if (command.split(File.separator, 2).length == 1 && + JavaFiles.isRegularFile(Paths.get(command)) && + JavaFiles.isExecutable(Paths.get(command))) { + Some(Paths.get(command).toAbsolutePath.toString) + } else { + sys.env("PATH").split(Pattern.quote(File.pathSeparator)) + .map(path => Paths.get(s"${StringUtils.strip(path, "\"")}${File.separator}$command")) + .find(p => JavaFiles.isRegularFile(p) && JavaFiles.isExecutable(p)) + .map(_.toString) + } + } + /** * Returns the response code from an HTTP(S) URL. */ diff --git a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala index 77fbbc08c2103..1024d9b5060bc 100644 --- a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala +++ b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala @@ -133,22 +133,30 @@ private[spark] class TorrentBroadcast[T: ClassTag](obj: T, id: Long) if (!blockManager.putSingle(broadcastId, value, MEMORY_AND_DISK, tellMaster = false)) { throw new SparkException(s"Failed to store $broadcastId in BlockManager") } - val blocks = - TorrentBroadcast.blockifyObject(value, blockSize, SparkEnv.get.serializer, compressionCodec) - if (checksumEnabled) { - checksums = new Array[Int](blocks.length) - } - blocks.zipWithIndex.foreach { case (block, i) => + try { + val blocks = + TorrentBroadcast.blockifyObject(value, blockSize, SparkEnv.get.serializer, compressionCodec) if (checksumEnabled) { - checksums(i) = calcChecksum(block) + checksums = new Array[Int](blocks.length) } - val pieceId = BroadcastBlockId(id, "piece" + i) - val bytes = new ChunkedByteBuffer(block.duplicate()) - if (!blockManager.putBytes(pieceId, bytes, MEMORY_AND_DISK_SER, tellMaster = true)) { - throw new SparkException(s"Failed to store $pieceId of $broadcastId in local BlockManager") + blocks.zipWithIndex.foreach { case (block, i) => + if (checksumEnabled) { + checksums(i) = calcChecksum(block) + } + val pieceId = BroadcastBlockId(id, "piece" + i) + val bytes = new ChunkedByteBuffer(block.duplicate()) + if (!blockManager.putBytes(pieceId, bytes, MEMORY_AND_DISK_SER, tellMaster = true)) { + throw new SparkException(s"Failed to store $pieceId of $broadcastId " + + s"in local BlockManager") + } } + blocks.length + } catch { + case t: Throwable => + logError(s"Store broadcast $broadcastId fail, remove all pieces of the broadcast") + blockManager.removeBroadcast(id, tellMaster = true) + throw t } - blocks.length } /** Fetch torrent blocks from the driver and/or other executors. */ diff --git a/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala b/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala index c8c6e5a192a24..d5b5375d64f4d 100644 --- a/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala @@ -61,13 +61,35 @@ private[deploy] object DeployMessages { } /** + * An internal message that used by Master itself, in order to handle the + * `DecommissionWorkersOnHosts` request from `MasterWebUI` asynchronously. + * @param ids A collection of Worker ids, which should be decommissioned. + */ + case class DecommissionWorkers(ids: Seq[String]) extends DeployMessage + + /** + * A message that sent from Master to Worker to decommission the Worker. + * It's used for the case where decommission is triggered at MasterWebUI. + * + * Note that decommission a Worker will cause all the executors on that Worker + * to be decommissioned as well. + */ + object DecommissionWorker extends DeployMessage + + /** + * A message that sent by the Worker to itself when it receives PWR signal, + * indicating the Worker starts to decommission. + */ + object WorkerSigPWRReceived extends DeployMessage + + /** + * A message sent from Worker to Master to tell Master that the Worker has started + * decommissioning. It's used for the case where decommission is triggered at Worker. + * * @param id the worker id - * @param worker the worker endpoint ref + * @param workerRef the worker endpoint ref */ - case class WorkerDecommission( - id: String, - worker: RpcEndpointRef) - extends DeployMessage + case class WorkerDecommissioning(id: String, workerRef: RpcEndpointRef) extends DeployMessage case class ExecutorStateChanged( appId: String, @@ -165,8 +187,6 @@ private[deploy] object DeployMessages { case object ReregisterWithMaster // used when a worker attempts to reconnect to a master - case object DecommissionSelf // Mark as decommissioned. May be Master to Worker in the future. - // AppClient to Master case class RegisterApplication(appDescription: ApplicationDescription, driver: RpcEndpointRef) @@ -189,8 +209,10 @@ private[deploy] object DeployMessages { Utils.checkHostPort(hostPort) } + // When the host of Worker is lost or decommissioned, the `workerHost` is the host address + // of that Worker. Otherwise, it's None. case class ExecutorUpdated(id: Int, state: ExecutorState, message: Option[String], - exitStatus: Option[Int], workerLost: Boolean) + exitStatus: Option[Int], workerHost: Option[String]) case class ApplicationRemoved(message: String) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala index 1180501e8c738..6f799a542bc1e 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala @@ -462,6 +462,9 @@ private[spark] object SparkHadoopUtil { for ((key, value) <- conf.getAll if key.startsWith("spark.hadoop.")) { hadoopConf.set(key.substring("spark.hadoop.".length), value) } + if (conf.getOption("spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version").isEmpty) { + hadoopConf.set("mapreduce.fileoutputcommitter.algorithm.version", "1") + } } private def appendSparkHiveConfigs(conf: SparkConf, hadoopConf: Configuration): Unit = { diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 6d38a1d281464..9a316e8c5b5a9 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -28,7 +28,7 @@ import java.util.jar.JarInputStream import scala.annotation.tailrec import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer -import scala.util.{Failure, Properties, Success, Try} +import scala.util.{Properties, Try} import org.apache.commons.io.FilenameUtils import org.apache.commons.lang3.StringUtils @@ -1160,13 +1160,16 @@ private[spark] object SparkSubmitUtils { val br: IBiblioResolver = new IBiblioResolver br.setM2compatible(true) br.setUsepoms(true) + val defaultInternalRepo : Option[String] = sys.env.get("DEFAULT_ARTIFACT_REPOSITORY") + br.setRoot(defaultInternalRepo.getOrElse("https://repo1.maven.org/maven2/")) br.setName("central") cr.add(br) val sp: IBiblioResolver = new IBiblioResolver sp.setM2compatible(true) sp.setUsepoms(true) - sp.setRoot("https://dl.bintray.com/spark-packages/maven") + sp.setRoot(sys.env.getOrElse( + "DEFAULT_ARTIFACT_REPOSITORY", "https://dl.bintray.com/spark-packages/maven")) sp.setName("spark-packages") cr.add(sp) cr @@ -1182,10 +1185,12 @@ private[spark] object SparkSubmitUtils { def resolveDependencyPaths( artifacts: Array[AnyRef], cacheDirectory: File): String = { - artifacts.map { artifactInfo => - val artifact = artifactInfo.asInstanceOf[Artifact].getModuleRevisionId + artifacts.map { ai => + val artifactInfo = ai.asInstanceOf[Artifact] + val artifact = artifactInfo.getModuleRevisionId + val testSuffix = if (artifactInfo.getType == "test-jar") "-tests" else "" cacheDirectory.getAbsolutePath + File.separator + - s"${artifact.getOrganisation}_${artifact.getName}-${artifact.getRevision}.jar" + s"${artifact.getOrganisation}_${artifact.getName}-${artifact.getRevision}${testSuffix}.jar" }.mkString(",") } @@ -1347,6 +1352,13 @@ private[spark] object SparkSubmitUtils { "" } else { val sysOut = System.out + // Default configuration name for ivy + val ivyConfName = "default" + + // A Module descriptor must be specified. Entries are dummy strings + val md = getModuleDescriptor + + md.setDefaultConf(ivyConfName) try { // To prevent ivy from logging to system out System.setOut(printStream) @@ -1374,14 +1386,6 @@ private[spark] object SparkSubmitUtils { resolveOptions.setDownload(true) } - // Default configuration name for ivy - val ivyConfName = "default" - - // A Module descriptor must be specified. Entries are dummy strings - val md = getModuleDescriptor - - md.setDefaultConf(ivyConfName) - // Add exclusion rules for Spark and Scala Library addExclusionRules(ivySettings, ivyConfName, md) // add all supplied maven artifacts as dependencies @@ -1399,12 +1403,10 @@ private[spark] object SparkSubmitUtils { packagesDirectory.getAbsolutePath + File.separator + "[organization]_[artifact]-[revision](-[classifier]).[ext]", retrieveOptions.setConfs(Array(ivyConfName))) - val paths = resolveDependencyPaths(rr.getArtifacts.toArray, packagesDirectory) - val mdId = md.getModuleRevisionId - clearIvyResolutionFiles(mdId, ivySettings, ivyConfName) - paths + resolveDependencyPaths(rr.getArtifacts.toArray, packagesDirectory) } finally { System.setOut(sysOut) + clearIvyResolutionFiles(md.getModuleRevisionId, ivySettings, ivyConfName) } } } diff --git a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala index e08709ec36ab0..c7c31a85b0636 100644 --- a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala +++ b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala @@ -149,11 +149,11 @@ private[spark] object StandaloneResourceUtils extends Logging { // used for UI def formatResourcesUsed( - resourcesTotal: Map[String, ResourceInformation], - resourcesUsed: Map[String, ResourceInformation]): String = { - resourcesTotal.map { case (rName, rInfo) => - val used = resourcesUsed(rName).addresses.length - val total = rInfo.addresses.length + resourcesTotal: Map[String, Int], + resourcesUsed: Map[String, Int]): String = { + resourcesTotal.map { case (rName, totalSize) => + val used = resourcesUsed(rName) + val total = totalSize s"$used / $total $rName" }.mkString(", ") } diff --git a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala index a6da8393bf405..e5efb15f6bc51 100644 --- a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala +++ b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala @@ -175,15 +175,15 @@ private[spark] class StandaloneAppClient( cores)) listener.executorAdded(fullId, workerId, hostPort, cores, memory) - case ExecutorUpdated(id, state, message, exitStatus, workerLost) => + case ExecutorUpdated(id, state, message, exitStatus, workerHost) => val fullId = appId + "/" + id val messageText = message.map(s => " (" + s + ")").getOrElse("") logInfo("Executor updated: %s is now %s%s".format(fullId, state, messageText)) if (ExecutorState.isFinished(state)) { - listener.executorRemoved(fullId, message.getOrElse(""), exitStatus, workerLost) + listener.executorRemoved(fullId, message.getOrElse(""), exitStatus, workerHost) } else if (state == ExecutorState.DECOMMISSIONED) { listener.executorDecommissioned(fullId, - ExecutorDecommissionInfo(message.getOrElse(""), isHostDecommissioned = workerLost)) + ExecutorDecommissionInfo(message.getOrElse(""), workerHost)) } case WorkerRemoved(id, host, message) => diff --git a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClientListener.scala b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClientListener.scala index e72f7e976bb0a..76970ac9829c9 100644 --- a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClientListener.scala +++ b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClientListener.scala @@ -39,7 +39,7 @@ private[spark] trait StandaloneAppClientListener { fullId: String, workerId: String, hostPort: String, cores: Int, memory: Int): Unit def executorRemoved( - fullId: String, message: String, exitStatus: Option[Int], workerLost: Boolean): Unit + fullId: String, message: String, exitStatus: Option[Int], workerHost: Option[String]): Unit def executorDecommissioned(fullId: String, decommissionInfo: ExecutorDecommissionInfo): Unit diff --git a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala index 472b52957ed7f..f3f7db6bb0aba 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala @@ -150,4 +150,11 @@ private[history] abstract class ApplicationHistoryProvider { */ def onUIDetached(appId: String, attemptId: Option[String], ui: SparkUI): Unit = { } + /** + * Returns true if the given user has permission to view the UI of the given attempt. + * + * @throws NoSuchElementException if the given attempt doesn't exist + */ + def checkUIViewPermissions(appId: String, attemptId: Option[String], user: String): Boolean + } diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala index a73a5e9463204..400c82c1f9e63 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala @@ -27,6 +27,7 @@ import java.util.zip.ZipOutputStream import scala.collection.JavaConverters._ import scala.collection.mutable import scala.io.Source +import scala.util.control.NonFatal import scala.xml.Node import com.fasterxml.jackson.annotation.JsonIgnore @@ -358,15 +359,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) } val conf = this.conf.clone() - val secManager = new SecurityManager(conf) - - secManager.setAcls(historyUiAclsEnable) - // make sure to set admin acls before view acls so they are properly picked up - secManager.setAdminAcls(historyUiAdminAcls ++ stringToSeq(attempt.adminAcls.getOrElse(""))) - secManager.setViewAcls(attempt.info.sparkUser, stringToSeq(attempt.viewAcls.getOrElse(""))) - secManager.setAdminAclsGroups(historyUiAdminAclsGroups ++ - stringToSeq(attempt.adminAclsGroups.getOrElse(""))) - secManager.setViewAclsGroups(stringToSeq(attempt.viewAclsGroups.getOrElse(""))) + val secManager = createSecurityManager(conf, attempt) val kvstore = try { diskManager match { @@ -460,6 +453,17 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) } } + override def checkUIViewPermissions(appId: String, attemptId: Option[String], + user: String): Boolean = { + val app = load(appId) + val attempt = app.attempts.find(_.info.attemptId == attemptId).orNull + if (attempt == null) { + throw new NoSuchElementException() + } + val secManager = createSecurityManager(this.conf.clone(), attempt) + secManager.checkUIViewPermissions(user) + } + /** * Builds the application list based on the current contents of the log directory. * Tries to reuse as much of the data already in memory as possible, by not reading @@ -530,10 +534,21 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) // If the file is currently not being tracked by the SHS, add an entry for it and try // to parse it. This will allow the cleaner code to detect the file as stale later on // if it was not possible to parse it. - listing.write(LogInfo(reader.rootPath.toString(), newLastScanTime, LogType.EventLogs, - None, None, reader.fileSizeForLastIndex, reader.lastIndex, None, - reader.completed)) - reader.fileSizeForLastIndex > 0 + try { + listing.write(LogInfo(reader.rootPath.toString(), newLastScanTime, + LogType.EventLogs, None, None, reader.fileSizeForLastIndex, reader.lastIndex, + None, reader.completed)) + reader.fileSizeForLastIndex > 0 + } catch { + case _: FileNotFoundException => false + case NonFatal(e) => + logWarning(s"Error while reading new log ${reader.rootPath}", e) + false + } + + case NonFatal(e) => + logWarning(s"Error while filtering log ${reader.rootPath}", e) + false } } .sortWith { case (entry1, entry2) => @@ -1364,6 +1379,19 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) endProcessing(rootPath) } } + + private def createSecurityManager(conf: SparkConf, + attempt: AttemptInfoWrapper): SecurityManager = { + val secManager = new SecurityManager(conf) + secManager.setAcls(historyUiAclsEnable) + // make sure to set admin acls before view acls so they are properly picked up + secManager.setAdminAcls(historyUiAdminAcls ++ stringToSeq(attempt.adminAcls.getOrElse(""))) + secManager.setViewAcls(attempt.info.sparkUser, stringToSeq(attempt.viewAcls.getOrElse(""))) + secManager.setAdminAclsGroups(historyUiAdminAclsGroups ++ + stringToSeq(attempt.adminAclsGroups.getOrElse(""))) + secManager.setViewAclsGroups(stringToSeq(attempt.viewAclsGroups.getOrElse(""))) + secManager + } } private[history] object FsHistoryProvider { diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index ca21a8056d1b5..bb13f34818a62 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -128,6 +128,11 @@ class HistoryServer( appCache.withSparkUI(appId, attemptId)(fn) } + override def checkUIViewPermissions(appId: String, attemptId: Option[String], + user: String): Boolean = { + provider.checkUIViewPermissions(appId, attemptId, user) + } + initialize() /** diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala index 7fc0722233854..00e58cbdc57b9 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala @@ -33,8 +33,9 @@ private class HistoryServerMemoryManager( conf: SparkConf) extends Logging { private val maxUsage = conf.get(MAX_IN_MEMORY_STORE_USAGE) - private val currentUsage = new AtomicLong(0L) - private val active = new HashMap[(String, Option[String]), Long]() + // Visible for testing. + private[history] val currentUsage = new AtomicLong(0L) + private[history] val active = new HashMap[(String, Option[String]), Long]() def initialize(): Unit = { logInfo("Initialized memory manager: " + diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HybridStore.scala b/core/src/main/scala/org/apache/spark/deploy/history/HybridStore.scala index 08db2bd0766c3..58714f16e8417 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HybridStore.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HybridStore.scala @@ -54,7 +54,8 @@ private[history] class HybridStore extends KVStore { private var backgroundThread: Thread = null // A hash map that stores all classes that had been writen to inMemoryStore - private val klassMap = new ConcurrentHashMap[Class[_], Boolean] + // Visible for testing + private[history] val klassMap = new ConcurrentHashMap[Class[_], Boolean] override def getMetadata[T](klass: Class[T]): T = { getStore().getMetadata(klass) @@ -165,8 +166,9 @@ private[history] class HybridStore extends KVStore { /** * This method return the store that we should use. + * Visible for testing. */ - private def getStore(): KVStore = { + private[history] def getStore(): KVStore = { if (shouldUseInMemoryStore.get) { inMemoryStore } else { diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala index 220e1c963d5ea..ceeb01149f5db 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala @@ -245,8 +245,7 @@ private[deploy] class Master( logError("Leadership has been revoked -- master shutting down.") System.exit(0) - case WorkerDecommission(id, workerRef) => - logInfo("Recording worker %s decommissioning".format(id)) + case WorkerDecommissioning(id, workerRef) => if (state == RecoveryState.STANDBY) { workerRef.send(MasterInStandby) } else { @@ -254,6 +253,19 @@ private[deploy] class Master( idToWorker.get(id).foreach(decommissionWorker) } + case DecommissionWorkers(ids) => + // The caller has already checked the state when handling DecommissionWorkersOnHosts, + // so it should not be the STANDBY + assert(state != RecoveryState.STANDBY) + ids.foreach ( id => + // We use foreach since get gives us an option and we can skip the failures. + idToWorker.get(id).foreach { w => + decommissionWorker(w) + // Also send a message to the worker node to notify. + w.endpoint.send(DecommissionWorker) + } + ) + case RegisterWorker( id, workerHost, workerPort, workerRef, cores, memory, workerWebUiUrl, masterAddress, resources) => @@ -308,7 +320,7 @@ private[deploy] class Master( appInfo.resetRetryCount() } - exec.application.driver.send(ExecutorUpdated(execId, state, message, exitStatus, false)) + exec.application.driver.send(ExecutorUpdated(execId, state, message, exitStatus, None)) if (ExecutorState.isFinished(state)) { // Remove this executor from the worker and app @@ -891,10 +903,7 @@ private[deploy] class Master( logInfo(s"Decommissioning the workers with host:ports ${workersToRemoveHostPorts}") // The workers are removed async to avoid blocking the receive loop for the entire batch - workersToRemove.foreach(wi => { - logInfo(s"Sending the worker decommission to ${wi.id} and ${wi.endpoint}") - self.send(WorkerDecommission(wi.id, wi.endpoint)) - }) + self.send(DecommissionWorkers(workersToRemove.map(_.id).toSeq)) // Return the count of workers actually removed workersToRemove.size @@ -909,9 +918,10 @@ private[deploy] class Master( exec.application.driver.send(ExecutorUpdated( exec.id, ExecutorState.DECOMMISSIONED, Some("worker decommissioned"), None, - // workerLost is being set to true here to let the driver know that the host (aka. worker) - // is also being decommissioned. - workerLost = true)) + // worker host is being set here to let the driver know that the host (aka. worker) + // is also being decommissioned. So the driver can unregister all the shuffle map + // statues located at this host when it receives the executor lost event. + Some(worker.host))) exec.state = ExecutorState.DECOMMISSIONED exec.application.removeExecutor(exec) } @@ -932,7 +942,7 @@ private[deploy] class Master( for (exec <- worker.executors.values) { logInfo("Telling app of lost executor: " + exec.id) exec.application.driver.send(ExecutorUpdated( - exec.id, ExecutorState.LOST, Some("worker lost"), None, workerLost = true)) + exec.id, ExecutorState.LOST, Some("worker lost"), None, Some(worker.host))) exec.state = ExecutorState.LOST exec.application.removeExecutor(exec) } diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala index 47bc34e1b7930..9e1f753b51e5a 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala @@ -76,19 +76,17 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") { private def formatMasterResourcesInUse(aliveWorkers: Array[WorkerInfo]): String = { val totalInfo = aliveWorkers.map(_.resourcesInfo) - .map(resources => toMutable(resources)) .flatMap(_.toIterator) .groupBy(_._1) // group by resource name .map { case (rName, rInfoArr) => - rName -> rInfoArr.map(_._2).reduce(_ + _) - }.map { case (k, v) => (k, v.toResourceInformation) } + rName -> rInfoArr.map(_._2.addresses.size).sum + } val usedInfo = aliveWorkers.map(_.resourcesInfoUsed) - .map (resources => toMutable(resources)) .flatMap(_.toIterator) .groupBy(_._1) // group by resource name .map { case (rName, rInfoArr) => - rName -> rInfoArr.map(_._2).reduce(_ + _) - }.map { case (k, v) => (k, v.toResourceInformation) } + rName -> rInfoArr.map(_._2.addresses.size).sum + } formatResourcesUsed(totalInfo, usedInfo) } diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala index 4e91e72361488..a46864e2d3c9c 100644 --- a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala @@ -139,7 +139,9 @@ private[deploy] object HadoopFSDelegationTokenProvider { def hadoopFSsToAccess( sparkConf: SparkConf, hadoopConf: Configuration): Set[FileSystem] = { + // scalastyle:off FileSystemGet val defaultFS = FileSystem.get(hadoopConf) + // scalastyle:on FileSystemGet val filesystemsToAccess = sparkConf.get(KERBEROS_FILESYSTEMS_TO_ACCESS) .map(new Path(_).getFileSystem(hadoopConf)) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala index aa8c46fc68315..0660dbdafd605 100755 --- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala @@ -67,10 +67,13 @@ private[deploy] class Worker( assert (port > 0) // If worker decommissioning is enabled register a handler on PWR to shutdown. - if (conf.get(WORKER_DECOMMISSION_ENABLED)) { + if (conf.get(config.DECOMMISSION_ENABLED)) { logInfo("Registering SIGPWR handler to trigger decommissioning.") SignalUtils.register("PWR", "Failed to register SIGPWR handler - " + - "disabling worker decommission feature.")(decommissionSelf) + "disabling worker decommission feature.") { + self.send(WorkerSigPWRReceived) + true + } } else { logInfo("Worker decommissioning not enabled, SIGPWR will result in exiting.") } @@ -137,7 +140,8 @@ private[deploy] class Worker( private var registered = false private var connected = false private var decommissioned = false - private val workerId = generateWorkerId() + // expose for test + private[spark] val workerId = generateWorkerId() private val sparkHome = if (sys.props.contains(IS_TESTING.key)) { assert(sys.props.contains("spark.test.home"), "spark.test.home is not set!") @@ -668,8 +672,14 @@ private[deploy] class Worker( finishedApps += id maybeCleanupApplication(id) - case DecommissionSelf => + case DecommissionWorker => + decommissionSelf() + + case WorkerSigPWRReceived => decommissionSelf() + // Tell the Master that we are starting decommissioning + // so it stops trying to launch executor/driver on us + sendToMaster(WorkerDecommissioning(workerId, self)) } override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = { @@ -768,16 +778,15 @@ private[deploy] class Worker( } } - private[deploy] def decommissionSelf(): Boolean = { - if (conf.get(WORKER_DECOMMISSION_ENABLED)) { - logDebug("Decommissioning self") + private[deploy] def decommissionSelf(): Unit = { + if (conf.get(config.DECOMMISSION_ENABLED) && !decommissioned) { decommissioned = true - sendToMaster(WorkerDecommission(workerId, self)) + logInfo(s"Decommission worker $workerId.") + } else if (decommissioned) { + logWarning(s"Worker $workerId already started decommissioning.") } else { - logWarning("Asked to decommission self, but decommissioning not enabled") + logWarning(s"Receive decommission request, but decommission feature is disabled.") } - // Return true since can be called as a signal handler - true } private[worker] def handleDriverStateChanged(driverStateChanged: DriverStateChanged): Unit = { diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala index def125bb6bfb6..b2bc6b3b68007 100644 --- a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala +++ b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala @@ -40,7 +40,7 @@ import org.apache.spark.resource.ResourceProfile import org.apache.spark.resource.ResourceProfile._ import org.apache.spark.resource.ResourceUtils._ import org.apache.spark.rpc._ -import org.apache.spark.scheduler.{ExecutorDecommissionInfo, ExecutorLossReason, TaskDescription} +import org.apache.spark.scheduler.{ExecutorLossReason, TaskDescription} import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages._ import org.apache.spark.serializer.SerializerInstance import org.apache.spark.util.{ChildFirstURLClassLoader, MutableURLClassLoader, SignalUtils, ThreadUtils, Utils} @@ -64,7 +64,6 @@ private[spark] class CoarseGrainedExecutorBackend( private[this] val stopping = new AtomicBoolean(false) var executor: Executor = null - @volatile private var decommissioned = false @volatile var driver: Option[RpcEndpointRef] = None // If this CoarseGrainedExecutorBackend is changed to support multiple threads, then this may need @@ -80,10 +79,14 @@ private[spark] class CoarseGrainedExecutorBackend( */ private[executor] val taskResources = new mutable.HashMap[Long, Map[String, ResourceInformation]] + private var decommissioned = false + override def onStart(): Unit = { - logInfo("Registering PWR handler.") - SignalUtils.register("PWR", "Failed to register SIGPWR handler - " + - "disabling decommission feature.")(decommissionSelf) + if (env.conf.get(DECOMMISSION_ENABLED)) { + logInfo("Registering PWR handler to trigger decommissioning.") + SignalUtils.register("PWR", "Failed to register SIGPWR handler - " + + "disabling executor decommission feature.") (self.askSync[Boolean](ExecutorSigPWRReceived)) + } logInfo("Connecting to driver: " + driverUrl) try { @@ -165,20 +168,6 @@ private[spark] class CoarseGrainedExecutorBackend( if (executor == null) { exitExecutor(1, "Received LaunchTask command but executor was null") } else { - if (decommissioned) { - val msg = "Asked to launch a task while decommissioned." - logError(msg) - driver match { - case Some(endpoint) => - logInfo("Sending DecommissionExecutor to driver.") - endpoint.send( - DecommissionExecutor( - executorId, - ExecutorDecommissionInfo(msg, isHostDecommissioned = false))) - case _ => - logError("No registered driver to send Decommission to.") - } - } val taskDesc = TaskDescription.decode(data.value) logInfo("Got assigned task " + taskDesc.taskId) taskResources(taskDesc.taskId) = taskDesc.resources @@ -214,6 +203,30 @@ private[spark] class CoarseGrainedExecutorBackend( case UpdateDelegationTokens(tokenBytes) => logInfo(s"Received tokens of ${tokenBytes.length} bytes") SparkHadoopUtil.get.addDelegationTokens(tokenBytes, env.conf) + + case DecommissionExecutor => + decommissionSelf() + } + + override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = { + case ExecutorSigPWRReceived => + var driverNotified = false + try { + driver.foreach { driverRef => + // Tell driver that we are starting decommissioning so it stops trying to schedule us + driverNotified = driverRef.askSync[Boolean](ExecutorDecommissioning(executorId)) + if (driverNotified) decommissionSelf() + } + } catch { + case e: Exception => + if (driverNotified) { + logError("Fail to decommission self (but driver has been notified).", e) + } else { + logError("Fail to tell driver that we are starting decommissioning", e) + } + decommissioned = false + } + context.reply(decommissioned) } override def onDisconnected(remoteAddress: RpcAddress): Unit = { @@ -262,28 +275,82 @@ private[spark] class CoarseGrainedExecutorBackend( System.exit(code) } - private def decommissionSelf(): Boolean = { - val msg = "Decommissioning self w/sync" + private def decommissionSelf(): Unit = { + if (!env.conf.get(DECOMMISSION_ENABLED)) { + logWarning(s"Receive decommission request, but decommission feature is disabled.") + return + } else if (decommissioned) { + logWarning(s"Executor $executorId already started decommissioning.") + return + } + val msg = s"Decommission executor $executorId." logInfo(msg) try { decommissioned = true - // Tell master we are are decommissioned so it stops trying to schedule us - if (driver.nonEmpty) { - driver.get.askSync[Boolean](DecommissionExecutor( - executorId, ExecutorDecommissionInfo(msg, false))) - } else { - logError("No driver to message decommissioning.") + if (env.conf.get(STORAGE_DECOMMISSION_ENABLED)) { + env.blockManager.decommissionBlockManager() } if (executor != null) { executor.decommission() } - logInfo("Done decommissioning self.") - // Return true since we are handling a signal - true + // Shutdown the executor once all tasks are gone & any configured migrations completed. + // Detecting migrations completion doesn't need to be perfect and we want to minimize the + // overhead for executors that are not in decommissioning state as overall that will be + // more of the executors. For example, this will not catch a block which is already in + // the process of being put from a remote executor before migration starts. This trade-off + // is viewed as acceptable to minimize introduction of any new locking structures in critical + // code paths. + + val shutdownThread = new Thread("wait-for-blocks-to-migrate") { + override def run(): Unit = { + var lastTaskRunningTime = System.nanoTime() + val sleep_time = 1000 // 1s + // This config is internal and only used by unit tests to force an executor + // to hang around for longer when decommissioned. + val initialSleepMillis = env.conf.getInt( + "spark.test.executor.decommission.initial.sleep.millis", sleep_time) + if (initialSleepMillis > 0) { + Thread.sleep(initialSleepMillis) + } + while (true) { + logInfo("Checking to see if we can shutdown.") + if (executor == null || executor.numRunningTasks == 0) { + if (env.conf.get(STORAGE_DECOMMISSION_ENABLED)) { + logInfo("No running tasks, checking migrations") + val (migrationTime, allBlocksMigrated) = env.blockManager.lastMigrationInfo() + // We can only trust allBlocksMigrated boolean value if there were no tasks running + // since the start of computing it. + if (allBlocksMigrated && (migrationTime > lastTaskRunningTime)) { + logInfo("No running tasks, all blocks migrated, stopping.") + exitExecutor(0, "Finished decommissioning", notifyDriver = true) + } else { + logInfo("All blocks not yet migrated.") + } + } else { + logInfo("No running tasks, no block migration configured, stopping.") + exitExecutor(0, "Finished decommissioning", notifyDriver = true) + } + } else { + logInfo("Blocked from shutdown by running ${executor.numRunningtasks} tasks") + // If there is a running task it could store blocks, so make sure we wait for a + // migration loop to complete after the last task is done. + // Note: this is only advanced if there is a running task, if there + // is no running task but the blocks are not done migrating this does not + // move forward. + lastTaskRunningTime = System.nanoTime() + } + Thread.sleep(sleep_time) + } + } + } + shutdownThread.setDaemon(true) + shutdownThread.start() + + logInfo("Will exit when finished decommissioning") } catch { case e: Exception => - logError(s"Error ${e} during attempt to decommission self") - false + decommissioned = false + logError("Unexpected error while decommissioning self", e) } } } diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index d22002917472a..6653650615192 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -220,6 +220,20 @@ private[spark] class Executor( heartbeater.start() + private val appStartTime = conf.getLong("spark.app.startTime", 0) + + // To allow users to distribute plugins and their required files + // specified by --jars and --files on application submission, those jars/files should be + // downloaded and added to the class loader via updateDependencies. + // This should be done before plugin initialization below + // because executors search plugins from the class loader and initialize them. + private val Seq(initialUserJars, initialUserFiles) = Seq("jar", "file").map { key => + conf.getOption(s"spark.app.initial.$key.urls").map { urls => + Map(urls.split(",").map(url => (url, appStartTime)): _*) + }.getOrElse(Map.empty) + } + updateDependencies(initialUserFiles, initialUserJars) + // Plugins need to load using a class loader that includes the executor's user classpath. // Plugins also needs to be initialized after the heartbeater started // to avoid blocking to send heartbeat (see SPARK-32175). @@ -239,7 +253,7 @@ private[spark] class Executor( } def launchTask(context: ExecutorBackend, taskDescription: TaskDescription): Unit = { - val tr = new TaskRunner(context, taskDescription) + val tr = new TaskRunner(context, taskDescription, plugins) runningTasks.put(taskDescription.taskId, tr) threadPool.execute(tr) if (decommissioned) { @@ -318,12 +332,13 @@ private[spark] class Executor( class TaskRunner( execBackend: ExecutorBackend, - private val taskDescription: TaskDescription) + private val taskDescription: TaskDescription, + private val plugins: Option[PluginContainer]) extends Runnable { val taskId = taskDescription.taskId - val threadName = s"Executor task launch worker for task $taskId" val taskName = taskDescription.name + val threadName = s"Executor task launch worker for $taskName" val mdcProperties = taskDescription.properties.asScala .filter(_._1.startsWith("mdc.")).toSeq @@ -350,7 +365,7 @@ private[spark] class Executor( @volatile var task: Task[Any] = _ def kill(interruptThread: Boolean, reason: String): Unit = { - logInfo(s"Executor is trying to kill $taskName (TID $taskId), reason: $reason") + logInfo(s"Executor is trying to kill $taskName, reason: $reason") reasonIfKilled = Some(reason) if (task != null) { synchronized { @@ -386,7 +401,9 @@ private[spark] class Executor( // Report executor runtime and JVM gc time Option(task).foreach(t => { t.metrics.setExecutorRunTime(TimeUnit.NANOSECONDS.toMillis( - System.nanoTime() - taskStartTimeNs)) + // SPARK-32898: it's possible that a task is killed when taskStartTimeNs has the initial + // value(=0) still. In this case, the executorRunTime should be considered as 0. + if (taskStartTimeNs > 0) System.nanoTime() - taskStartTimeNs else 0)) t.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) }) @@ -411,7 +428,7 @@ private[spark] class Executor( } else 0L Thread.currentThread.setContextClassLoader(replClassLoader) val ser = env.closureSerializer.newInstance() - logInfo(s"Running $taskName (TID $taskId)") + logInfo(s"Running $taskName") execBackend.statusUpdate(taskId, TaskState.RUNNING, EMPTY_BYTE_BUFFER) var taskStartTimeNs: Long = 0 var taskStartCpu: Long = 0 @@ -445,7 +462,7 @@ private[spark] class Executor( // MapOutputTrackerMaster and its cache invalidation is not based on epoch numbers so // we don't need to make any special calls here. if (!isLocal) { - logDebug("Task " + taskId + "'s epoch is " + task.epoch) + logDebug(s"$taskName's epoch is ${task.epoch}") env.mapOutputTracker.asInstanceOf[MapOutputTrackerWorker].updateEpoch(task.epoch) } @@ -463,7 +480,8 @@ private[spark] class Executor( taskAttemptId = taskId, attemptNumber = taskDescription.attemptNumber, metricsSystem = env.metricsSystem, - resources = taskDescription.resources) + resources = taskDescription.resources, + plugins = plugins) threwException = false res } { @@ -471,7 +489,7 @@ private[spark] class Executor( val freedMemory = taskMemoryManager.cleanUpAllAllocatedMemory() if (freedMemory > 0 && !threwException) { - val errMsg = s"Managed memory leak detected; size = $freedMemory bytes, TID = $taskId" + val errMsg = s"Managed memory leak detected; size = $freedMemory bytes, $taskName" if (conf.get(UNSAFE_EXCEPTION_ON_MEMORY_LEAK)) { throw new SparkException(errMsg) } else { @@ -481,7 +499,7 @@ private[spark] class Executor( if (releasedLocks.nonEmpty && !threwException) { val errMsg = - s"${releasedLocks.size} block locks were not released by TID = $taskId:\n" + + s"${releasedLocks.size} block locks were not released by $taskName\n" + releasedLocks.mkString("[", ", ", "]") if (conf.get(STORAGE_EXCEPTION_PIN_LEAK)) { throw new SparkException(errMsg) @@ -494,7 +512,7 @@ private[spark] class Executor( // uh-oh. it appears the user code has caught the fetch-failure without throwing any // other exceptions. Its *possible* this is what the user meant to do (though highly // unlikely). So we will log an error and keep going. - logError(s"TID ${taskId} completed successfully though internally it encountered " + + logError(s"$taskName completed successfully though internally it encountered " + s"unrecoverable fetch failures! Most likely this means user code is incorrectly " + s"swallowing Spark's internal ${classOf[FetchFailedException]}", fetchFailure) } @@ -578,7 +596,7 @@ private[spark] class Executor( // directSend = sending directly back to the driver val serializedResult: ByteBuffer = { if (maxResultSize > 0 && resultSize > maxResultSize) { - logWarning(s"Finished $taskName (TID $taskId). Result is larger than maxResultSize " + + logWarning(s"Finished $taskName. Result is larger than maxResultSize " + s"(${Utils.bytesToString(resultSize)} > ${Utils.bytesToString(maxResultSize)}), " + s"dropping it.") ser.serialize(new IndirectTaskResult[Any](TaskResultBlockId(taskId), resultSize)) @@ -588,40 +606,40 @@ private[spark] class Executor( blockId, new ChunkedByteBuffer(serializedDirectResult.duplicate()), StorageLevel.MEMORY_AND_DISK_SER) - logInfo( - s"Finished $taskName (TID $taskId). $resultSize bytes result sent via BlockManager)") + logInfo(s"Finished $taskName. $resultSize bytes result sent via BlockManager)") ser.serialize(new IndirectTaskResult[Any](blockId, resultSize)) } else { - logInfo(s"Finished $taskName (TID $taskId). $resultSize bytes result sent to driver") + logInfo(s"Finished $taskName. $resultSize bytes result sent to driver") serializedDirectResult } } executorSource.SUCCEEDED_TASKS.inc(1L) setTaskFinishedAndClearInterruptStatus() + plugins.foreach(_.onTaskSucceeded()) execBackend.statusUpdate(taskId, TaskState.FINISHED, serializedResult) } catch { case t: TaskKilledException => - logInfo(s"Executor killed $taskName (TID $taskId), reason: ${t.reason}") + logInfo(s"Executor killed $taskName, reason: ${t.reason}") val (accums, accUpdates) = collectAccumulatorsAndResetStatusOnFailure(taskStartTimeNs) // Here and below, put task metric peaks in a WrappedArray to expose them as a Seq // without requiring a copy. val metricPeaks = WrappedArray.make(metricsPoller.getTaskMetricPeaks(taskId)) - val serializedTK = ser.serialize( - TaskKilled(t.reason, accUpdates, accums, metricPeaks.toSeq)) - execBackend.statusUpdate(taskId, TaskState.KILLED, serializedTK) + val reason = TaskKilled(t.reason, accUpdates, accums, metricPeaks.toSeq) + plugins.foreach(_.onTaskFailed(reason)) + execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason)) case _: InterruptedException | NonFatal(_) if task != null && task.reasonIfKilled.isDefined => val killReason = task.reasonIfKilled.getOrElse("unknown reason") - logInfo(s"Executor interrupted and killed $taskName (TID $taskId), reason: $killReason") + logInfo(s"Executor interrupted and killed $taskName, reason: $killReason") val (accums, accUpdates) = collectAccumulatorsAndResetStatusOnFailure(taskStartTimeNs) val metricPeaks = WrappedArray.make(metricsPoller.getTaskMetricPeaks(taskId)) - val serializedTK = ser.serialize( - TaskKilled(killReason, accUpdates, accums, metricPeaks.toSeq)) - execBackend.statusUpdate(taskId, TaskState.KILLED, serializedTK) + val reason = TaskKilled(killReason, accUpdates, accums, metricPeaks.toSeq) + plugins.foreach(_.onTaskFailed(reason)) + execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason)) case t: Throwable if hasFetchFailure && !Utils.isFatalError(t) => val reason = task.context.fetchFailed.get.toTaskFailedReason @@ -629,29 +647,31 @@ private[spark] class Executor( // there was a fetch failure in the task, but some user code wrapped that exception // and threw something else. Regardless, we treat it as a fetch failure. val fetchFailedCls = classOf[FetchFailedException].getName - logWarning(s"TID ${taskId} encountered a ${fetchFailedCls} and " + + logWarning(s"$taskName encountered a ${fetchFailedCls} and " + s"failed, but the ${fetchFailedCls} was hidden by another " + s"exception. Spark is handling this like a fetch failure and ignoring the " + s"other exception: $t") } setTaskFinishedAndClearInterruptStatus() + plugins.foreach(_.onTaskFailed(reason)) execBackend.statusUpdate(taskId, TaskState.FAILED, ser.serialize(reason)) case CausedBy(cDE: CommitDeniedException) => val reason = cDE.toTaskCommitDeniedReason setTaskFinishedAndClearInterruptStatus() + plugins.foreach(_.onTaskFailed(reason)) execBackend.statusUpdate(taskId, TaskState.KILLED, ser.serialize(reason)) case t: Throwable if env.isStopped => // Log the expected exception after executor.stop without stack traces // see: SPARK-19147 - logError(s"Exception in $taskName (TID $taskId): ${t.getMessage}") + logError(s"Exception in $taskName: ${t.getMessage}") case t: Throwable => // Attempt to exit cleanly by informing the driver of our failure. // If anything goes wrong (or this was a fatal exception), we will delegate to // the default uncaught exception handler, which will terminate the Executor. - logError(s"Exception in $taskName (TID $taskId)", t) + logError(s"Exception in $taskName", t) // SPARK-20904: Do not report failure to driver if if happened during shut down. Because // libraries may set up shutdown hooks that race with running tasks during shutdown, @@ -662,21 +682,22 @@ private[spark] class Executor( val (accums, accUpdates) = collectAccumulatorsAndResetStatusOnFailure(taskStartTimeNs) val metricPeaks = WrappedArray.make(metricsPoller.getTaskMetricPeaks(taskId)) - val serializedTaskEndReason = { + val (taskFailureReason, serializedTaskFailureReason) = { try { val ef = new ExceptionFailure(t, accUpdates).withAccums(accums) .withMetricPeaks(metricPeaks.toSeq) - ser.serialize(ef) + (ef, ser.serialize(ef)) } catch { case _: NotSerializableException => // t is not serializable so just send the stacktrace val ef = new ExceptionFailure(t, accUpdates, false).withAccums(accums) .withMetricPeaks(metricPeaks.toSeq) - ser.serialize(ef) + (ef, ser.serialize(ef)) } } setTaskFinishedAndClearInterruptStatus() - execBackend.statusUpdate(taskId, TaskState.FAILED, serializedTaskEndReason) + plugins.foreach(_.onTaskFailed(taskFailureReason)) + execBackend.statusUpdate(taskId, TaskState.FAILED, serializedTaskFailureReason) } else { logInfo("Not reporting error to driver during JVM shutdown.") } diff --git a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala index a1ebe5ce0ca32..7b8b204bab640 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala @@ -26,11 +26,11 @@ private[spark] object Tests { .longConf .createWithDefault(Runtime.getRuntime.maxMemory) - val TEST_SCHEDULE_INTERVAL = - ConfigBuilder("spark.testing.dynamicAllocation.scheduleInterval") - .version("2.3.0") - .longConf - .createWithDefault(100) + val TEST_DYNAMIC_ALLOCATION_SCHEDULE_ENABLED = + ConfigBuilder("spark.testing.dynamicAllocation.schedule.enabled") + .version("3.1.0") + .booleanConf + .createWithDefault(true) val IS_TESTING = ConfigBuilder("spark.testing") .version("1.0.1") diff --git a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala index 52fcafad8b49a..a8072712c46ce 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala @@ -82,10 +82,4 @@ private[spark] object Worker { .version("2.0.2") .intConf .createWithDefault(100) - - private[spark] val WORKER_DECOMMISSION_ENABLED = - ConfigBuilder("spark.worker.decommission.enabled") - .version("3.1.0") - .booleanConf - .createWithDefault(false) } diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 200cde0a2d3ed..9a7039a9cfe93 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -1415,10 +1415,9 @@ package object config { private[spark] val SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED = ConfigBuilder("spark.shuffle.readHostLocalDisk") - .doc(s"If enabled (and `${SHUFFLE_USE_OLD_FETCH_PROTOCOL.key}` is disabled and external " + - s"shuffle `${SHUFFLE_SERVICE_ENABLED.key}` is enabled), shuffle " + - "blocks requested from those block managers which are running on the same host are read " + - "from the disk directly instead of being fetched as remote blocks over the network.") + .doc(s"If enabled (and `${SHUFFLE_USE_OLD_FETCH_PROTOCOL.key}` is disabled, shuffle " + + "blocks requested from those block managers which are running on the same host are " + + "read from the disk directly instead of being fetched as remote blocks over the network.") .version("3.0.0") .booleanConf .createWithDefault(true) @@ -1866,6 +1865,19 @@ package object config { .timeConf(TimeUnit.MILLISECONDS) .createOptional + private[spark] val DECOMMISSION_ENABLED = + ConfigBuilder("spark.decommission.enabled") + .doc("When decommission enabled, Spark will try its best to shutdown the executor " + + s"gracefully. Spark will try to migrate all the RDD blocks (controlled by " + + s"${STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED.key}) and shuffle blocks (controlled by " + + s"${STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED.key}) from the decommissioning " + + s"executor to a remote executor when ${STORAGE_DECOMMISSION_ENABLED.key} is enabled. " + + s"With decommission enabled, Spark will also decommission an executor instead of " + + s"killing when ${DYN_ALLOCATION_ENABLED.key} enabled.") + .version("3.1.0") + .booleanConf + .createWithDefault(false) + private[spark] val EXECUTOR_DECOMMISSION_KILL_INTERVAL = ConfigBuilder("spark.executor.decommission.killInterval") .doc("Duration after which a decommissioned executor will be killed forcefully." + diff --git a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala index a619f10bbf064..6d174b5e0f81b 100644 --- a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala +++ b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala @@ -222,7 +222,9 @@ class HadoopMapRedWriteConfigUtil[K, V: ClassTag](conf: SerializableJobConf) if (path != null) { path.getFileSystem(getConf) } else { + // scalastyle:off FileSystemGet FileSystem.get(getConf) + // scalastyle:on FileSystemGet } } @@ -285,7 +287,9 @@ class HadoopMapRedWriteConfigUtil[K, V: ClassTag](conf: SerializableJobConf) if (SparkHadoopWriterUtils.isOutputSpecValidationEnabled(conf)) { // FileOutputFormat ignores the filesystem parameter + // scalastyle:off FileSystemGet val ignoredFs = FileSystem.get(getConf) + // scalastyle:on FileSystemGet getOutputFormat().checkOutputSpecs(ignoredFs, getConf) } } diff --git a/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala b/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala index 4eda4767094ad..f78ec250f7173 100644 --- a/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala +++ b/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala @@ -20,7 +20,7 @@ package org.apache.spark.internal.plugin import scala.collection.JavaConverters._ import scala.util.{Either, Left, Right} -import org.apache.spark.{SparkContext, SparkEnv} +import org.apache.spark.{SparkContext, SparkEnv, TaskFailedReason} import org.apache.spark.api.plugin._ import org.apache.spark.internal.Logging import org.apache.spark.internal.config._ @@ -31,6 +31,9 @@ sealed abstract class PluginContainer { def shutdown(): Unit def registerMetrics(appId: String): Unit + def onTaskStart(): Unit + def onTaskSucceeded(): Unit + def onTaskFailed(failureReason: TaskFailedReason): Unit } @@ -85,6 +88,17 @@ private class DriverPluginContainer( } } + override def onTaskStart(): Unit = { + throw new IllegalStateException("Should not be called for the driver container.") + } + + override def onTaskSucceeded(): Unit = { + throw new IllegalStateException("Should not be called for the driver container.") + } + + override def onTaskFailed(failureReason: TaskFailedReason): Unit = { + throw new IllegalStateException("Should not be called for the driver container.") + } } private class ExecutorPluginContainer( @@ -134,6 +148,39 @@ private class ExecutorPluginContainer( } } } + + override def onTaskStart(): Unit = { + executorPlugins.foreach { case (name, plugin) => + try { + plugin.onTaskStart() + } catch { + case t: Throwable => + logInfo(s"Exception while calling onTaskStart on plugin $name.", t) + } + } + } + + override def onTaskSucceeded(): Unit = { + executorPlugins.foreach { case (name, plugin) => + try { + plugin.onTaskSucceeded() + } catch { + case t: Throwable => + logInfo(s"Exception while calling onTaskSucceeded on plugin $name.", t) + } + } + } + + override def onTaskFailed(failureReason: TaskFailedReason): Unit = { + executorPlugins.foreach { case (name, plugin) => + try { + plugin.onTaskFailed(failureReason) + } catch { + case t: Throwable => + logInfo(s"Exception while calling onTaskFailed on plugin $name.", t) + } + } + } } object PluginContainer { diff --git a/core/src/main/scala/org/apache/spark/network/BlockDataManager.scala b/core/src/main/scala/org/apache/spark/network/BlockDataManager.scala index 0bd5774b632bf..62fbc166167d3 100644 --- a/core/src/main/scala/org/apache/spark/network/BlockDataManager.scala +++ b/core/src/main/scala/org/apache/spark/network/BlockDataManager.scala @@ -27,6 +27,11 @@ import org.apache.spark.storage.{BlockId, ShuffleBlockId, StorageLevel} private[spark] trait BlockDataManager { + /** + * Get the local directories that used by BlockManager to save the blocks to disk + */ + def getLocalDiskDirs: Array[String] + /** * Interface to get host-local shuffle block data. Throws an exception if the block cannot be * found or cannot be read successfully. diff --git a/core/src/main/scala/org/apache/spark/network/BlockTransferService.scala b/core/src/main/scala/org/apache/spark/network/BlockTransferService.scala index 70a159f3eeecf..c7f5a97e35612 100644 --- a/core/src/main/scala/org/apache/spark/network/BlockTransferService.scala +++ b/core/src/main/scala/org/apache/spark/network/BlockTransferService.scala @@ -34,7 +34,7 @@ import org.apache.spark.util.ThreadUtils * BlockTransferService contains both client and server inside. */ private[spark] -abstract class BlockTransferService extends BlockStoreClient with Logging { +abstract class BlockTransferService extends BlockStoreClient { /** * Initialize the transfer service by giving it the BlockDataManager that can be used to fetch @@ -110,6 +110,7 @@ abstract class BlockTransferService extends BlockStoreClient with Logging { * This method is similar to [[uploadBlock]], except this one blocks the thread * until the upload finishes. */ + @throws[java.io.IOException] def uploadBlockSync( hostname: String, port: Int, diff --git a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala index 62726f7e147c5..5f831dc666ca5 100644 --- a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala +++ b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala @@ -29,7 +29,7 @@ import org.apache.spark.network.client.{RpcResponseCallback, StreamCallbackWithI import org.apache.spark.network.server.{OneForOneStreamManager, RpcHandler, StreamManager} import org.apache.spark.network.shuffle.protocol._ import org.apache.spark.serializer.Serializer -import org.apache.spark.storage.{BlockId, ShuffleBlockBatchId, ShuffleBlockId, StorageLevel} +import org.apache.spark.storage.{BlockId, BlockManager, ShuffleBlockBatchId, ShuffleBlockId, StorageLevel} /** * Serves requests to open blocks by simply registering one chunk per block requested. @@ -113,6 +113,26 @@ class NettyBlockRpcServer( s"when there is not sufficient space available to store the block.") responseContext.onFailure(exception) } + + case getLocalDirs: GetLocalDirsForExecutors => + val isIncorrectAppId = getLocalDirs.appId != appId + val execNum = getLocalDirs.execIds.length + if (isIncorrectAppId || execNum != 1) { + val errorMsg = "Invalid GetLocalDirsForExecutors request: " + + s"${if (isIncorrectAppId) s"incorrect application id: ${getLocalDirs.appId};"}" + + s"${if (execNum != 1) s"incorrect executor number: $execNum (expected 1);"}" + responseContext.onFailure(new IllegalStateException(errorMsg)) + } else { + val expectedExecId = blockManager.asInstanceOf[BlockManager].executorId + val actualExecId = getLocalDirs.execIds.head + if (actualExecId != expectedExecId) { + responseContext.onFailure(new IllegalStateException( + s"Invalid executor id: $actualExecId, expected $expectedExecId.")) + } else { + responseContext.onSuccess(new LocalDirsForExecutors( + Map(actualExecId -> blockManager.getLocalDiskDirs).asJava).toByteBuffer) + } + } } } diff --git a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala index 5d9cea068b097..806fbf52795bc 100644 --- a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala +++ b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala @@ -19,7 +19,9 @@ package org.apache.spark.network.netty import java.io.IOException import java.nio.ByteBuffer +import java.util import java.util.{HashMap => JHashMap, Map => JMap} +import java.util.concurrent.CompletableFuture import scala.collection.JavaConverters._ import scala.concurrent.{Future, Promise} @@ -33,11 +35,11 @@ import org.apache.spark.ExecutorDeadException import org.apache.spark.internal.config import org.apache.spark.network._ import org.apache.spark.network.buffer.{ManagedBuffer, NioManagedBuffer} -import org.apache.spark.network.client.{RpcResponseCallback, TransportClientBootstrap, TransportClientFactory} +import org.apache.spark.network.client.{RpcResponseCallback, TransportClient, TransportClientBootstrap, TransportClientFactory} import org.apache.spark.network.crypto.{AuthClientBootstrap, AuthServerBootstrap} import org.apache.spark.network.server._ import org.apache.spark.network.shuffle.{BlockFetchingListener, DownloadFileManager, OneForOneBlockFetcher, RetryingBlockFetcher} -import org.apache.spark.network.shuffle.protocol.{UploadBlock, UploadBlockStream} +import org.apache.spark.network.shuffle.protocol.{BlockTransferMessage, GetLocalDirsForExecutors, LocalDirsForExecutors, UploadBlock, UploadBlockStream} import org.apache.spark.network.util.JavaUtils import org.apache.spark.rpc.RpcEndpointRef import org.apache.spark.serializer.JavaSerializer @@ -65,8 +67,6 @@ private[spark] class NettyBlockTransferService( private[this] var transportContext: TransportContext = _ private[this] var server: TransportServer = _ - private[this] var clientFactory: TransportClientFactory = _ - private[this] var appId: String = _ override def init(blockDataManager: BlockDataManager): Unit = { val rpcHandler = new NettyBlockRpcServer(conf.getAppId, serializer, blockDataManager) @@ -80,7 +80,7 @@ private[spark] class NettyBlockTransferService( clientFactory = transportContext.createClientFactory(clientBootstrap.toSeq.asJava) server = createServer(serverBootstrap.toList) appId = conf.getAppId - logInfo(s"Server created on ${hostName}:${server.getPort}") + logger.info(s"Server created on $hostName:${server.getPort}") } /** Creates and binds the TransportServer, possibly trying multiple ports. */ @@ -113,7 +113,9 @@ private[spark] class NettyBlockTransferService( blockIds: Array[String], listener: BlockFetchingListener, tempFileManager: DownloadFileManager): Unit = { - logTrace(s"Fetch blocks from $host:$port (executor id $execId)") + if (logger.isTraceEnabled) { + logger.trace(s"Fetch blocks from $host:$port (executor id $execId)") + } try { val maxRetries = transportConf.maxIORetries() val blockFetchStarter = new RetryingBlockFetcher.BlockFetchStarter { @@ -146,7 +148,7 @@ private[spark] class NettyBlockTransferService( } } catch { case e: Exception => - logError("Exception while beginning fetchBlocks", e) + logger.error("Exception while beginning fetchBlocks", e) blockIds.foreach(listener.onBlockFetchFailure(_, e)) } } @@ -174,12 +176,14 @@ private[spark] class NettyBlockTransferService( blockId.isShuffle) val callback = new RpcResponseCallback { override def onSuccess(response: ByteBuffer): Unit = { - logTrace(s"Successfully uploaded block $blockId${if (asStream) " as stream" else ""}") + if (logger.isTraceEnabled) { + logger.trace(s"Successfully uploaded block $blockId${if (asStream) " as stream" else ""}") + } result.success((): Unit) } override def onFailure(e: Throwable): Unit = { - logError(s"Error while uploading $blockId${if (asStream) " as stream" else ""}", e) + logger.error(s"Error while uploading $blockId${if (asStream) " as stream" else ""}", e) result.failure(e) } } diff --git a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala index d5f21112c0c9e..5fc0b4f736d55 100644 --- a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala @@ -232,6 +232,10 @@ class HadoopRDD[K, V]( logWarning(s"${jobConf.get(FileInputFormat.INPUT_DIR)} doesn't exist and no" + s" partitions returned from this path.", e) Array.empty[Partition] + case e: IOException if e.getMessage.startsWith("Not a file:") => + val path = e.getMessage.split(":").map(_.trim).apply(2) + throw new IOException(s"Path: ${path} is a directory, which is not supported by the " + + s"record reader when `mapreduce.input.fileinputformat.input.dir.recursive` is false.") } } diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala index 11a9bb86d3034..d861e91771673 100644 --- a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala +++ b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala @@ -29,7 +29,7 @@ import org.apache.spark.util.Utils.executeAndGetOutput /** * The default plugin that is loaded into a Spark application to control how custom * resources are discovered. This executes the discovery script specified by the user - * and gets the json output back and contructs ResourceInformation objects from that. + * and gets the json output back and constructs ResourceInformation objects from that. * If the user specifies custom plugins, this is the last one to be executed and * throws if the resource isn't discovered. * diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala index 162f090f011c6..5a9435653920f 100644 --- a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala +++ b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala @@ -149,7 +149,12 @@ private[spark] object ResourceUtils extends Logging { def listResourceIds(sparkConf: SparkConf, componentName: String): Seq[ResourceID] = { sparkConf.getAllWithPrefix(s"$componentName.$RESOURCE_PREFIX.").map { case (key, _) => - key.substring(0, key.indexOf('.')) + val index = key.indexOf('.') + if (index < 0) { + throw new SparkException(s"You must specify an amount config for resource: $key " + + s"config: $componentName.$RESOURCE_PREFIX.$key") + } + key.substring(0, index) }.distinct.map(name => new ResourceID(componentName, name)) } diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala b/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala index 2ed03f7430c32..472401b23fe8e 100644 --- a/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala +++ b/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala @@ -200,6 +200,16 @@ private[netty] class Inbox(val endpointName: String, val endpoint: RpcEndpoint) * Calls action closure, and calls the endpoint's onError function in the case of exceptions. */ private def safelyCall(endpoint: RpcEndpoint)(action: => Unit): Unit = { + def dealWithFatalError(fatal: Throwable): Unit = { + inbox.synchronized { + assert(numActiveThreads > 0, "The number of active threads should be positive.") + // Should reduce the number of active threads before throw the error. + numActiveThreads -= 1 + } + logError(s"An error happened while processing message in the inbox for $endpointName", fatal) + throw fatal + } + try action catch { case NonFatal(e) => try endpoint.onError(e) catch { @@ -209,8 +219,18 @@ private[netty] class Inbox(val endpointName: String, val endpoint: RpcEndpoint) } else { logError("Ignoring error", ee) } + case fatal: Throwable => + dealWithFatalError(fatal) } + case fatal: Throwable => + dealWithFatalError(fatal) } } + // exposed only for testing + def getNumActiveThreads: Int = { + inbox.synchronized { + inbox.numActiveThreads + } + } } diff --git a/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala b/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala index 2274e6898adf6..043c6b90384b4 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala @@ -60,6 +60,6 @@ private[spark] object BarrierJobAllocationFailed { val ERROR_MESSAGE_BARRIER_REQUIRE_MORE_SLOTS_THAN_CURRENT_TOTAL_NUMBER = "[SPARK-24819]: Barrier execution mode does not allow run a barrier stage that requires " + "more slots than the total number of slots in the cluster currently. Please init a new " + - "cluster with more CPU cores or repartition the input RDD(s) to reduce the number of " + - "slots required to run this barrier stage." + "cluster with more resources(e.g. CPU, GPU) or repartition the input RDD(s) to reduce " + + "the number of slots required to run this barrier stage." } diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index 6b376cdadc66b..080e0e7f1552f 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -480,10 +480,12 @@ private[spark] class DAGScheduler( * submission. */ private def checkBarrierStageWithNumSlots(rdd: RDD[_], rp: ResourceProfile): Unit = { - val numPartitions = rdd.getNumPartitions - val maxNumConcurrentTasks = sc.maxNumConcurrentTasks(rp) - if (rdd.isBarrier() && numPartitions > maxNumConcurrentTasks) { - throw new BarrierJobSlotsNumberCheckFailed(numPartitions, maxNumConcurrentTasks) + if (rdd.isBarrier()) { + val numPartitions = rdd.getNumPartitions + val maxNumConcurrentTasks = sc.maxNumConcurrentTasks(rp) + if (numPartitions > maxNumConcurrentTasks) { + throw new BarrierJobSlotsNumberCheckFailed(numPartitions, maxNumConcurrentTasks) + } } } @@ -1823,8 +1825,8 @@ private[spark] class DAGScheduler( if (bmAddress != null) { val externalShuffleServiceEnabled = env.blockManager.externalShuffleServiceEnabled val isHostDecommissioned = taskScheduler - .getExecutorDecommissionInfo(bmAddress.executorId) - .exists(_.isHostDecommissioned) + .getExecutorDecommissionState(bmAddress.executorId) + .exists(_.workerHost.isDefined) // Shuffle output of all executors on host `bmAddress.host` may be lost if: // - External shuffle service is enabled, so we assume that all shuffle data on node is @@ -1844,7 +1846,14 @@ private[spark] class DAGScheduler( execId = bmAddress.executorId, fileLost = true, hostToUnregisterOutputs = hostToUnregisterOutputs, - maybeEpoch = Some(task.epoch)) + maybeEpoch = Some(task.epoch), + // shuffleFileLostEpoch is ignored when a host is decommissioned because some + // decommissioned executors on that host might have been removed before this fetch + // failure and might have bumped up the shuffleFileLostEpoch. We ignore that, and + // proceed with unconditional removal of shuffle outputs from all executors on that + // host, including from those that we still haven't confirmed as lost due to heartbeat + // delays. + ignoreShuffleFileLostEpoch = isHostDecommissioned) } } @@ -1980,15 +1989,15 @@ private[spark] class DAGScheduler( */ private[scheduler] def handleExecutorLost( execId: String, - workerLost: Boolean): Unit = { + workerHost: Option[String]): Unit = { // if the cluster manager explicitly tells us that the entire worker was lost, then // we know to unregister shuffle output. (Note that "worker" specifically refers to the process // from a Standalone cluster, where the shuffle service lives in the Worker.) - val fileLost = workerLost || !env.blockManager.externalShuffleServiceEnabled + val fileLost = workerHost.isDefined || !env.blockManager.externalShuffleServiceEnabled removeExecutorAndUnregisterOutputs( execId = execId, fileLost = fileLost, - hostToUnregisterOutputs = None, + hostToUnregisterOutputs = workerHost, maybeEpoch = None) } @@ -2010,7 +2019,8 @@ private[spark] class DAGScheduler( execId: String, fileLost: Boolean, hostToUnregisterOutputs: Option[String], - maybeEpoch: Option[Long] = None): Unit = { + maybeEpoch: Option[Long] = None, + ignoreShuffleFileLostEpoch: Boolean = false): Unit = { val currentEpoch = maybeEpoch.getOrElse(mapOutputTracker.getEpoch) logDebug(s"Considering removal of executor $execId; " + s"fileLost: $fileLost, currentEpoch: $currentEpoch") @@ -2020,16 +2030,25 @@ private[spark] class DAGScheduler( blockManagerMaster.removeExecutor(execId) clearCacheLocs() } - if (fileLost && - (!shuffleFileLostEpoch.contains(execId) || shuffleFileLostEpoch(execId) < currentEpoch)) { - shuffleFileLostEpoch(execId) = currentEpoch - hostToUnregisterOutputs match { - case Some(host) => - logInfo(s"Shuffle files lost for host: $host (epoch $currentEpoch)") - mapOutputTracker.removeOutputsOnHost(host) - case None => - logInfo(s"Shuffle files lost for executor: $execId (epoch $currentEpoch)") - mapOutputTracker.removeOutputsOnExecutor(execId) + if (fileLost) { + val remove = if (ignoreShuffleFileLostEpoch) { + true + } else if (!shuffleFileLostEpoch.contains(execId) || + shuffleFileLostEpoch(execId) < currentEpoch) { + shuffleFileLostEpoch(execId) = currentEpoch + true + } else { + false + } + if (remove) { + hostToUnregisterOutputs match { + case Some(host) => + logInfo(s"Shuffle files lost for host: $host (epoch $currentEpoch)") + mapOutputTracker.removeOutputsOnHost(host) + case None => + logInfo(s"Shuffle files lost for executor: $execId (epoch $currentEpoch)") + mapOutputTracker.removeOutputsOnExecutor(execId) + } } } } @@ -2347,11 +2366,12 @@ private[scheduler] class DAGSchedulerEventProcessLoop(dagScheduler: DAGScheduler dagScheduler.handleExecutorAdded(execId, host) case ExecutorLost(execId, reason) => - val workerLost = reason match { - case ExecutorProcessLost(_, true, _) => true - case _ => false + val workerHost = reason match { + case ExecutorProcessLost(_, workerHost, _) => workerHost + case ExecutorDecommission(workerHost) => workerHost + case _ => None } - dagScheduler.handleExecutorLost(execId, workerLost) + dagScheduler.handleExecutorLost(execId, workerHost) case WorkerRemoved(workerId, host, message) => dagScheduler.handleWorkerRemoved(workerId, host, message) diff --git a/core/src/main/scala/org/apache/spark/scheduler/ExecutorDecommissionInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/ExecutorDecommissionInfo.scala index a82b5d38afe9f..7eec070232c3b 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ExecutorDecommissionInfo.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ExecutorDecommissionInfo.scala @@ -18,11 +18,23 @@ package org.apache.spark.scheduler /** - * Provides more detail when an executor is being decommissioned. + * Message providing more detail when an executor is being decommissioned. * @param message Human readable reason for why the decommissioning is happening. - * @param isHostDecommissioned Whether the host (aka the `node` or `worker` in other places) is - * being decommissioned too. Used to infer if the shuffle data might - * be lost even if the external shuffle service is enabled. + * @param workerHost When workerHost is defined, it means the host (aka the `node` or `worker` + * in other places) has been decommissioned too. Used to infer if the + * shuffle data might be lost even if the external shuffle service is enabled. */ private[spark] -case class ExecutorDecommissionInfo(message: String, isHostDecommissioned: Boolean) +case class ExecutorDecommissionInfo(message: String, workerHost: Option[String] = None) + +/** + * State related to decommissioning that is kept by the TaskSchedulerImpl. This state is derived + * from the info message above but it is kept distinct to allow the state to evolve independently + * from the message. + */ +case class ExecutorDecommissionState( + // Timestamp the decommissioning commenced as per the Driver's clock, + // to estimate when the executor might eventually be lost if EXECUTOR_DECOMMISSION_KILL_INTERVAL + // is configured. + startTime: Long, + workerHost: Option[String] = None) diff --git a/core/src/main/scala/org/apache/spark/scheduler/ExecutorLossReason.scala b/core/src/main/scala/org/apache/spark/scheduler/ExecutorLossReason.scala index 671dedaa5a6e8..2644d0af2ac50 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ExecutorLossReason.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ExecutorLossReason.scala @@ -53,14 +53,15 @@ private [spark] object LossReasonPending extends ExecutorLossReason("Pending los /** * @param _message human readable loss reason - * @param workerLost whether the worker is confirmed lost too (i.e. including shuffle service) + * @param workerHost it's defined when the host is confirmed lost too (i.e. including + * shuffle service) * @param causedByApp whether the loss of the executor is the fault of the running app. * (assumed true by default unless known explicitly otherwise) */ private[spark] case class ExecutorProcessLost( _message: String = "Executor Process Lost", - workerLost: Boolean = false, + workerHost: Option[String] = None, causedByApp: Boolean = true) extends ExecutorLossReason(_message) @@ -69,5 +70,9 @@ case class ExecutorProcessLost( * * This is used by the task scheduler to remove state associated with the executor, but * not yet fail any tasks that were running in the executor before the executor is "fully" lost. + * If you update this code make sure to re-run the K8s integration tests. + * + * @param workerHost it is defined when the worker is decommissioned too */ -private [spark] object ExecutorDecommission extends ExecutorLossReason("Executor decommission.") +private [spark] case class ExecutorDecommission(workerHost: Option[String] = None) + extends ExecutorLossReason("Executor decommission.") diff --git a/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala index fd04db8c09d76..508c6cebd9fe3 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala @@ -36,4 +36,5 @@ private[spark] class ExecutorResourceInfo( override protected def resourceName = this.name override protected def resourceAddresses = this.addresses override protected def slotsPerAddress: Int = numParts + def totalAddressAmount: Int = resourceAddresses.length * slotsPerAddress } diff --git a/core/src/main/scala/org/apache/spark/scheduler/Pool.scala b/core/src/main/scala/org/apache/spark/scheduler/Pool.scala index 2e2851eb9070b..7333b31524f2a 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Pool.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Pool.scala @@ -59,6 +59,8 @@ private[spark] class Pool( } } + override def isSchedulable: Boolean = true + override def addSchedulable(schedulable: Schedulable): Unit = { require(schedulable != null) schedulableQueue.add(schedulable) @@ -105,7 +107,7 @@ private[spark] class Pool( val sortedSchedulableQueue = schedulableQueue.asScala.toSeq.sortWith(taskSetSchedulingAlgorithm.comparator) for (schedulable <- sortedSchedulableQueue) { - sortedTaskSetQueue ++= schedulable.getSortedTaskSetQueue + sortedTaskSetQueue ++= schedulable.getSortedTaskSetQueue.filter(_.isSchedulable) } sortedTaskSetQueue } diff --git a/core/src/main/scala/org/apache/spark/scheduler/Schedulable.scala b/core/src/main/scala/org/apache/spark/scheduler/Schedulable.scala index 8cc239c81d11a..0626f8fb8150a 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Schedulable.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Schedulable.scala @@ -39,6 +39,7 @@ private[spark] trait Schedulable { def stageId: Int def name: String + def isSchedulable: Boolean def addSchedulable(schedulable: Schedulable): Unit def removeSchedulable(schedulable: Schedulable): Unit def getSchedulableByName(name: String): Schedulable diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala index a5bba645be14c..a566d0a04387c 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala @@ -83,7 +83,7 @@ private[spark] trait SchedulerBackend { /** * Get the max number of tasks that can be concurrent launched based on the ResourceProfile - * being used. + * could be used, even if some of them are being used at the moment. * Note that please don't cache the value returned by this method, because the number can change * due to add/remove executors. * diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapStage.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapStage.scala index be1984de9837f..db09d19d0acff 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapStage.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapStage.scala @@ -52,7 +52,7 @@ private[spark] class ShuffleMapStage( * Partitions that either haven't yet been computed, or that were computed on an executor * that has since been lost, so should be re-computed. This variable is used by the * DAGScheduler to determine when a stage has completed. Task successes in both the active - * attempt for the stage or in earlier attempts for this stage can cause paritition ids to get + * attempt for the stage or in earlier attempts for this stage can cause partition ids to get * removed from pendingPartitions. As a result, this variable may be inconsistent with the pending * tasks in the TaskSetManager for the active attempt for the stage (the partitions stored here * will always be a subset of the partitions that the TaskSetManager thinks are pending). diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index ebc1c05435fee..81f984bb2b511 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -23,6 +23,7 @@ import java.util.Properties import org.apache.spark._ import org.apache.spark.executor.TaskMetrics import org.apache.spark.internal.config.APP_CALLER_CONTEXT +import org.apache.spark.internal.plugin.PluginContainer import org.apache.spark.memory.{MemoryMode, TaskMemoryManager} import org.apache.spark.metrics.MetricsSystem import org.apache.spark.rdd.InputFileBlockHolder @@ -82,7 +83,8 @@ private[spark] abstract class Task[T]( taskAttemptId: Long, attemptNumber: Int, metricsSystem: MetricsSystem, - resources: Map[String, ResourceInformation]): T = { + resources: Map[String, ResourceInformation], + plugins: Option[PluginContainer]): T = { SparkEnv.get.blockManager.registerTask(taskAttemptId) // TODO SPARK-24874 Allow create BarrierTaskContext based on partitions, instead of whether // the stage is barrier. @@ -123,6 +125,8 @@ private[spark] abstract class Task[T]( Option(taskAttemptId), Option(attemptNumber)).setCurrentContext() + plugins.foreach(_.onTaskStart()) + try { runTask(context) } catch { diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala index 247cfe721b553..863bf27088355 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskDescription.scala @@ -59,7 +59,7 @@ private[spark] class TaskDescription( val resources: immutable.Map[String, ResourceInformation], val serializedTask: ByteBuffer) { - override def toString: String = "TaskDescription(TID=%d, index=%d)".format(taskId, index) + override def toString: String = s"TaskDescription($name)" } private[spark] object TaskDescription { diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala index 6c3d2a4ee3125..2dabee391310a 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala @@ -83,7 +83,7 @@ private[spark] class TaskResultGetter(sparkEnv: SparkEnv, scheduler: TaskSchedul "Tasks result size has exceeded maxResultSize")) return } - logDebug("Fetching indirect task result for TID %s".format(tid)) + logDebug(s"Fetching indirect task result for ${taskSetManager.taskName(tid)}") scheduler.handleTaskGettingResult(taskSetManager, tid) val serializedTaskResult = sparkEnv.blockManager.getRemoteBytes(blockId) if (serializedTaskResult.isEmpty) { diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskScheduler.scala index 1101d0616d2bf..0fa80bbafdedd 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskScheduler.scala @@ -106,7 +106,7 @@ private[spark] trait TaskScheduler { /** * If an executor is decommissioned, return its corresponding decommission info */ - def getExecutorDecommissionInfo(executorId: String): Option[ExecutorDecommissionInfo] + def getExecutorDecommissionState(executorId: String): Option[ExecutorDecommissionState] /** * Process a lost executor diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala index 2551e497a165a..2fcf13d5268f8 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala @@ -26,6 +26,9 @@ import scala.collection.mutable import scala.collection.mutable.{ArrayBuffer, Buffer, HashMap, HashSet} import scala.util.Random +import com.google.common.base.Ticker +import com.google.common.cache.CacheBuilder + import org.apache.spark._ import org.apache.spark.TaskState.TaskState import org.apache.spark.executor.ExecutorMetrics @@ -136,7 +139,9 @@ private[spark] class TaskSchedulerImpl( // IDs of the tasks running on each executor private val executorIdToRunningTaskIds = new HashMap[String, HashSet[Long]] - private val executorsPendingDecommission = new HashMap[String, ExecutorDecommissionInfo] + // We add executors here when we first get decommission notification for them. Executors can + // continue to run even after being asked to decommission, but they will eventually exit. + val executorsPendingDecommission = new HashMap[String, ExecutorDecommissionState] def runningTasksByExecutors: Map[String, Int] = synchronized { executorIdToRunningTaskIds.toMap.mapValues(_.size).toMap @@ -276,7 +281,7 @@ private[spark] class TaskSchedulerImpl( private[scheduler] def createTaskSetManager( taskSet: TaskSet, maxTaskFailures: Int): TaskSetManager = { - new TaskSetManager(this, taskSet, maxTaskFailures, blacklistTrackerOpt) + new TaskSetManager(this, taskSet, maxTaskFailures, blacklistTrackerOpt, clock) } override def cancelTasks(stageId: Int, interruptThread: Boolean): Unit = synchronized { @@ -468,51 +473,6 @@ private[spark] class TaskSchedulerImpl( Some(localTaskReqAssign.toMap) } - // Use the resource that the resourceProfile has as the limiting resource to calculate the - // total number of slots available based on the current offers. - private def calculateAvailableSlots( - resourceProfileIds: Array[Int], - availableCpus: Array[Int], - availableResources: Array[Map[String, Buffer[String]]], - taskSet: TaskSetManager): Int = { - val resourceProfile = sc.resourceProfileManager.resourceProfileFromId( - taskSet.taskSet.resourceProfileId) - val offersForResourceProfile = resourceProfileIds.zipWithIndex.filter { case (id, _) => - (id == resourceProfile.id) - } - val coresKnown = resourceProfile.isCoresLimitKnown - var limitingResource = resourceProfile.limitingResource(conf) - val taskCpus = ResourceProfile.getTaskCpusOrDefaultForProfile(resourceProfile, conf) - - offersForResourceProfile.map { case (o, index) => - val numTasksPerExecCores = availableCpus(index) / taskCpus - // if limiting resource is empty then we have no other resources, so it has to be CPU - if (limitingResource == ResourceProfile.CPUS || limitingResource.isEmpty) { - numTasksPerExecCores - } else { - val taskLimit = resourceProfile.taskResources.get(limitingResource).map(_.amount) - .getOrElse { - val errorMsg = "limitingResource returns from ResourceProfile " + - s"$resourceProfile doesn't actually contain that task resource!" - taskSet.abort(errorMsg) - throw new SparkException(errorMsg) - } - // available addresses already takes into account if there are fractional - // task resource requests - val availAddrs = availableResources(index).get(limitingResource).map(_.size).getOrElse(0) - val resourceLimit = (availAddrs / taskLimit).toInt - if (!coresKnown) { - // when executor cores config isn't set, we can't calculate the real limiting resource - // and number of tasks per executor ahead of time, so calculate it now based on what - // is available. - if (numTasksPerExecCores <= resourceLimit) numTasksPerExecCores else resourceLimit - } else { - resourceLimit - } - } - }.sum - } - private def minTaskLocality( l1: Option[TaskLocality], l2: Option[TaskLocality]) : Option[TaskLocality] = { @@ -575,7 +535,7 @@ private[spark] class TaskSchedulerImpl( val availableResources = shuffledOffers.map(_.resources).toArray val availableCpus = shuffledOffers.map(o => o.cores).toArray val resourceProfileIds = shuffledOffers.map(o => o.resourceProfileId).toArray - val sortedTaskSets = rootPool.getSortedTaskSetQueue.filterNot(_.isZombie) + val sortedTaskSets = rootPool.getSortedTaskSetQueue for (taskSet <- sortedTaskSets) { logDebug("parentName: %s, name: %s, runningTasks: %s".format( taskSet.parent.name, taskSet.name, taskSet.runningTasks)) @@ -591,9 +551,14 @@ private[spark] class TaskSchedulerImpl( // we only need to calculate available slots if using barrier scheduling, otherwise the // value is -1 val numBarrierSlotsAvailable = if (taskSet.isBarrier) { - val slots = calculateAvailableSlots(resourceProfileIds, availableCpus, availableResources, - taskSet) - slots + val rpId = taskSet.taskSet.resourceProfileId + val availableResourcesAmount = availableResources.map { resourceMap => + // available addresses already takes into account if there are fractional + // task resource requests + resourceMap.map { case (name, addresses) => (name, addresses.length) } + } + calculateAvailableSlots(this, conf, rpId, resourceProfileIds, availableCpus, + availableResourcesAmount) } else { -1 } @@ -945,40 +910,21 @@ private[spark] class TaskSchedulerImpl( synchronized { // Don't bother noting decommissioning for executors that we don't know about if (executorIdToHost.contains(executorId)) { - // The scheduler can get multiple decommission updates from multiple sources, - // and some of those can have isHostDecommissioned false. We merge them such that - // if we heard isHostDecommissioned ever true, then we keep that one since it is - // most likely coming from the cluster manager and thus authoritative - val oldDecomInfo = executorsPendingDecommission.get(executorId) - if (oldDecomInfo.isEmpty || !oldDecomInfo.get.isHostDecommissioned) { - executorsPendingDecommission(executorId) = decommissionInfo - } + executorsPendingDecommission(executorId) = + ExecutorDecommissionState(clock.getTimeMillis(), decommissionInfo.workerHost) } } rootPool.executorDecommission(executorId) backend.reviveOffers() } - override def getExecutorDecommissionInfo(executorId: String) - : Option[ExecutorDecommissionInfo] = synchronized { - executorsPendingDecommission.get(executorId) + override def getExecutorDecommissionState(executorId: String) + : Option[ExecutorDecommissionState] = synchronized { + executorsPendingDecommission.get(executorId) } - override def executorLost(executorId: String, givenReason: ExecutorLossReason): Unit = { + override def executorLost(executorId: String, reason: ExecutorLossReason): Unit = { var failedExecutor: Option[String] = None - val reason = givenReason match { - // Handle executor process loss due to decommissioning - case ExecutorProcessLost(message, origWorkerLost, origCausedByApp) => - val executorDecommissionInfo = getExecutorDecommissionInfo(executorId) - ExecutorProcessLost( - message, - // Also mark the worker lost if we know that the host was decommissioned - origWorkerLost || executorDecommissionInfo.exists(_.isHostDecommissioned), - // Executor loss is certainly not caused by app if we knew that this executor is being - // decommissioned - causedByApp = executorDecommissionInfo.isEmpty && origCausedByApp) - case e => e - } synchronized { if (executorIdToRunningTaskIds.contains(executorId)) { @@ -1067,7 +1013,7 @@ private[spark] class TaskSchedulerImpl( } } - executorsPendingDecommission -= executorId + executorsPendingDecommission.remove(executorId) if (reason != LossReasonPending) { executorIdToHost -= executorId @@ -1081,25 +1027,38 @@ private[spark] class TaskSchedulerImpl( } def getExecutorsAliveOnHost(host: String): Option[Set[String]] = synchronized { - hostToExecutors.get(host).map(_.toSet) + hostToExecutors.get(host).map(_.filterNot(isExecutorDecommissioned)).map(_.toSet) } def hasExecutorsAliveOnHost(host: String): Boolean = synchronized { - hostToExecutors.contains(host) + hostToExecutors.get(host) + .exists(executors => executors.exists(e => !isExecutorDecommissioned(e))) } def hasHostAliveOnRack(rack: String): Boolean = synchronized { - hostsByRack.contains(rack) + hostsByRack.get(rack) + .exists(hosts => hosts.exists(h => !isHostDecommissioned(h))) } def isExecutorAlive(execId: String): Boolean = synchronized { - executorIdToRunningTaskIds.contains(execId) + executorIdToRunningTaskIds.contains(execId) && !isExecutorDecommissioned(execId) } def isExecutorBusy(execId: String): Boolean = synchronized { executorIdToRunningTaskIds.get(execId).exists(_.nonEmpty) } + // exposed for test + protected final def isExecutorDecommissioned(execId: String): Boolean = + getExecutorDecommissionState(execId).isDefined + + // exposed for test + protected final def isHostDecommissioned(host: String): Boolean = { + hostToExecutors.get(host).exists { executors => + executors.exists(e => getExecutorDecommissionState(e).exists(_.workerHost.isDefined)) + } + } + /** * Get a snapshot of the currently blacklisted nodes for the entire application. This is * thread-safe -- it can be called without a lock on the TaskScheduler. @@ -1166,6 +1125,63 @@ private[spark] object TaskSchedulerImpl { val SCHEDULER_MODE_PROPERTY = SCHEDULER_MODE.key + /** + * Calculate the max available task slots given the `availableCpus` and `availableResources` + * from a collection of ResourceProfiles. And only those ResourceProfiles who has the + * same id with the `rpId` can be used to calculate the task slots. + * + * @param scheduler the TaskSchedulerImpl instance + * @param conf SparkConf used to calculate the limiting resource and get the cpu amount per task + * @param rpId the target ResourceProfile id. Only those ResourceProfiles who has the same id + * with it can be used to calculate the task slots. + * @param availableRPIds an Array of ids of the available ResourceProfiles from the executors. + * @param availableCpus an Array of the amount of available cpus from the executors. + * @param availableResources an Array of the resources map from the executors. In the resource + * map, it maps from the resource name to its amount. + * @return the number of max task slots + */ + def calculateAvailableSlots( + scheduler: TaskSchedulerImpl, + conf: SparkConf, + rpId: Int, + availableRPIds: Array[Int], + availableCpus: Array[Int], + availableResources: Array[Map[String, Int]]): Int = { + val resourceProfile = scheduler.sc.resourceProfileManager.resourceProfileFromId(rpId) + val coresKnown = resourceProfile.isCoresLimitKnown + val (limitingResource, limitedByCpu) = { + val limiting = resourceProfile.limitingResource(conf) + // if limiting resource is empty then we have no other resources, so it has to be CPU + if (limiting == ResourceProfile.CPUS || limiting.isEmpty) { + (ResourceProfile.CPUS, true) + } else { + (limiting, false) + } + } + val cpusPerTask = ResourceProfile.getTaskCpusOrDefaultForProfile(resourceProfile, conf) + val taskLimit = resourceProfile.taskResources.get(limitingResource).map(_.amount).get + + availableCpus.zip(availableResources).zip(availableRPIds) + .filter { case (_, id) => id == rpId } + .map { case ((cpu, resources), _) => + val numTasksPerExecCores = cpu / cpusPerTask + if (limitedByCpu) { + numTasksPerExecCores + } else { + val availAddrs = resources.getOrElse(limitingResource, 0) + val resourceLimit = (availAddrs / taskLimit).toInt + // when executor cores config isn't set, we can't calculate the real limiting resource + // and number of tasks per executor ahead of time, so calculate it now based on what + // is available. + if (!coresKnown && numTasksPerExecCores <= resourceLimit) { + numTasksPerExecCores + } else { + resourceLimit + } + } + }.sum + } + /** * Used to balance containers across hosts. * diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala index d69f358cd19de..78fd412ef154c 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala @@ -71,6 +71,7 @@ private[spark] class TaskSetManager( val ser = env.closureSerializer.newInstance() val tasks = taskSet.tasks + private val isShuffleMapTasks = tasks(0).isInstanceOf[ShuffleMapTask] private[scheduler] val partitionToIndex = tasks.zipWithIndex .map { case (t, idx) => t.partitionId -> idx }.toMap val numTasks = tasks.length @@ -102,8 +103,9 @@ private[spark] class TaskSetManager( } numTasks <= slots } - val executorDecommissionKillInterval = conf.get(EXECUTOR_DECOMMISSION_KILL_INTERVAL).map( - TimeUnit.SECONDS.toMillis) + + private val executorDecommissionKillInterval = + conf.get(EXECUTOR_DECOMMISSION_KILL_INTERVAL).map(TimeUnit.SECONDS.toMillis) // For each task, tracks whether a copy of the task has succeeded. A task will also be // marked as "succeeded" if it failed with a fetch failure, in which case it should not @@ -167,7 +169,6 @@ private[spark] class TaskSetManager( // Task index, start and finish time for each task attempt (indexed by task ID) private[scheduler] val taskInfos = new HashMap[Long, TaskInfo] - private[scheduler] val tidToExecutorKillTimeMapping = new HashMap[Long, Long] // Use a MedianHeap to record durations of successful tasks so we know when to launch // speculative tasks. This is only used when speculation is enabled, to avoid the overhead @@ -477,8 +478,8 @@ private[spark] class TaskSetManager( // We used to log the time it takes to serialize the task, but task size is already // a good proxy to task serialization time. // val timeTaken = clock.getTime() - startTime - val taskName = s"task ${info.id} in stage ${taskSet.id}" - logInfo(s"Starting $taskName (TID $taskId, $host, executor ${info.executorId}, " + + val tName = taskName(taskId) + logInfo(s"Starting $tName ($host, executor ${info.executorId}, " + s"partition ${task.partitionId}, $taskLocality, ${serializedTask.limit()} bytes) " + s"taskResourceAssignments ${taskResourceAssignments}") @@ -487,7 +488,7 @@ private[spark] class TaskSetManager( taskId, attemptNum, execId, - taskName, + tName, index, task.partitionId, addedFiles, @@ -507,6 +508,12 @@ private[spark] class TaskSetManager( } } + def taskName(tid: Long): String = { + val info = taskInfos.get(tid) + assert(info.isDefined, s"Can not find TaskInfo for task (TID $tid)") + s"task ${info.get.id} in stage ${taskSet.id} (TID $tid)" + } + private def maybeFinishTaskSet(): Unit = { if (isZombie && runningTasks == 0) { sched.taskSetFinished(this) @@ -690,12 +697,14 @@ private[spark] class TaskSetManager( } /** - * Check whether has enough quota to fetch the result with `size` bytes + * Check whether has enough quota to fetch the result with `size` bytes. + * This check does not apply to shuffle map tasks as they return map status and metrics updates, + * which will be discarded by the driver after being processed. */ def canFetchMoreResults(size: Long): Boolean = sched.synchronized { totalResultSize += size calculatedTasks += 1 - if (maxResultSize > 0 && totalResultSize > maxResultSize) { + if (!isShuffleMapTasks && maxResultSize > 0 && totalResultSize > maxResultSize) { val msg = s"Total size of serialized results of ${calculatedTasks} tasks " + s"(${Utils.bytesToString(totalResultSize)}) is bigger than ${config.MAX_RESULT_SIZE.key} " + s"(${Utils.bytesToString(maxResultSize)})" @@ -739,9 +748,8 @@ private[spark] class TaskSetManager( // Kill any other attempts for the same task (since those are unnecessary now that one // attempt completed successfully). for (attemptInfo <- taskAttempts(index) if attemptInfo.running) { - logInfo(s"Killing attempt ${attemptInfo.attemptNumber} for task ${attemptInfo.id} " + - s"in stage ${taskSet.id} (TID ${attemptInfo.taskId}) on ${attemptInfo.host} " + - s"as the attempt ${info.attemptNumber} succeeded on ${info.host}") + logInfo(s"Killing attempt ${attemptInfo.attemptNumber} for ${taskName(attemptInfo.taskId)}" + + s" on ${attemptInfo.host} as the attempt ${info.attemptNumber} succeeded on ${info.host}") killedByOtherAttempt += attemptInfo.taskId sched.backend.killTask( attemptInfo.taskId, @@ -751,17 +759,16 @@ private[spark] class TaskSetManager( } if (!successful(index)) { tasksSuccessful += 1 - logInfo(s"Finished task ${info.id} in stage ${taskSet.id} (TID ${info.taskId}) in" + - s" ${info.duration} ms on ${info.host} (executor ${info.executorId})" + - s" ($tasksSuccessful/$numTasks)") + logInfo(s"Finished ${taskName(info.taskId)} in ${info.duration} ms " + + s"on ${info.host} (executor ${info.executorId}) ($tasksSuccessful/$numTasks)") // Mark successful and stop if all the tasks have succeeded. successful(index) = true if (tasksSuccessful == numTasks) { isZombie = true } } else { - logInfo("Ignoring task-finished event for " + info.id + " in stage " + taskSet.id + - " because task " + index + " has already completed successfully") + logInfo(s"Ignoring task-finished event for ${taskName(info.taskId)} " + + s"because it has already completed successfully") } // This method is called by "TaskSchedulerImpl.handleSuccessfulTask" which holds the // "TaskSchedulerImpl" lock until exiting. To avoid the SPARK-7655 issue, we should not @@ -802,8 +809,8 @@ private[spark] class TaskSetManager( copiesRunning(index) -= 1 var accumUpdates: Seq[AccumulatorV2[_, _]] = Seq.empty var metricPeaks: Array[Long] = Array.empty - val failureReason = s"Lost task ${info.id} in stage ${taskSet.id} (TID $tid, ${info.host}," + - s" executor ${info.executorId}): ${reason.toErrorString}" + val failureReason = s"Lost ${taskName(tid)} (${info.host} " + + s"executor ${info.executorId}): ${reason.toErrorString}" val failureException: Option[Throwable] = reason match { case fetchFailed: FetchFailed => logWarning(failureReason) @@ -824,12 +831,11 @@ private[spark] class TaskSetManager( // ExceptionFailure's might have accumulator updates accumUpdates = ef.accums metricPeaks = ef.metricPeaks.toArray + val task = taskName(tid) if (ef.className == classOf[NotSerializableException].getName) { // If the task result wasn't serializable, there's no point in trying to re-execute it. - logError("Task %s in stage %s (TID %d) had a not serializable result: %s; not retrying" - .format(info.id, taskSet.id, tid, ef.description)) - abort("Task %s in stage %s (TID %d) had a not serializable result: %s".format( - info.id, taskSet.id, tid, ef.description)) + logError(s"$task had a not serializable result: ${ef.description}; not retrying") + abort(s"$task had a not serializable result: ${ef.description}") return } if (ef.className == classOf[TaskOutputFileAlreadyExistException].getName) { @@ -862,8 +868,8 @@ private[spark] class TaskSetManager( logWarning(failureReason) } else { logInfo( - s"Lost task ${info.id} in stage ${taskSet.id} (TID $tid) on ${info.host}, executor" + - s" ${info.executorId}: ${ef.className} (${ef.description}) [duplicate $dupCount]") + s"Lost $task on ${info.host}, executor ${info.executorId}: " + + s"${ef.className} (${ef.description}) [duplicate $dupCount]") } ef.exception @@ -875,7 +881,7 @@ private[spark] class TaskSetManager( None case e: ExecutorLostFailure if !e.exitCausedByApp => - logInfo(s"Task $tid failed because while it was being computed, its executor " + + logInfo(s"${taskName(tid)} failed because while it was being computed, its executor " + "exited for a reason unrelated to the task. Not counting this failure towards the " + "maximum number of failures for the task.") None @@ -906,10 +912,10 @@ private[spark] class TaskSetManager( } if (successful(index)) { - logInfo(s"Task ${info.id} in stage ${taskSet.id} (TID $tid) failed, but the task will not" + - s" be re-executed (either because the task failed with a shuffle data fetch failure," + - s" so the previous stage needs to be re-run, or because a different copy of the task" + - s" has already succeeded).") + logInfo(s"${taskName(info.taskId)} failed, but the task will not" + + " be re-executed (either because the task failed with a shuffle data fetch failure," + + " so the previous stage needs to be re-run, or because a different copy of the task" + + " has already succeeded).") } else { addPendingTask(index) } @@ -936,7 +942,6 @@ private[spark] class TaskSetManager( /** If the given task ID is in the set of running tasks, removes it. */ def removeRunningTask(tid: Long): Unit = { - tidToExecutorKillTimeMapping.remove(tid) if (runningTasksSet.remove(tid) && parent != null) { parent.decreaseRunningTasks(1) } @@ -946,6 +951,9 @@ private[spark] class TaskSetManager( null } + override def isSchedulable: Boolean = !isZombie && + (pendingTasks.all.nonEmpty || pendingSpeculatableTasks.all.nonEmpty) + override def addSchedulable(schedulable: Schedulable): Unit = {} override def removeSchedulable(schedulable: Schedulable): Unit = {} @@ -962,8 +970,7 @@ private[spark] class TaskSetManager( // and we are not using an external shuffle server which could serve the shuffle outputs. // The reason is the next stage wouldn't be able to fetch the data from this dead executor // so we would need to rerun these tasks on other executors. - if (tasks(0).isInstanceOf[ShuffleMapTask] && !env.blockManager.externalShuffleServiceEnabled - && !isZombie) { + if (isShuffleMapTasks && !env.blockManager.externalShuffleServiceEnabled && !isZombie) { for ((tid, info) <- taskInfos if info.executorId == execId) { val index = taskInfos(tid).index // We may have a running task whose partition has been marked as successful, @@ -984,7 +991,7 @@ private[spark] class TaskSetManager( for ((tid, info) <- taskInfos if info.running && info.executorId == execId) { val exitCausedByApp: Boolean = reason match { case exited: ExecutorExited => exited.exitCausedByApp - case ExecutorKilled => false + case ExecutorKilled | ExecutorDecommission(_) => false case ExecutorProcessLost(_, _, false) => false case _ => true } @@ -1048,15 +1055,21 @@ private[spark] class TaskSetManager( logDebug("Task length threshold for speculation: " + threshold) for (tid <- runningTasksSet) { var speculated = checkAndSubmitSpeculatableTask(tid, time, threshold) - if (!speculated && tidToExecutorKillTimeMapping.contains(tid)) { - // Check whether this task will finish before the exectorKillTime assuming - // it will take medianDuration overall. If this task cannot finish within - // executorKillInterval, then this task is a candidate for speculation - val taskEndTimeBasedOnMedianDuration = taskInfos(tid).launchTime + medianDuration - val canExceedDeadline = tidToExecutorKillTimeMapping(tid) < - taskEndTimeBasedOnMedianDuration - if (canExceedDeadline) { - speculated = checkAndSubmitSpeculatableTask(tid, time, 0) + if (!speculated && executorDecommissionKillInterval.isDefined) { + val taskInfo = taskInfos(tid) + val decomState = sched.getExecutorDecommissionState(taskInfo.executorId) + if (decomState.isDefined) { + // Check if this task might finish after this executor is decommissioned. + // We estimate the task's finish time by using the median task duration. + // Whereas the time when the executor might be decommissioned is estimated using the + // config executorDecommissionKillInterval. If the task is going to finish after + // decommissioning, then we will eagerly speculate the task. + val taskEndTimeBasedOnMedianDuration = taskInfos(tid).launchTime + medianDuration + val executorDecomTime = decomState.get.startTime + executorDecommissionKillInterval.get + val canExceedDeadline = executorDecomTime < taskEndTimeBasedOnMedianDuration + if (canExceedDeadline) { + speculated = checkAndSubmitSpeculatableTask(tid, time, 0) + } } } foundTasks |= speculated @@ -1117,12 +1130,6 @@ private[spark] class TaskSetManager( def executorDecommission(execId: String): Unit = { recomputeLocality() - executorDecommissionKillInterval.foreach { interval => - val executorKillTime = clock.getTimeMillis() + interval - runningTasksSet.filter(taskInfos(_).executorId == execId).foreach { tid => - tidToExecutorKillTimeMapping(tid) = executorKillTime - } - } } def recomputeLocality(): Unit = { diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala index 91485f01bf007..d1b0e798c51be 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedClusterMessage.scala @@ -95,8 +95,17 @@ private[spark] object CoarseGrainedClusterMessages { case class RemoveExecutor(executorId: String, reason: ExecutorLossReason) extends CoarseGrainedClusterMessage - case class DecommissionExecutor(executorId: String, decommissionInfo: ExecutorDecommissionInfo) - extends CoarseGrainedClusterMessage + // A message that sent from executor to driver to tell driver that the executor has started + // decommissioning. It's used for the case where decommission is triggered at executor (e.g., K8S) + case class ExecutorDecommissioning(executorId: String) extends CoarseGrainedClusterMessage + + // A message that sent from driver to executor to decommission that executor. + // It's used for Standalone's cases, where decommission is triggered at MasterWebUI or Worker. + object DecommissionExecutor extends CoarseGrainedClusterMessage + + // A message that sent to the executor itself when it receives PWR signal, + // indicating the executor starts to decommission. + object ExecutorSigPWRReceived extends CoarseGrainedClusterMessage case class RemoveWorker(workerId: String, host: String, message: String) extends CoarseGrainedClusterMessage diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala index 8fbefae58af14..1d2689034f1ff 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala @@ -92,8 +92,8 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp // Executors that have been lost, but for which we don't yet know the real exit reason. private val executorsPendingLossReason = new HashSet[String] - // Executors which are being decommissioned - protected val executorsPendingDecommission = new HashSet[String] + // Executors which are being decommissioned. Maps from executorId to workerHost. + protected val executorsPendingDecommission = new HashMap[String, Option[String]] // A map of ResourceProfile id to map of hostname with its possible task number running on it @GuardedBy("CoarseGrainedSchedulerBackend.this") @@ -191,10 +191,6 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp executorDataMap.get(executorId).foreach(_.executorEndpoint.send(StopExecutor)) removeExecutor(executorId, reason) - case DecommissionExecutor(executorId, decommissionInfo) => - logError(s"Received decommission executor message ${executorId}: $decommissionInfo") - decommissionExecutor(executorId, decommissionInfo) - case RemoveWorker(workerId, host, message) => removeWorker(workerId, host, message) @@ -272,10 +268,15 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp removeWorker(workerId, host, message) context.reply(true) - case DecommissionExecutor(executorId, decommissionInfo) => - logError(s"Received decommission executor message ${executorId}: ${decommissionInfo}.") - decommissionExecutor(executorId, decommissionInfo) - context.reply(true) + // Do not change this code without running the K8s integration suites + case ExecutorDecommissioning(executorId) => + logWarning(s"Received executor $executorId decommissioned message") + context.reply( + decommissionExecutor( + executorId, + ExecutorDecommissionInfo(s"Executor $executorId is decommissioned."), + adjustTargetNumExecutors = false, + triggeredByExecutor = true)) case RetrieveSparkAppConfig(resourceProfileId) => val rp = scheduler.sc.resourceProfileManager.resourceProfileFromId(resourceProfileId) @@ -390,16 +391,23 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp case Some(executorInfo) => // This must be synchronized because variables mutated // in this block are read when requesting executors - val killed = CoarseGrainedSchedulerBackend.this.synchronized { + val lossReason = CoarseGrainedSchedulerBackend.this.synchronized { addressToExecutorId -= executorInfo.executorAddress executorDataMap -= executorId executorsPendingLossReason -= executorId - executorsPendingDecommission -= executorId - executorsPendingToRemove.remove(executorId).getOrElse(false) + val killedByDriver = executorsPendingToRemove.remove(executorId).getOrElse(false) + val workerHostOpt = executorsPendingDecommission.remove(executorId) + if (killedByDriver) { + ExecutorKilled + } else if (workerHostOpt.isDefined) { + ExecutorDecommission(workerHostOpt.get) + } else { + reason + } } totalCoreCount.addAndGet(-executorInfo.totalCores) totalRegisteredExecutors.addAndGet(-1) - scheduler.executorLost(executorId, if (killed) ExecutorKilled else reason) + scheduler.executorLost(executorId, lossReason) listenerBus.post( SparkListenerExecutorRemoved(System.currentTimeMillis(), executorId, reason.toString)) case None => @@ -419,49 +427,6 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp scheduler.workerRemoved(workerId, host, message) } - /** - * Mark a given executor as decommissioned and stop making resource offers for it. - */ - private def decommissionExecutor( - executorId: String, decommissionInfo: ExecutorDecommissionInfo): Boolean = { - val shouldDisable = CoarseGrainedSchedulerBackend.this.synchronized { - // Only bother decommissioning executors which are alive. - if (isExecutorActive(executorId)) { - executorsPendingDecommission += executorId - true - } else { - false - } - } - - if (shouldDisable) { - logInfo(s"Starting decommissioning executor $executorId.") - try { - scheduler.executorDecommission(executorId, decommissionInfo) - } catch { - case e: Exception => - logError(s"Unexpected error during decommissioning ${e.toString}", e) - } - logInfo(s"Finished decommissioning executor $executorId.") - - if (conf.get(STORAGE_DECOMMISSION_ENABLED)) { - try { - logInfo("Starting decommissioning block manager corresponding to " + - s"executor $executorId.") - scheduler.sc.env.blockManager.master.decommissionBlockManagers(Seq(executorId)) - } catch { - case e: Exception => - logError("Unexpected error during block manager " + - s"decommissioning for executor $executorId: ${e.toString}", e) - } - logInfo(s"Acknowledged decommissioning block manager corresponding to $executorId.") - } - } else { - logInfo(s"Skipping decommissioning of executor $executorId.") - } - shouldDisable - } - /** * Stop making resource offers for the given executor. The executor is marked as lost with * the loss reason still pending. @@ -493,6 +458,56 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp protected def minRegisteredRatio: Double = _minRegisteredRatio + /** + * Request that the cluster manager decommission the specified executors. + * + * @param executorsAndDecomInfo Identifiers of executors & decommission info. + * @param adjustTargetNumExecutors whether the target number of executors will be adjusted down + * after these executors have been decommissioned. + * @param triggeredByExecutor whether the decommission is triggered at executor. + * @return the ids of the executors acknowledged by the cluster manager to be removed. + */ + override def decommissionExecutors( + executorsAndDecomInfo: Array[(String, ExecutorDecommissionInfo)], + adjustTargetNumExecutors: Boolean, + triggeredByExecutor: Boolean): Seq[String] = withLock { + // Do not change this code without running the K8s integration suites + val executorsToDecommission = executorsAndDecomInfo.flatMap { case (executorId, decomInfo) => + // Only bother decommissioning executors which are alive. + if (isExecutorActive(executorId)) { + scheduler.executorDecommission(executorId, decomInfo) + executorsPendingDecommission(executorId) = decomInfo.workerHost + Some(executorId) + } else { + None + } + } + logInfo(s"Decommission executors: ${executorsToDecommission.mkString(", ")}") + + // If we don't want to replace the executors we are decommissioning + if (adjustTargetNumExecutors) { + adjustExecutors(executorsToDecommission) + } + + // Mark those corresponding BlockManagers as decommissioned first before we sending + // decommission notification to executors. So, it's less likely to lead to the race + // condition where `getPeer` request from the decommissioned executor comes first + // before the BlockManagers are marked as decommissioned. + // Note that marking BlockManager as decommissioned doesn't need depend on + // `spark.storage.decommission.enabled`. Because it's meaningless to save more blocks + // for the BlockManager since the executor will be shutdown soon. + scheduler.sc.env.blockManager.master.decommissionBlockManagers(executorsToDecommission) + + if (!triggeredByExecutor) { + executorsToDecommission.foreach { executorId => + logInfo(s"Notify executor $executorId to decommissioning.") + executorDataMap(executorId).executorEndpoint.send(DecommissionExecutor) + } + } + + executorsToDecommission + } + override def start(): Unit = { if (UserGroupInformation.isSecurityEnabled()) { delegationTokenManager = createTokenManager() @@ -588,17 +603,6 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp driverEndpoint.send(RemoveWorker(workerId, host, message)) } - /** - * Called by subclasses when notified of a decommissioning executor. - */ - private[spark] def decommissionExecutor( - executorId: String, decommissionInfo: ExecutorDecommissionInfo): Unit = { - if (driverEndpoint != null) { - logInfo("Propagating executor decommission to driver.") - driverEndpoint.send(DecommissionExecutor(executorId, decommissionInfo)) - } - } - def sufficientResourcesRegistered(): Boolean = true override def isReady(): Boolean = { @@ -629,13 +633,30 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp !executorsPendingToRemove.contains(id) && !executorsPendingLossReason.contains(id) && !executorsPendingDecommission.contains(id) - } + /** + * Get the max number of tasks that can be concurrent launched based on the ResourceProfile + * could be used, even if some of them are being used at the moment. + * Note that please don't cache the value returned by this method, because the number can change + * due to add/remove executors. + * + * @param rp ResourceProfile which to use to calculate max concurrent tasks. + * @return The max number of tasks that can be concurrent launched currently. + */ override def maxNumConcurrentTasks(rp: ResourceProfile): Int = synchronized { - val cpusPerTask = ResourceProfile.getTaskCpusOrDefaultForProfile(rp, conf) - val executorsWithResourceProfile = executorDataMap.values.filter(_.resourceProfileId == rp.id) - executorsWithResourceProfile.map(_.totalCores / cpusPerTask).sum + val (rpIds, cpus, resources) = { + executorDataMap + .filter { case (id, _) => isExecutorActive(id) } + .values.toArray.map { executor => + ( + executor.resourceProfileId, + executor.totalCores, + executor.resourcesInfo.map { case (name, rInfo) => (name, rInfo.totalAddressAmount) } + ) + }.unzip3 + } + TaskSchedulerImpl.calculateAvailableSlots(scheduler, conf, rp.id, rpIds, cpus, resources) } // this function is for testing only @@ -732,6 +753,31 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp resourceProfileToTotalExecs: Map[ResourceProfile, Int]): Future[Boolean] = Future.successful(false) + /** + * Adjust the number of executors being requested to no longer include the provided executors. + */ + private def adjustExecutors(executorIds: Seq[String]) = { + if (executorIds.nonEmpty) { + executorIds.foreach { exec => + withLock { + val rpId = executorDataMap(exec).resourceProfileId + val rp = scheduler.sc.resourceProfileManager.resourceProfileFromId(rpId) + if (requestedTotalExecutorsPerResourceProfile.isEmpty) { + // Assume that we are killing an executor that was started by default and + // not through the request api + requestedTotalExecutorsPerResourceProfile(rp) = 0 + } else { + val requestedTotalForRp = requestedTotalExecutorsPerResourceProfile(rp) + requestedTotalExecutorsPerResourceProfile(rp) = math.max(requestedTotalForRp - 1, 0) + } + } + } + doRequestTotalExecutors(requestedTotalExecutorsPerResourceProfile.toMap) + } else { + Future.successful(true) + } + } + /** * Request that the cluster manager kill the specified executors. * @@ -770,19 +816,7 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp // take into account executors that are pending to be added or removed. val adjustTotalExecutors = if (adjustTargetNumExecutors) { - executorsToKill.foreach { exec => - val rpId = executorDataMap(exec).resourceProfileId - val rp = scheduler.sc.resourceProfileManager.resourceProfileFromId(rpId) - if (requestedTotalExecutorsPerResourceProfile.isEmpty) { - // Assume that we are killing an executor that was started by default and - // not through the request api - requestedTotalExecutorsPerResourceProfile(rp) = 0 - } else { - val requestedTotalForRp = requestedTotalExecutorsPerResourceProfile(rp) - requestedTotalExecutorsPerResourceProfile(rp) = math.max(requestedTotalForRp - 1, 0) - } - } - doRequestTotalExecutors(requestedTotalExecutorsPerResourceProfile.toMap) + adjustExecutors(executorsToKill) } else { Future.successful(true) } diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala index d921af602b254..b9ac8d2ba2784 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala @@ -165,18 +165,25 @@ private[spark] class StandaloneSchedulerBackend( } override def executorRemoved( - fullId: String, message: String, exitStatus: Option[Int], workerLost: Boolean): Unit = { + fullId: String, + message: String, + exitStatus: Option[Int], + workerHost: Option[String]): Unit = { val reason: ExecutorLossReason = exitStatus match { case Some(code) => ExecutorExited(code, exitCausedByApp = true, message) - case None => ExecutorProcessLost(message, workerLost = workerLost) + case None => ExecutorProcessLost(message, workerHost) } logInfo("Executor %s removed: %s".format(fullId, message)) removeExecutor(fullId.split("/")(1), reason) } override def executorDecommissioned(fullId: String, decommissionInfo: ExecutorDecommissionInfo) { - logInfo("Asked to decommission executor") - decommissionExecutor(fullId.split("/")(1), decommissionInfo) + logInfo(s"Asked to decommission executor $fullId") + val execId = fullId.split("/")(1) + decommissionExecutors( + Array((execId, decommissionInfo)), + adjustTargetNumExecutors = false, + triggeredByExecutor = false) logInfo("Executor %s decommissioned: %s".format(fullId, decommissionInfo)) } diff --git a/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala b/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala index 4d7190726f067..8dbdc84905e36 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala @@ -28,7 +28,7 @@ import org.apache.spark.internal.Logging import org.apache.spark.internal.config._ import org.apache.spark.resource.ResourceProfile.UNKNOWN_RESOURCE_PROFILE_ID import org.apache.spark.scheduler._ -import org.apache.spark.storage.RDDBlockId +import org.apache.spark.storage.{RDDBlockId, ShuffleDataBlockId} import org.apache.spark.util.Clock /** @@ -114,7 +114,8 @@ private[spark] class ExecutorMonitor( var newNextTimeout = Long.MaxValue timedOutExecs = executors.asScala - .filter { case (_, exec) => !exec.pendingRemoval && !exec.hasActiveShuffle } + .filter { case (_, exec) => + !exec.pendingRemoval && !exec.hasActiveShuffle && !exec.decommissioning} .filter { case (_, exec) => val deadline = exec.timeoutAt if (deadline > now) { @@ -135,6 +136,7 @@ private[spark] class ExecutorMonitor( /** * Mark the given executors as pending to be removed. Should only be called in the EAM thread. + * This covers both kills and decommissions. */ def executorsKilled(ids: Seq[String]): Unit = { ids.foreach { id => @@ -149,6 +151,19 @@ private[spark] class ExecutorMonitor( nextTimeout.set(Long.MinValue) } + private[spark] def executorsDecommissioned(ids: Seq[String]): Unit = { + ids.foreach { id => + val tracker = executors.get(id) + if (tracker != null) { + tracker.decommissioning = true + } + } + + // Recompute timed out executors in the next EAM callback, since this call invalidates + // the current list. + nextTimeout.set(Long.MinValue) + } + def executorCount: Int = executors.size() def executorCountWithResourceProfile(id: Int): Int = { @@ -171,6 +186,16 @@ private[spark] class ExecutorMonitor( executors.asScala.filter { case (k, v) => v.resourceProfileId == id && v.pendingRemoval }.size } + def decommissioningCount: Int = executors.asScala.count { case (_, exec) => + exec.decommissioning + } + + def decommissioningPerResourceProfileId(id: Int): Int = { + executors.asScala.filter { case (k, v) => + v.resourceProfileId == id && v.decommissioning + }.size + } + override def onJobStart(event: SparkListenerJobStart): Unit = { if (!shuffleTrackingEnabled) { return @@ -298,6 +323,7 @@ private[spark] class ExecutorMonitor( // // This means that an executor may be marked as having shuffle data, and thus prevented // from being removed, even though the data may not be used. + // TODO: Only track used files (SPARK-31974) if (shuffleTrackingEnabled && event.reason == Success) { stageToShuffleID.get(event.stageId).foreach { shuffleId => exec.addShuffle(shuffleId) @@ -326,18 +352,35 @@ private[spark] class ExecutorMonitor( val removed = executors.remove(event.executorId) if (removed != null) { decrementExecResourceProfileCount(removed.resourceProfileId) - if (!removed.pendingRemoval) { + if (!removed.pendingRemoval || !removed.decommissioning) { nextTimeout.set(Long.MinValue) } } } override def onBlockUpdated(event: SparkListenerBlockUpdated): Unit = { + val exec = ensureExecutorIsTracked(event.blockUpdatedInfo.blockManagerId.executorId, + UNKNOWN_RESOURCE_PROFILE_ID) + + // Check if it is a shuffle file, or RDD to pick the correct codepath for update if (!event.blockUpdatedInfo.blockId.isInstanceOf[RDDBlockId]) { + if (event.blockUpdatedInfo.blockId.isInstanceOf[ShuffleDataBlockId] && + shuffleTrackingEnabled) { + /** + * The executor monitor keeps track of locations of cache and shuffle blocks and this can + * be used to decide which executor(s) Spark should shutdown first. Since we move shuffle + * blocks around now this wires it up so that it keeps track of it. We only do this for + * data blocks as index and other blocks blocks do not necessarily mean the entire block + * has been committed. + */ + event.blockUpdatedInfo.blockId match { + case ShuffleDataBlockId(shuffleId, _, _) => exec.addShuffle(shuffleId) + case _ => // For now we only update on data blocks + } + } return } - val exec = ensureExecutorIsTracked(event.blockUpdatedInfo.blockManagerId.executorId, - UNKNOWN_RESOURCE_PROFILE_ID) + val storageLevel = event.blockUpdatedInfo.storageLevel val blockId = event.blockUpdatedInfo.blockId.asInstanceOf[RDDBlockId] @@ -410,10 +453,15 @@ private[spark] class ExecutorMonitor( } // Visible for testing - def executorsPendingToRemove(): Set[String] = { + private[spark] def executorsPendingToRemove(): Set[String] = { executors.asScala.filter { case (_, exec) => exec.pendingRemoval }.keys.toSet } + // Visible for testing + private[spark] def executorsDecommissioning(): Set[String] = { + executors.asScala.filter { case (_, exec) => exec.decommissioning }.keys.toSet + } + /** * This method should be used when updating executor state. It guards against a race condition in * which the `SparkListenerTaskStart` event is posted before the `SparkListenerBlockManagerAdded` @@ -466,6 +514,7 @@ private[spark] class ExecutorMonitor( @volatile var timedOut: Boolean = false var pendingRemoval: Boolean = false + var decommissioning: Boolean = false var hasActiveShuffle: Boolean = false private var idleStart: Long = -1 diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdate.scala b/core/src/main/scala/org/apache/spark/security/SecurityConfigurationLock.scala similarity index 70% rename from sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdate.scala rename to core/src/main/scala/org/apache/spark/security/SecurityConfigurationLock.scala index 32be74a345c5a..0741a8c1580df 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdate.scala +++ b/core/src/main/scala/org/apache/spark/security/SecurityConfigurationLock.scala @@ -15,12 +15,10 @@ * limitations under the License. */ -package org.apache.spark.sql.internal.connector +package org.apache.spark.security -import org.apache.spark.sql.connector.write.WriteBuilder - -// An internal `WriteBuilder` mixin to support UPDATE streaming output mode. -// TODO: design an official API for streaming output mode UPDATE. -trait SupportsStreamingUpdate extends WriteBuilder { - def update(): WriteBuilder -} +/** + * There are cases when global JVM security configuration must be modified. + * In order to avoid race the modification must be synchronized with this. + */ +object SecurityConfigurationLock diff --git a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala index 0d0dad6d77ac1..e5df27c0d3c7a 100644 --- a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala +++ b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala @@ -225,19 +225,37 @@ private[spark] class IndexShuffleBlockResolver( * Get the index & data block for migration. */ def getMigrationBlocks(shuffleBlockInfo: ShuffleBlockInfo): List[(BlockId, ManagedBuffer)] = { - val shuffleId = shuffleBlockInfo.shuffleId - val mapId = shuffleBlockInfo.mapId - // Load the index block - val indexFile = getIndexFile(shuffleId, mapId) - val indexBlockId = ShuffleIndexBlockId(shuffleId, mapId, NOOP_REDUCE_ID) - val indexFileSize = indexFile.length() - val indexBlockData = new FileSegmentManagedBuffer(transportConf, indexFile, 0, indexFileSize) + try { + val shuffleId = shuffleBlockInfo.shuffleId + val mapId = shuffleBlockInfo.mapId + // Load the index block + val indexFile = getIndexFile(shuffleId, mapId) + val indexBlockId = ShuffleIndexBlockId(shuffleId, mapId, NOOP_REDUCE_ID) + val indexFileSize = indexFile.length() + val indexBlockData = new FileSegmentManagedBuffer( + transportConf, indexFile, 0, indexFileSize) + + // Load the data block + val dataFile = getDataFile(shuffleId, mapId) + val dataBlockId = ShuffleDataBlockId(shuffleId, mapId, NOOP_REDUCE_ID) + val dataBlockData = new FileSegmentManagedBuffer( + transportConf, dataFile, 0, dataFile.length()) - // Load the data block - val dataFile = getDataFile(shuffleId, mapId) - val dataBlockId = ShuffleDataBlockId(shuffleId, mapId, NOOP_REDUCE_ID) - val dataBlockData = new FileSegmentManagedBuffer(transportConf, dataFile, 0, dataFile.length()) - List((indexBlockId, indexBlockData), (dataBlockId, dataBlockData)) + // Make sure the index exist. + if (!indexFile.exists()) { + throw new FileNotFoundException("Index file is deleted already.") + } + if (dataFile.exists()) { + List((indexBlockId, indexBlockData), (dataBlockId, dataBlockData)) + } else { + List((indexBlockId, indexBlockData)) + } + } catch { + case _: Exception => // If we can't load the blocks ignore them. + logWarning(s"Failed to resolve shuffle block ${shuffleBlockInfo}. " + + "This is expected to occur if a block is removed after decommissioning has started.") + List.empty[(BlockId, ManagedBuffer)] + } } @@ -302,6 +320,7 @@ private[spark] class IndexShuffleBlockResolver( } } } finally { + logDebug(s"Shuffle index for mapId $mapId: ${lengths.mkString("[", ",", "]")}") if (indexTmp.exists() && !indexTmp.delete()) { logError(s"Failed to delete temporary index file at ${indexTmp.getAbsolutePath}") } diff --git a/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala b/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala index 400c4526f0114..4e2183451c258 100644 --- a/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala +++ b/core/src/main/scala/org/apache/spark/shuffle/ShuffleManager.scala @@ -24,8 +24,12 @@ import org.apache.spark.{ShuffleDependency, TaskContext} * and on each executor, based on the spark.shuffle.manager setting. The driver registers shuffles * with it, and executors (or tasks running locally in the driver) can ask to read and write data. * - * NOTE: this will be instantiated by SparkEnv so its constructor can take a SparkConf and + * NOTE: + * 1. This will be instantiated by SparkEnv so its constructor can take a SparkConf and * boolean isDriver as parameters. + * 2. This contains a method ShuffleBlockResolver which interacts with External Shuffle Service + * when it is enabled. Need to pay attention to that, if implementing a custom ShuffleManager, to + * make sure the custom ShuffleManager could co-exist with External Shuffle Service. */ private[spark] trait ShuffleManager { diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala index 83f76db7e89da..cc21c1488f67c 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala @@ -95,6 +95,8 @@ private[spark] trait UIRoot { .build() } def securityManager: SecurityManager + + def checkUIViewPermissions(appId: String, attemptId: Option[String], user: String): Boolean } private[v1] object UIRootFromServletContext { @@ -145,6 +147,19 @@ private[v1] trait BaseAppResource extends ApiRequestContext { throw new NotFoundException(s"no such app: $appKey") } } + + protected def checkUIViewPermissions(): Unit = { + try { + val user = httpRequest.getRemoteUser() + if (!uiRoot.checkUIViewPermissions(appId, Option(attemptId), user)) { + throw new ForbiddenException(raw"""user "$user" is not authorized""") + } + } catch { + case _: NoSuchElementException => + val appKey = Option(attemptId).map(appId + "/" + _).getOrElse(appId) + throw new NotFoundException(s"no such app: $appKey") + } + } } private[v1] class ForbiddenException(msg: String) extends WebApplicationException( diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala index 536a1fcd59cd0..fb64ff5e60247 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala @@ -115,15 +115,14 @@ private[v1] class AbstractApplicationResource extends BaseAppResource { @Path("logs") @Produces(Array(MediaType.APPLICATION_OCTET_STREAM)) def getEventLogs(): Response = { - // Retrieve the UI for the application just to do access permission checks. For backwards - // compatibility, this code also tries with attemptId "1" if the UI without an attempt ID does - // not exist. + // For backwards compatibility, this code also tries with attemptId "1" if the UI + // without an attempt ID does not exist. try { - withUI { _ => } + checkUIViewPermissions() } catch { case _: NotFoundException if attemptId == null => attemptId = "1" - withUI { _ => } + checkUIViewPermissions() attemptId = null } diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index d207a6023f7f9..5a8cf09e1cba6 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -136,6 +136,10 @@ private[spark] class ExecutorMetricsJsonDeserializer new TypeReference[Option[Map[String, java.lang.Long]]] {}) metricsMap.map(metrics => new ExecutorMetrics(metrics)) } + + override def getNullValue(ctxt: DeserializationContext): Option[ExecutorMetrics] = { + None + } } /** serializer for peakMemoryMetrics: convert ExecutorMetrics to map with metric name as key */ private[spark] class ExecutorMetricsJsonSerializer @@ -144,11 +148,15 @@ private[spark] class ExecutorMetricsJsonSerializer metrics: Option[ExecutorMetrics], jsonGenerator: JsonGenerator, serializerProvider: SerializerProvider): Unit = { - metrics.foreach { m: ExecutorMetrics => - val metricsMap = ExecutorMetricType.metricToOffset.map { case (metric, _) => - metric -> m.getMetricValue(metric) + if (metrics.isEmpty) { + jsonGenerator.writeNull() + } else { + metrics.foreach { m: ExecutorMetrics => + val metricsMap = ExecutorMetricType.metricToOffset.map { case (metric, _) => + metric -> m.getMetricValue(metric) + } + jsonGenerator.writeObject(metricsMap) } - jsonGenerator.writeObject(metricsMap) } } diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala index 47af854b6e8ff..3909c02c5bb1f 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala @@ -56,7 +56,7 @@ import org.apache.spark.scheduler.ExecutorCacheTaskLocation import org.apache.spark.serializer.{SerializerInstance, SerializerManager} import org.apache.spark.shuffle.{MigratableResolver, ShuffleManager, ShuffleWriteMetricsReporter} import org.apache.spark.shuffle.{ShuffleManager, ShuffleWriteMetricsReporter} -import org.apache.spark.storage.BlockManagerMessages.ReplicateBlock +import org.apache.spark.storage.BlockManagerMessages.{DecommissionBlockManager, ReplicateBlock} import org.apache.spark.storage.memory._ import org.apache.spark.unsafe.Platform import org.apache.spark.util._ @@ -120,9 +120,7 @@ private[spark] class ByteBufferBlockData( private[spark] class HostLocalDirManager( futureExecutionContext: ExecutionContext, cacheSize: Int, - externalBlockStoreClient: ExternalBlockStoreClient, - host: String, - externalShuffleServicePort: Int) extends Logging { + blockStoreClient: BlockStoreClient) extends Logging { private val executorIdToLocalDirsCache = CacheBuilder @@ -130,24 +128,25 @@ private[spark] class HostLocalDirManager( .maximumSize(cacheSize) .build[String, Array[String]]() - private[spark] def getCachedHostLocalDirs() - : scala.collection.Map[String, Array[String]] = executorIdToLocalDirsCache.synchronized { - import scala.collection.JavaConverters._ - return executorIdToLocalDirsCache.asMap().asScala - } + private[spark] def getCachedHostLocalDirs: Map[String, Array[String]] = + executorIdToLocalDirsCache.synchronized { + executorIdToLocalDirsCache.asMap().asScala.toMap + } private[spark] def getHostLocalDirs( + host: String, + port: Int, executorIds: Array[String])( - callback: Try[java.util.Map[String, Array[String]]] => Unit): Unit = { + callback: Try[Map[String, Array[String]]] => Unit): Unit = { val hostLocalDirsCompletable = new CompletableFuture[java.util.Map[String, Array[String]]] - externalBlockStoreClient.getHostLocalDirs( + blockStoreClient.getHostLocalDirs( host, - externalShuffleServicePort, + port, executorIds, hostLocalDirsCompletable) hostLocalDirsCompletable.whenComplete { (hostLocalDirs, throwable) => if (hostLocalDirs != null) { - callback(Success(hostLocalDirs)) + callback(Success(hostLocalDirs.asScala.toMap)) executorIdToLocalDirsCache.synchronized { executorIdToLocalDirsCache.putAll(hostLocalDirs) } @@ -165,7 +164,7 @@ private[spark] class HostLocalDirManager( * Note that [[initialize()]] must be called before the BlockManager is usable. */ private[spark] class BlockManager( - executorId: String, + val executorId: String, rpcEnv: RpcEnv, val master: BlockManagerMaster, val serializerManager: SerializerManager, @@ -212,7 +211,7 @@ private[spark] class BlockManager( private val maxOnHeapMemory = memoryManager.maxOnHeapStorageMemory private val maxOffHeapMemory = memoryManager.maxOffHeapStorageMemory - private val externalShuffleServicePort = StorageUtils.externalShuffleServicePort(conf) + private[spark] val externalShuffleServicePort = StorageUtils.externalShuffleServicePort(conf) var blockManagerId: BlockManagerId = _ @@ -244,8 +243,9 @@ private[spark] class BlockManager( private var blockReplicationPolicy: BlockReplicationPolicy = _ + // visible for test // This is volatile since if it's defined we should not accept remote blocks. - @volatile private var decommissioner: Option[BlockManagerDecommissioner] = None + @volatile private[spark] var decommissioner: Option[BlockManagerDecommissioner] = None // A DownloadFileManager used to track all the files of remote blocks which are above the // specified memory threshold. Files will be deleted automatically based on weak reference. @@ -265,6 +265,8 @@ private[spark] class BlockManager( shuffleManager.shuffleBlockResolver.asInstanceOf[MigratableResolver] } + override def getLocalDiskDirs: Array[String] = diskBlockManager.localDirsString + /** * Abstraction for storing blocks from bytes, whether they start in memory or on disk. * @@ -492,20 +494,17 @@ private[spark] class BlockManager( registerWithExternalShuffleServer() } - hostLocalDirManager = + hostLocalDirManager = { if (conf.get(config.SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED) && !conf.get(config.SHUFFLE_USE_OLD_FETCH_PROTOCOL)) { - externalBlockStoreClient.map { blockStoreClient => - new HostLocalDirManager( - futureExecutionContext, - conf.get(config.STORAGE_LOCAL_DISK_BY_EXECUTORS_CACHE_SIZE), - blockStoreClient, - blockManagerId.host, - externalShuffleServicePort) - } + Some(new HostLocalDirManager( + futureExecutionContext, + conf.get(config.STORAGE_LOCAL_DISK_BY_EXECUTORS_CACHE_SIZE), + blockStoreClient)) } else { None } + } logInfo(s"Initialized BlockManager: $blockManagerId") } @@ -628,7 +627,7 @@ private[spark] class BlockManager( */ override def getLocalBlockData(blockId: BlockId): ManagedBuffer = { if (blockId.isShuffle) { - logInfo(s"Getting local shuffle block ${blockId}") + logDebug(s"Getting local shuffle block ${blockId}") shuffleManager.shuffleBlockResolver.getBlockData(blockId) } else { getLocalBytes(blockId) match { @@ -1811,7 +1810,9 @@ private[spark] class BlockManager( blocksToRemove.size } - def decommissionBlockManager(): Unit = synchronized { + def decommissionBlockManager(): Unit = storageEndpoint.ask(DecommissionBlockManager) + + private[spark] def decommissionSelf(): Unit = synchronized { decommissioner match { case None => logInfo("Starting block manager decommissioning process...") @@ -1822,6 +1823,14 @@ private[spark] class BlockManager( } } + /** + * Returns the last migration time and a boolean denoting if all the blocks have been migrated. + * If there are any tasks running since that time the boolean may be incorrect. + */ + private[spark] def lastMigrationInfo(): (Long, Boolean) = { + decommissioner.map(_.lastMigrationInfo()).getOrElse((0, false)) + } + private[storage] def getMigratableRDDBlocks(): Seq[ReplicateBlock] = master.getReplicateInfoForRDDBlocks(blockManagerId) diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala index 1cc7ef6a25f92..d1e89418a4897 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala @@ -17,7 +17,9 @@ package org.apache.spark.storage +import java.io.IOException import java.util.concurrent.ExecutorService +import java.util.concurrent.atomic.AtomicInteger import scala.collection.JavaConverters._ import scala.collection.mutable @@ -41,6 +43,12 @@ private[storage] class BlockManagerDecommissioner( private val maxReplicationFailuresForDecommission = conf.get(config.STORAGE_DECOMMISSION_MAX_REPLICATION_FAILURE_PER_BLOCK) + // Used for tracking if our migrations are complete. Readable for testing + @volatile private[storage] var lastRDDMigrationTime: Long = 0 + @volatile private[storage] var lastShuffleMigrationTime: Long = 0 + @volatile private[storage] var rddBlocksLeft: Boolean = true + @volatile private[storage] var shuffleBlocksLeft: Boolean = true + /** * This runnable consumes any shuffle blocks in the queue for migration. This part of a * producer/consumer where the main migration loop updates the queue of blocks to be migrated @@ -75,26 +83,47 @@ private[storage] class BlockManagerDecommissioner( Thread.sleep(SLEEP_TIME_SECS * 1000L) case Some((shuffleBlockInfo, retryCount)) => if (retryCount < maxReplicationFailuresForDecommission) { - logInfo(s"Trying to migrate shuffle ${shuffleBlockInfo} to ${peer}") - val blocks = - bm.migratableResolver.getMigrationBlocks(shuffleBlockInfo) - logDebug(s"Got migration sub-blocks ${blocks}") - blocks.foreach { case (blockId, buffer) => - logDebug(s"Migrating sub-block ${blockId}") - bm.blockTransferService.uploadBlockSync( - peer.host, - peer.port, - peer.executorId, - blockId, - buffer, - StorageLevel.DISK_ONLY, - null)// class tag, we don't need for shuffle - logDebug(s"Migrated sub block ${blockId}") + val blocks = bm.migratableResolver.getMigrationBlocks(shuffleBlockInfo) + if (blocks.isEmpty) { + logInfo(s"Ignore empty shuffle block $shuffleBlockInfo") + } else { + logInfo(s"Got migration sub-blocks ${blocks}") + logInfo(s"Trying to migrate shuffle ${shuffleBlockInfo} to ${peer} " + + s"($retryCount / $maxReplicationFailuresForDecommission)") + + // Migrate the components of the blocks. + try { + blocks.foreach { case (blockId, buffer) => + logDebug(s"Migrating sub-block ${blockId}") + bm.blockTransferService.uploadBlockSync( + peer.host, + peer.port, + peer.executorId, + blockId, + buffer, + StorageLevel.DISK_ONLY, + null) // class tag, we don't need for shuffle + logDebug(s"Migrated sub block ${blockId}") + } + logInfo(s"Migrated ${shuffleBlockInfo} to ${peer}") + } catch { + case e: IOException => + // If a block got deleted before netty opened the file handle, then trying to + // load the blocks now will fail. This is most likely to occur if we start + // migrating blocks and then the shuffle TTL cleaner kicks in. However this + // could also happen with manually managed shuffles or a GC event on the + // driver a no longer referenced RDD with shuffle files. + if (bm.migratableResolver.getMigrationBlocks(shuffleBlockInfo).isEmpty) { + logWarning(s"Skipping block ${shuffleBlockInfo}, block deleted.") + } else { + throw e + } + } } - logInfo(s"Migrated ${shuffleBlockInfo} to ${peer}") } else { logError(s"Skipping block ${shuffleBlockInfo} because it has failed ${retryCount}") } + numMigratedShuffles.incrementAndGet() } } // This catch is intentionally outside of the while running block. @@ -105,6 +134,7 @@ private[storage] class BlockManagerDecommissioner( case Some((shuffleMap, retryCount)) => logError(s"Error during migration, adding ${shuffleMap} back to migration queue", e) shufflesToMigrate.add((shuffleMap, retryCount + 1)) + running = false case None => logError(s"Error while waiting for block to migrate", e) } @@ -113,14 +143,23 @@ private[storage] class BlockManagerDecommissioner( } // Shuffles which are either in queue for migrations or migrated - private val migratingShuffles = mutable.HashSet[ShuffleBlockInfo]() + protected[storage] val migratingShuffles = mutable.HashSet[ShuffleBlockInfo]() + + // Shuffles which have migrated. This used to know when we are "done", being done can change + // if a new shuffle file is created by a running task. + private[storage] val numMigratedShuffles = new AtomicInteger(0) // Shuffles which are queued for migration & number of retries so far. + // Visible in storage for testing. private[storage] val shufflesToMigrate = new java.util.concurrent.ConcurrentLinkedQueue[(ShuffleBlockInfo, Int)]() // Set if we encounter an error attempting to migrate and stop. @volatile private var stopped = false + @volatile private var stoppedRDD = + !conf.get(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED) + @volatile private var stoppedShuffle = + !conf.get(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED) private val migrationPeers = mutable.HashMap[BlockManagerId, ShuffleMigrationRunnable]() @@ -133,22 +172,31 @@ private[storage] class BlockManagerDecommissioner( override def run(): Unit = { assert(conf.get(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED)) - while (!stopped && !Thread.interrupted()) { + while (!stopped && !stoppedRDD && !Thread.interrupted()) { logInfo("Iterating on migrating from the block manager.") + // Validate we have peers to migrate to. + val peers = bm.getPeers(false) + // If we have no peers give up. + if (peers.isEmpty) { + stopped = true + stoppedRDD = true + } try { + val startTime = System.nanoTime() logDebug("Attempting to replicate all cached RDD blocks") - decommissionRddCacheBlocks() + rddBlocksLeft = decommissionRddCacheBlocks() + lastRDDMigrationTime = startTime logInfo("Attempt to replicate all cached blocks done") logInfo(s"Waiting for ${sleepInterval} before refreshing migrations.") Thread.sleep(sleepInterval) } catch { case e: InterruptedException => - logInfo("Interrupted during migration, will not refresh migrations.") - stopped = true + logInfo("Interrupted during RDD migration, stopping") + stoppedRDD = true case NonFatal(e) => - logError("Error occurred while trying to replicate for block manager decommissioning.", + logError("Error occurred replicating RDD for block manager decommissioning.", e) - stopped = true + stoppedRDD = true } } } @@ -162,20 +210,22 @@ private[storage] class BlockManagerDecommissioner( override def run() { assert(conf.get(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED)) - while (!stopped && !Thread.interrupted()) { + while (!stopped && !stoppedShuffle && !Thread.interrupted()) { try { logDebug("Attempting to replicate all shuffle blocks") - refreshOffloadingShuffleBlocks() + val startTime = System.nanoTime() + shuffleBlocksLeft = refreshOffloadingShuffleBlocks() + lastShuffleMigrationTime = startTime logInfo("Done starting workers to migrate shuffle blocks") Thread.sleep(sleepInterval) } catch { case e: InterruptedException => logInfo("Interrupted during migration, will not refresh migrations.") - stopped = true + stoppedShuffle = true case NonFatal(e) => logError("Error occurred while trying to replicate for block manager decommissioning.", e) - stopped = true + stoppedShuffle = true } } } @@ -191,14 +241,17 @@ private[storage] class BlockManagerDecommissioner( * but rather shadows them. * Requires an Indexed based shuffle resolver. * Note: if called in testing please call stopOffloadingShuffleBlocks to avoid thread leakage. + * Returns true if we are not done migrating shuffle blocks. */ - private[storage] def refreshOffloadingShuffleBlocks(): Unit = { + private[storage] def refreshOffloadingShuffleBlocks(): Boolean = { // Update the queue of shuffles to be migrated logInfo("Offloading shuffle blocks") val localShuffles = bm.migratableResolver.getStoredShuffles().toSet - val newShufflesToMigrate = localShuffles.diff(migratingShuffles).toSeq + val newShufflesToMigrate = (localShuffles.diff(migratingShuffles)).toSeq shufflesToMigrate.addAll(newShufflesToMigrate.map(x => (x, 0)).asJava) migratingShuffles ++= newShufflesToMigrate + logInfo(s"${newShufflesToMigrate.size} of ${localShuffles.size} local shuffles " + + s"are added. In total, ${migratingShuffles.size} shuffles are remained.") // Update the threads doing migrations val livePeerSet = bm.getPeers(false).toSet @@ -215,6 +268,12 @@ private[storage] class BlockManagerDecommissioner( deadPeers.foreach { peer => migrationPeers.get(peer).foreach(_.running = false) } + // If we don't have anyone to migrate to give up + if (migrationPeers.values.find(_.running == true).isEmpty) { + stoppedShuffle = true + } + // If we found any new shuffles to migrate or otherwise have not migrated everything. + newShufflesToMigrate.nonEmpty || migratingShuffles.size > numMigratedShuffles.get() } /** @@ -231,16 +290,18 @@ private[storage] class BlockManagerDecommissioner( /** * Tries to offload all cached RDD blocks from this BlockManager to peer BlockManagers * Visible for testing + * Returns true if we have not migrated all of our RDD blocks. */ - private[storage] def decommissionRddCacheBlocks(): Unit = { + private[storage] def decommissionRddCacheBlocks(): Boolean = { val replicateBlocksInfo = bm.getMigratableRDDBlocks() + // Refresh peers and validate we have somewhere to move blocks. if (replicateBlocksInfo.nonEmpty) { logInfo(s"Need to replicate ${replicateBlocksInfo.size} RDD blocks " + "for block manager decommissioning") } else { logWarning(s"Asked to decommission RDD cache blocks, but no blocks to migrate") - return + return false } // TODO: We can sort these blocks based on some policy (LRU/blockSize etc) @@ -252,7 +313,9 @@ private[storage] class BlockManagerDecommissioner( if (blocksFailedReplication.nonEmpty) { logWarning("Blocks failed replication in cache decommissioning " + s"process: ${blocksFailedReplication.mkString(",")}") + return true } + return false } private def migrateBlock(blockToReplicate: ReplicateBlock): Boolean = { @@ -327,4 +390,33 @@ private[storage] class BlockManagerDecommissioner( } logInfo("Stopped storage decommissioner") } + + /* + * Returns the last migration time and a boolean for if all blocks have been migrated. + * The last migration time is calculated to be the minimum of the last migration of any + * running migration (and if there are now current running migrations it is set to current). + * This provides a timeStamp which, if there have been no tasks running since that time + * we can know that all potential blocks that can be have been migrated off. + */ + private[storage] def lastMigrationInfo(): (Long, Boolean) = { + if (stopped || (stoppedRDD && stoppedShuffle)) { + // Since we don't have anything left to migrate ever (since we don't restart once + // stopped), return that we're done with a validity timestamp that doesn't expire. + (Long.MaxValue, true) + } else { + // Chose the min of the active times. See the function description for more information. + val lastMigrationTime = if (!stoppedRDD && !stoppedShuffle) { + Math.min(lastRDDMigrationTime, lastShuffleMigrationTime) + } else if (!stoppedShuffle) { + lastShuffleMigrationTime + } else { + lastRDDMigrationTime + } + + // Technically we could have blocks left if we encountered an error, but those blocks will + // never be migrated, so we don't care about them. + val blocksMigrated = (!shuffleBlocksLeft || stoppedShuffle) && (!rddBlocksLeft || stoppedRDD) + (lastMigrationTime, blocksMigrated) + } + } } diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala index 93492cc6d7db6..f544d47b8e13c 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala @@ -43,9 +43,11 @@ class BlockManagerMaster( logInfo("Removed " + execId + " successfully in removeExecutor") } - /** Decommission block managers corresponding to given set of executors */ + /** Decommission block managers corresponding to given set of executors + * Non-blocking. + */ def decommissionBlockManagers(executorIds: Seq[String]): Unit = { - driverEndpoint.ask[Unit](DecommissionBlockManagers(executorIds)) + driverEndpoint.ask[Boolean](DecommissionBlockManagers(executorIds)) } /** Get Replication Info for all the RDD blocks stored in given blockManagerId */ diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala index a3d42348befaa..569d7d32284bc 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala @@ -163,8 +163,14 @@ class BlockManagerMasterEndpoint( context.reply(true) case DecommissionBlockManagers(executorIds) => - val bmIds = executorIds.flatMap(blockManagerIdByExecutor.get) - decommissionBlockManagers(bmIds) + // Mark corresponding BlockManagers as being decommissioning by adding them to + // decommissioningBlockManagerSet, so they won't be used to replicate or migrate blocks. + // Note that BlockManagerStorageEndpoint will be notified about decommissioning when the + // executor is notified(see BlockManager.decommissionSelf), so we don't need to send the + // notification here. + val bms = executorIds.flatMap(blockManagerIdByExecutor.get) + logInfo(s"Mark BlockManagers (${bms.mkString(", ")}) as being decommissioning.") + decommissioningBlockManagerSet ++= bms context.reply(true) case GetReplicateInfoForRDDBlocks(blockManagerId) => @@ -359,37 +365,28 @@ class BlockManagerMasterEndpoint( blockManagerIdByExecutor.get(execId).foreach(removeBlockManager) } - /** - * Decommission the given Seq of blockmanagers - * - Adds these block managers to decommissioningBlockManagerSet Set - * - Sends the DecommissionBlockManager message to each of the [[BlockManagerReplicaEndpoint]] - */ - def decommissionBlockManagers(blockManagerIds: Seq[BlockManagerId]): Future[Seq[Unit]] = { - val newBlockManagersToDecommission = blockManagerIds.toSet.diff(decommissioningBlockManagerSet) - val futures = newBlockManagersToDecommission.map { blockManagerId => - decommissioningBlockManagerSet.add(blockManagerId) - val info = blockManagerInfo(blockManagerId) - info.storageEndpoint.ask[Unit](DecommissionBlockManager) - } - Future.sequence{ futures.toSeq } - } - /** * Returns a Seq of ReplicateBlock for each RDD block stored by given blockManagerId * @param blockManagerId - block manager id for which ReplicateBlock info is needed * @return Seq of ReplicateBlock */ private def getReplicateInfoForRDDBlocks(blockManagerId: BlockManagerId): Seq[ReplicateBlock] = { - val info = blockManagerInfo(blockManagerId) + try { + val info = blockManagerInfo(blockManagerId) - val rddBlocks = info.blocks.keySet().asScala.filter(_.isRDD) - rddBlocks.map { blockId => - val currentBlockLocations = blockLocations.get(blockId) - val maxReplicas = currentBlockLocations.size + 1 - val remainingLocations = currentBlockLocations.toSeq.filter(bm => bm != blockManagerId) - val replicateMsg = ReplicateBlock(blockId, remainingLocations, maxReplicas) - replicateMsg - }.toSeq + val rddBlocks = info.blocks.keySet().asScala.filter(_.isRDD) + rddBlocks.map { blockId => + val currentBlockLocations = blockLocations.get(blockId) + val maxReplicas = currentBlockLocations.size + 1 + val remainingLocations = currentBlockLocations.toSeq.filter(bm => bm != blockManagerId) + val replicateMsg = ReplicateBlock(blockId, remainingLocations, maxReplicas) + replicateMsg + }.toSeq + } catch { + // If the block manager has already exited, nothing to replicate. + case e: java.util.NoSuchElementException => + Seq.empty[ReplicateBlock] + } } // Remove a block from the workers that have it. This can only be used to remove diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala index a69bebc23c661..54a72568b18fa 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala @@ -62,7 +62,7 @@ class BlockManagerStorageEndpoint( } case DecommissionBlockManager => - context.reply(blockManager.decommissionBlockManager()) + context.reply(blockManager.decommissionSelf()) case RemoveBroadcast(broadcastId, _) => doAsync[Int]("removing broadcast " + broadcastId, context) { diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala index a2843da0561e0..e3b3fc5cc4565 100644 --- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala +++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala @@ -210,13 +210,18 @@ final class ShuffleBlockFetcherIterator( while (iter.hasNext) { val result = iter.next() result match { - case SuccessFetchResult(_, _, address, _, buf, _) => + case SuccessFetchResult(blockId, mapIndex, address, _, buf, _) => if (address != blockManager.blockManagerId) { - shuffleMetrics.incRemoteBytesRead(buf.size) - if (buf.isInstanceOf[FileSegmentManagedBuffer]) { - shuffleMetrics.incRemoteBytesReadToDisk(buf.size) + if (hostLocalBlocks.contains(blockId -> mapIndex)) { + shuffleMetrics.incLocalBlocksFetched(1) + shuffleMetrics.incLocalBytesRead(buf.size) + } else { + shuffleMetrics.incRemoteBytesRead(buf.size) + if (buf.isInstanceOf[FileSegmentManagedBuffer]) { + shuffleMetrics.incRemoteBytesReadToDisk(buf.size) + } + shuffleMetrics.incRemoteBlocksFetched(1) } - shuffleMetrics.incRemoteBlocksFetched(1) } buf.release() case _ => @@ -290,9 +295,6 @@ final class ShuffleBlockFetcherIterator( var hostLocalBlockBytes = 0L var remoteBlockBytes = 0L - val hostLocalDirReadingEnabled = - blockManager.hostLocalDirManager != null && blockManager.hostLocalDirManager.isDefined - for ((address, blockInfos) <- blocksByAddress) { if (address.executorId == blockManager.blockManagerId.executorId) { checkBlockSizes(blockInfos) @@ -301,7 +303,8 @@ final class ShuffleBlockFetcherIterator( numBlocksToFetch += mergedBlockInfos.size localBlocks ++= mergedBlockInfos.map(info => (info.blockId, info.mapIndex)) localBlockBytes += mergedBlockInfos.map(_.size).sum - } else if (hostLocalDirReadingEnabled && address.host == blockManager.blockManagerId.host) { + } else if (blockManager.hostLocalDirManager.isDefined && + address.host == blockManager.blockManagerId.host) { checkBlockSizes(blockInfos) val mergedBlockInfos = mergeContinuousShuffleBlockIdsIfNeeded( blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)), doBatchFetch) @@ -463,54 +466,72 @@ final class ShuffleBlockFetcherIterator( * track in-memory are the ManagedBuffer references themselves. */ private[this] def fetchHostLocalBlocks(hostLocalDirManager: HostLocalDirManager): Unit = { - val cachedDirsByExec = hostLocalDirManager.getCachedHostLocalDirs() - val (hostLocalBlocksWithCachedDirs, hostLocalBlocksWithMissingDirs) = - hostLocalBlocksByExecutor - .map { case (hostLocalBmId, bmInfos) => - (hostLocalBmId, bmInfos, cachedDirsByExec.get(hostLocalBmId.executorId)) - }.partition(_._3.isDefined) - val bmId = blockManager.blockManagerId - val immutableHostLocalBlocksWithoutDirs = - hostLocalBlocksWithMissingDirs.map { case (hostLocalBmId, bmInfos, _) => - hostLocalBmId -> bmInfos - }.toMap - if (immutableHostLocalBlocksWithoutDirs.nonEmpty) { + val cachedDirsByExec = hostLocalDirManager.getCachedHostLocalDirs + val (hostLocalBlocksWithCachedDirs, hostLocalBlocksWithMissingDirs) = { + val (hasCache, noCache) = hostLocalBlocksByExecutor.partition { case (hostLocalBmId, _) => + cachedDirsByExec.contains(hostLocalBmId.executorId) + } + (hasCache.toMap, noCache.toMap) + } + + if (hostLocalBlocksWithMissingDirs.nonEmpty) { logDebug(s"Asynchronous fetching host-local blocks without cached executors' dir: " + - s"${immutableHostLocalBlocksWithoutDirs.mkString(", ")}") - val execIdsWithoutDirs = immutableHostLocalBlocksWithoutDirs.keys.map(_.executorId).toArray - hostLocalDirManager.getHostLocalDirs(execIdsWithoutDirs) { - case Success(dirs) => - immutableHostLocalBlocksWithoutDirs.foreach { case (hostLocalBmId, blockInfos) => - blockInfos.takeWhile { case (blockId, _, mapIndex) => - fetchHostLocalBlock( - blockId, - mapIndex, - dirs.get(hostLocalBmId.executorId), - hostLocalBmId) - } - } - logDebug(s"Got host-local blocks (without cached executors' dir) in " + - s"${Utils.getUsedTimeNs(startTimeNs)}") - - case Failure(throwable) => - logError(s"Error occurred while fetching host local blocks", throwable) - val (hostLocalBmId, blockInfoSeq) = immutableHostLocalBlocksWithoutDirs.head - val (blockId, _, mapIndex) = blockInfoSeq.head - results.put(FailureFetchResult(blockId, mapIndex, hostLocalBmId, throwable)) + s"${hostLocalBlocksWithMissingDirs.mkString(", ")}") + + // If the external shuffle service is enabled, we'll fetch the local directories for + // multiple executors from the external shuffle service, which located at the same host + // with the executors, in once. Otherwise, we'll fetch the local directories from those + // executors directly one by one. The fetch requests won't be too much since one host is + // almost impossible to have many executors at the same time practically. + val dirFetchRequests = if (blockManager.externalShuffleServiceEnabled) { + val host = blockManager.blockManagerId.host + val port = blockManager.externalShuffleServicePort + Seq((host, port, hostLocalBlocksWithMissingDirs.keys.toArray)) + } else { + hostLocalBlocksWithMissingDirs.keys.map(bmId => (bmId.host, bmId.port, Array(bmId))).toSeq + } + + dirFetchRequests.foreach { case (host, port, bmIds) => + hostLocalDirManager.getHostLocalDirs(host, port, bmIds.map(_.executorId)) { + case Success(dirsByExecId) => + fetchMultipleHostLocalBlocks( + hostLocalBlocksWithMissingDirs.filterKeys(bmIds.contains).toMap, + dirsByExecId, + cached = false) + + case Failure(throwable) => + logError("Error occurred while fetching host local blocks", throwable) + val bmId = bmIds.head + val blockInfoSeq = hostLocalBlocksWithMissingDirs(bmId) + val (blockId, _, mapIndex) = blockInfoSeq.head + results.put(FailureFetchResult(blockId, mapIndex, bmId, throwable)) + } } } + if (hostLocalBlocksWithCachedDirs.nonEmpty) { logDebug(s"Synchronous fetching host-local blocks with cached executors' dir: " + s"${hostLocalBlocksWithCachedDirs.mkString(", ")}") - hostLocalBlocksWithCachedDirs.foreach { case (_, blockInfos, localDirs) => - blockInfos.foreach { case (blockId, _, mapIndex) => - if (!fetchHostLocalBlock(blockId, mapIndex, localDirs.get, bmId)) { - return - } - } + fetchMultipleHostLocalBlocks(hostLocalBlocksWithCachedDirs, cachedDirsByExec, cached = true) + } + } + + private def fetchMultipleHostLocalBlocks( + bmIdToBlocks: Map[BlockManagerId, Seq[(BlockId, Long, Int)]], + localDirsByExecId: Map[String, Array[String]], + cached: Boolean): Unit = { + // We use `forall` because once there's a failed block fetch, `fetchHostLocalBlock` will put + // a `FailureFetchResult` immediately to the `results`. So there's no reason to fetch the + // remaining blocks. + val allFetchSucceeded = bmIdToBlocks.forall { case (bmId, blockInfos) => + blockInfos.forall { case (blockId, _, mapIndex) => + fetchHostLocalBlock(blockId, mapIndex, localDirsByExecId(bmId.executorId), bmId) } - logDebug(s"Got host-local blocks (with cached executors' dir) in " + - s"${Utils.getUsedTimeNs(startTimeNs)}") + } + if (allFetchSucceeded) { + logDebug(s"Got host-local blocks from ${bmIdToBlocks.keys.mkString(", ")} " + + s"(${if (cached) "with" else "without"} cached executors' dir) " + + s"in ${Utils.getUsedTimeNs(startTimeNs)}") } } diff --git a/core/src/main/scala/org/apache/spark/storage/StorageLevel.scala b/core/src/main/scala/org/apache/spark/storage/StorageLevel.scala index 4c6998d7a8e20..f6db73ba805b1 100644 --- a/core/src/main/scala/org/apache/spark/storage/StorageLevel.scala +++ b/core/src/main/scala/org/apache/spark/storage/StorageLevel.scala @@ -153,6 +153,7 @@ object StorageLevel { val NONE = new StorageLevel(false, false, false, false) val DISK_ONLY = new StorageLevel(true, false, false, false) val DISK_ONLY_2 = new StorageLevel(true, false, false, false, 2) + val DISK_ONLY_3 = new StorageLevel(true, false, false, false, 3) val MEMORY_ONLY = new StorageLevel(false, true, false, true) val MEMORY_ONLY_2 = new StorageLevel(false, true, false, true, 2) val MEMORY_ONLY_SER = new StorageLevel(false, true, false, false) @@ -172,6 +173,7 @@ object StorageLevel { case "NONE" => NONE case "DISK_ONLY" => DISK_ONLY case "DISK_ONLY_2" => DISK_ONLY_2 + case "DISK_ONLY_3" => DISK_ONLY_3 case "MEMORY_ONLY" => MEMORY_ONLY case "MEMORY_ONLY_2" => MEMORY_ONLY_2 case "MEMORY_ONLY_SER" => MEMORY_ONLY_SER diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala index 8ae9828c3fee1..b1769a8a9c9ee 100644 --- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala @@ -110,6 +110,11 @@ private[spark] class SparkUI private ( } } + override def checkUIViewPermissions(appId: String, attemptId: Option[String], + user: String): Boolean = { + securityManager.checkUIViewPermissions(user) + } + def getApplicationInfoList: Iterator[ApplicationInfo] = { Iterator(new ApplicationInfo( id = appId, diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index a070cc9c7b39d..dba6f8e8440cb 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -231,7 +231,7 @@ private[spark] object UIUtils extends Logging { - + diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 4e76ea289ede6..5f5a08fe0e574 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -147,7 +147,8 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We | 'Removed at ${UIUtils.formatDate(removeTime)}' + | '${ e.removeReason.map { reason => - s"""
Reason: ${reason.replace("\n", " ")}""" + s"""
Reason: ${StringEscapeUtils.escapeEcmaScript( + reason.replace("\n", " "))}""" }.getOrElse("") }"' + | 'data-html="true">Executor ${e.id} removed' diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala index df239d6d0e187..19eccc5209b8e 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala @@ -127,7 +127,8 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP | 'Removed at ${UIUtils.formatDate(removeTime)}' + | '${ e.removeReason.map { reason => - s"""
Reason: ${reason.replace("\n", " ")}""" + s"""
Reason: ${StringEscapeUtils.escapeEcmaScript( + reason.replace("\n", " "))}""" }.getOrElse("") }"' + | 'data-html="true">Executor ${e.id} removed' diff --git a/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala b/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala new file mode 100644 index 0000000000000..c0a135e04bac5 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.util + +import java.io.FileNotFoundException + +import scala.collection.mutable + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs._ +import org.apache.hadoop.fs.viewfs.ViewFileSystem +import org.apache.hadoop.hdfs.DistributedFileSystem + +import org.apache.spark._ +import org.apache.spark.annotation.Private +import org.apache.spark.internal.Logging +import org.apache.spark.metrics.source.HiveCatalogMetrics + +/** + * Utility functions to simplify and speed-up file listing. + */ +private[spark] object HadoopFSUtils extends Logging { + /** + * Lists a collection of paths recursively. Picks the listing strategy adaptively depending + * on the number of paths to list. + * + * This may only be called on the driver. + * + * @param sc Spark context used to run parallel listing. + * @param paths Input paths to list + * @param hadoopConf Hadoop configuration + * @param filter Path filter used to exclude leaf files from result + * @param isRootLevel Whether the input paths are at the root level, i.e., they are the root + * paths as opposed to nested paths encountered during recursive calls of this. + * @param ignoreMissingFiles Ignore missing files that happen during recursive listing + * (e.g., due to race conditions) + * @param ignoreLocality Whether to fetch data locality info when listing leaf files. If false, + * this will return `FileStatus` without `BlockLocation` info. + * @param parallelismThreshold The threshold to enable parallelism. If the number of input paths + * is smaller than this value, this will fallback to use + * sequential listing. + * @param parallelismMax The maximum parallelism for listing. If the number of input paths is + * larger than this value, parallelism will be throttled to this value + * to avoid generating too many tasks. + * @param filterFun Optional predicate on the leaf files. Files who failed the check will be + * excluded from the results + * @return for each input path, the set of discovered files for the path + */ + def parallelListLeafFiles( + sc: SparkContext, + paths: Seq[Path], + hadoopConf: Configuration, + filter: PathFilter, + isRootLevel: Boolean, + ignoreMissingFiles: Boolean, + ignoreLocality: Boolean, + parallelismThreshold: Int, + parallelismMax: Int, + filterFun: Option[String => Boolean] = None): Seq[(Path, Seq[FileStatus])] = { + + // Short-circuits parallel listing when serial listing is likely to be faster. + if (paths.size <= parallelismThreshold) { + return paths.map { path => + val leafFiles = listLeafFiles( + path, + hadoopConf, + filter, + Some(sc), + ignoreMissingFiles = ignoreMissingFiles, + ignoreLocality = ignoreLocality, + isRootPath = isRootLevel, + parallelismThreshold = parallelismThreshold, + parallelismMax = parallelismMax, + filterFun = filterFun) + (path, leafFiles) + } + } + + logInfo(s"Listing leaf files and directories in parallel under ${paths.length} paths." + + s" The first several paths are: ${paths.take(10).mkString(", ")}.") + HiveCatalogMetrics.incrementParallelListingJobCount(1) + + val serializableConfiguration = new SerializableConfiguration(hadoopConf) + val serializedPaths = paths.map(_.toString) + + // Set the number of parallelism to prevent following file listing from generating many tasks + // in case of large #defaultParallelism. + val numParallelism = Math.min(paths.size, parallelismMax) + + val previousJobDescription = sc.getLocalProperty(SparkContext.SPARK_JOB_DESCRIPTION) + val statusMap = try { + val description = paths.size match { + case 0 => + "Listing leaf files and directories 0 paths" + case 1 => + s"Listing leaf files and directories for 1 path:
${paths(0)}" + case s => + s"Listing leaf files and directories for $s paths:
${paths(0)}, ..." + } + sc.setJobDescription(description) + sc + .parallelize(serializedPaths, numParallelism) + .mapPartitions { pathStrings => + val hadoopConf = serializableConfiguration.value + pathStrings.map(new Path(_)).toSeq.map { path => + val leafFiles = listLeafFiles( + path = path, + hadoopConf = hadoopConf, + filter = filter, + contextOpt = None, // Can't execute parallel scans on workers + ignoreMissingFiles = ignoreMissingFiles, + ignoreLocality = ignoreLocality, + isRootPath = isRootLevel, + filterFun = filterFun, + parallelismThreshold = Int.MaxValue, + parallelismMax = 0) + (path, leafFiles) + }.iterator + }.map { case (path, statuses) => + val serializableStatuses = statuses.map { status => + // Turn FileStatus into SerializableFileStatus so we can send it back to the driver + val blockLocations = status match { + case f: LocatedFileStatus => + f.getBlockLocations.map { loc => + SerializableBlockLocation( + loc.getNames, + loc.getHosts, + loc.getOffset, + loc.getLength) + } + + case _ => + Array.empty[SerializableBlockLocation] + } + + SerializableFileStatus( + status.getPath.toString, + status.getLen, + status.isDirectory, + status.getReplication, + status.getBlockSize, + status.getModificationTime, + status.getAccessTime, + blockLocations) + } + (path.toString, serializableStatuses) + }.collect() + } finally { + sc.setJobDescription(previousJobDescription) + } + + // turn SerializableFileStatus back to Status + statusMap.map { case (path, serializableStatuses) => + val statuses = serializableStatuses.map { f => + val blockLocations = f.blockLocations.map { loc => + new BlockLocation(loc.names, loc.hosts, loc.offset, loc.length) + } + new LocatedFileStatus( + new FileStatus( + f.length, f.isDir, f.blockReplication, f.blockSize, f.modificationTime, + new Path(f.path)), + blockLocations) + } + (new Path(path), statuses) + } + } + + // scalastyle:off argcount + /** + * Lists a single filesystem path recursively. If a `SparkContext` object is specified, this + * function may launch Spark jobs to parallelize listing based on `parallelismThreshold`. + * + * If sessionOpt is None, this may be called on executors. + * + * @return all children of path that match the specified filter. + */ + private def listLeafFiles( + path: Path, + hadoopConf: Configuration, + filter: PathFilter, + contextOpt: Option[SparkContext], + ignoreMissingFiles: Boolean, + ignoreLocality: Boolean, + isRootPath: Boolean, + filterFun: Option[String => Boolean], + parallelismThreshold: Int, + parallelismMax: Int): Seq[FileStatus] = { + + logTrace(s"Listing $path") + val fs = path.getFileSystem(hadoopConf) + + // Note that statuses only include FileStatus for the files and dirs directly under path, + // and does not include anything else recursively. + val statuses: Array[FileStatus] = try { + fs match { + // DistributedFileSystem overrides listLocatedStatus to make 1 single call to namenode + // to retrieve the file status with the file block location. The reason to still fallback + // to listStatus is because the default implementation would potentially throw a + // FileNotFoundException which is better handled by doing the lookups manually below. + case (_: DistributedFileSystem | _: ViewFileSystem) if !ignoreLocality => + val remoteIter = fs.listLocatedStatus(path) + new Iterator[LocatedFileStatus]() { + def next(): LocatedFileStatus = remoteIter.next + def hasNext(): Boolean = remoteIter.hasNext + }.toArray + case _ => fs.listStatus(path) + } + } catch { + // If we are listing a root path for SQL (e.g. a top level directory of a table), we need to + // ignore FileNotFoundExceptions during this root level of the listing because + // + // (a) certain code paths might construct an InMemoryFileIndex with root paths that + // might not exist (i.e. not all callers are guaranteed to have checked + // path existence prior to constructing InMemoryFileIndex) and, + // (b) we need to ignore deleted root paths during REFRESH TABLE, otherwise we break + // existing behavior and break the ability drop SessionCatalog tables when tables' + // root directories have been deleted (which breaks a number of Spark's own tests). + // + // If we are NOT listing a root path then a FileNotFoundException here means that the + // directory was present in a previous level of file listing but is absent in this + // listing, likely indicating a race condition (e.g. concurrent table overwrite or S3 + // list inconsistency). + // + // The trade-off in supporting existing behaviors / use-cases is that we won't be + // able to detect race conditions involving root paths being deleted during + // InMemoryFileIndex construction. However, it's still a net improvement to detect and + // fail-fast on the non-root cases. For more info see the SPARK-27676 review discussion. + case _: FileNotFoundException if isRootPath || ignoreMissingFiles => + logWarning(s"The directory $path was not found. Was it deleted very recently?") + Array.empty[FileStatus] + } + + def doFilter(statuses: Array[FileStatus]) = filterFun match { + case Some(shouldFilterOut) => + statuses.filterNot(status => shouldFilterOut(status.getPath.getName)) + case None => + statuses + } + + val filteredStatuses = doFilter(statuses) + val allLeafStatuses = { + val (dirs, topLevelFiles) = filteredStatuses.partition(_.isDirectory) + val nestedFiles: Seq[FileStatus] = contextOpt match { + case Some(context) if dirs.size > parallelismThreshold => + parallelListLeafFiles( + context, + dirs.map(_.getPath), + hadoopConf = hadoopConf, + filter = filter, + isRootLevel = false, + ignoreMissingFiles = ignoreMissingFiles, + ignoreLocality = ignoreLocality, + filterFun = filterFun, + parallelismThreshold = parallelismThreshold, + parallelismMax = parallelismMax + ).flatMap(_._2) + case _ => + dirs.flatMap { dir => + listLeafFiles( + path = dir.getPath, + hadoopConf = hadoopConf, + filter = filter, + contextOpt = contextOpt, + ignoreMissingFiles = ignoreMissingFiles, + ignoreLocality = ignoreLocality, + isRootPath = false, + filterFun = filterFun, + parallelismThreshold = parallelismThreshold, + parallelismMax = parallelismMax) + } + } + val allFiles = topLevelFiles ++ nestedFiles + if (filter != null) allFiles.filter(f => filter.accept(f.getPath)) else allFiles + } + + val missingFiles = mutable.ArrayBuffer.empty[String] + val filteredLeafStatuses = doFilter(allLeafStatuses) + val resolvedLeafStatuses = filteredLeafStatuses.flatMap { + case f: LocatedFileStatus => + Some(f) + + // NOTE: + // + // - Although S3/S3A/S3N file system can be quite slow for remote file metadata + // operations, calling `getFileBlockLocations` does no harm here since these file system + // implementations don't actually issue RPC for this method. + // + // - Here we are calling `getFileBlockLocations` in a sequential manner, but it should not + // be a big deal since we always use to `parallelListLeafFiles` when the number of + // paths exceeds threshold. + case f if !ignoreLocality => + // The other constructor of LocatedFileStatus will call FileStatus.getPermission(), + // which is very slow on some file system (RawLocalFileSystem, which is launch a + // subprocess and parse the stdout). + try { + val locations = fs.getFileBlockLocations(f, 0, f.getLen).map { loc => + // Store BlockLocation objects to consume less memory + if (loc.getClass == classOf[BlockLocation]) { + loc + } else { + new BlockLocation(loc.getNames, loc.getHosts, loc.getOffset, loc.getLength) + } + } + val lfs = new LocatedFileStatus(f.getLen, f.isDirectory, f.getReplication, f.getBlockSize, + f.getModificationTime, 0, null, null, null, null, f.getPath, locations) + if (f.isSymlink) { + lfs.setSymlink(f.getSymlink) + } + Some(lfs) + } catch { + case _: FileNotFoundException if ignoreMissingFiles => + missingFiles += f.getPath.toString + None + } + + case f => Some(f) + } + + if (missingFiles.nonEmpty) { + logWarning( + s"the following files were missing during file scan:\n ${missingFiles.mkString("\n ")}") + } + + resolvedLeafStatuses + } + // scalastyle:on argcount + + /** A serializable variant of HDFS's BlockLocation. */ + private case class SerializableBlockLocation( + names: Array[String], + hosts: Array[String], + offset: Long, + length: Long) + + /** A serializable variant of HDFS's FileStatus. */ + private case class SerializableFileStatus( + path: String, + length: Long, + isDir: Boolean, + blockReplication: Short, + blockSize: Long, + modificationTime: Long, + accessTime: Long, + blockLocations: Array[SerializableBlockLocation]) +} diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 35d60bb514405..b8b044bbad30e 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -50,7 +50,7 @@ import com.google.common.net.InetAddresses import org.apache.commons.codec.binary.Hex import org.apache.commons.lang3.SystemUtils import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileSystem, FileUtil, Path} +import org.apache.hadoop.fs.{FileSystem, FileUtil, Path, Trash} import org.apache.hadoop.io.compress.{CompressionCodecFactory, SplittableCompressionCodec} import org.apache.hadoop.security.UserGroupInformation import org.apache.hadoop.yarn.conf.YarnConfiguration @@ -269,6 +269,29 @@ private[spark] object Utils extends Logging { file.setExecutable(true, true) } + /** + * Move data to trash if 'spark.sql.truncate.trash.enabled' is true, else + * delete the data permanently. If move data to trash failed fallback to hard deletion. + */ + def moveToTrashOrDelete( + fs: FileSystem, + partitionPath: Path, + isTrashEnabled: Boolean, + hadoopConf: Configuration): Boolean = { + if (isTrashEnabled) { + logDebug(s"Try to move data ${partitionPath.toString} to trash") + val isSuccess = Trash.moveToAppropriateTrash(fs, partitionPath, hadoopConf) + if (!isSuccess) { + logWarning(s"Failed to move data ${partitionPath.toString} to trash. " + + "Fallback to hard deletion") + return fs.delete(partitionPath, true) + } + isSuccess + } else { + fs.delete(partitionPath, true) + } + } + /** * Create a directory given the abstract pathname * @return true, if the directory is successfully created; otherwise, return false. diff --git a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java index 6e995a3929a75..f4e952f465e54 100644 --- a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java +++ b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java @@ -172,6 +172,7 @@ public void emptyMap() { final byte[] key = getRandomByteArray(keyLengthInWords); Assert.assertFalse(map.lookup(key, Platform.BYTE_ARRAY_OFFSET, keyLengthInBytes).isDefined()); Assert.assertFalse(map.iterator().hasNext()); + Assert.assertFalse(map.iteratorWithKeyIndex().hasNext()); } finally { map.free(); } @@ -233,9 +234,10 @@ public void setAndRetrieveAKey() { } } - private void iteratorTestBase(boolean destructive) throws Exception { + private void iteratorTestBase(boolean destructive, boolean isWithKeyIndex) throws Exception { final int size = 4096; BytesToBytesMap map = new BytesToBytesMap(taskMemoryManager, size / 2, PAGE_SIZE_BYTES); + Assert.assertEquals(size / 2, map.maxNumKeysIndex()); try { for (long i = 0; i < size; i++) { final long[] value = new long[] { i }; @@ -267,6 +269,8 @@ private void iteratorTestBase(boolean destructive) throws Exception { final Iterator iter; if (destructive) { iter = map.destructiveIterator(); + } else if (isWithKeyIndex) { + iter = map.iteratorWithKeyIndex(); } else { iter = map.iterator(); } @@ -291,6 +295,12 @@ private void iteratorTestBase(boolean destructive) throws Exception { countFreedPages++; } } + if (keyLength != 0 && isWithKeyIndex) { + final BytesToBytesMap.Location expectedLoc = map.lookup( + loc.getKeyBase(), loc.getKeyOffset(), loc.getKeyLength()); + Assert.assertTrue(expectedLoc.isDefined() && + expectedLoc.getKeyIndex() == loc.getKeyIndex()); + } } if (destructive) { // Latest page is not freed by iterator but by map itself @@ -304,12 +314,17 @@ private void iteratorTestBase(boolean destructive) throws Exception { @Test public void iteratorTest() throws Exception { - iteratorTestBase(false); + iteratorTestBase(false, false); } @Test public void destructiveIteratorTest() throws Exception { - iteratorTestBase(true); + iteratorTestBase(true, false); + } + + @Test + public void iteratorWithKeyIndexTest() throws Exception { + iteratorTestBase(false, true); } @Test @@ -603,6 +618,12 @@ public void multipleValuesForSameKey() { final BytesToBytesMap.Location loc = iter.next(); assert loc.isDefined(); } + BytesToBytesMap.MapIteratorWithKeyIndex iterWithKeyIndex = map.iteratorWithKeyIndex(); + for (i = 0; i < 2048; i++) { + assert iterWithKeyIndex.hasNext(); + final BytesToBytesMap.Location loc = iterWithKeyIndex.next(); + assert loc.isDefined() && loc.getKeyIndex() >= 0; + } } finally { map.free(); } diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java index 43977717f6c97..dc2b4814c8284 100644 --- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java +++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java @@ -23,7 +23,6 @@ import java.util.LinkedList; import java.util.UUID; -import org.hamcrest.Matchers; import scala.Tuple2$; import org.junit.After; @@ -38,7 +37,6 @@ import org.apache.spark.executor.TaskMetrics; import org.apache.spark.internal.config.package$; import org.apache.spark.memory.TestMemoryManager; -import org.apache.spark.memory.SparkOutOfMemoryError; import org.apache.spark.memory.TaskMemoryManager; import org.apache.spark.serializer.JavaSerializer; import org.apache.spark.serializer.SerializerInstance; @@ -359,6 +357,69 @@ public void forcedSpillingWithReadIterator() throws Exception { assertSpillFilesWereCleanedUp(); } + @Test + public void forcedSpillingNullsWithReadIterator() throws Exception { + final UnsafeExternalSorter sorter = newSorter(); + long[] record = new long[100]; + final int recordSize = record.length * 8; + final int n = (int) pageSizeBytes / recordSize * 3; + for (int i = 0; i < n; i++) { + boolean isNull = i % 2 == 0; + sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0, isNull); + } + assertTrue(sorter.getNumberOfAllocatedPages() >= 2); + + UnsafeExternalSorter.SpillableIterator iter = + (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator(); + final int numRecordsToReadBeforeSpilling = n / 3; + for (int i = 0; i < numRecordsToReadBeforeSpilling; i++) { + assertTrue(iter.hasNext()); + iter.loadNext(); + } + + assertTrue(iter.spill() > 0); + assertEquals(0, iter.spill()); + + for (int i = numRecordsToReadBeforeSpilling; i < n; i++) { + assertTrue(iter.hasNext()); + iter.loadNext(); + } + assertFalse(iter.hasNext()); + + sorter.cleanupResources(); + assertSpillFilesWereCleanedUp(); + } + + @Test + public void forcedSpillingWithFullyReadIterator() throws Exception { + final UnsafeExternalSorter sorter = newSorter(); + long[] record = new long[100]; + final int recordSize = record.length * 8; + final int n = (int) pageSizeBytes / recordSize * 3; + for (int i = 0; i < n; i++) { + record[0] = i; + sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0, false); + } + assertTrue(sorter.getNumberOfAllocatedPages() >= 2); + + UnsafeExternalSorter.SpillableIterator iter = + (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator(); + for (int i = 0; i < n; i++) { + assertTrue(iter.hasNext()); + iter.loadNext(); + assertEquals(i, Platform.getLong(iter.getBaseObject(), iter.getBaseOffset())); + } + assertFalse(iter.hasNext()); + + assertTrue(iter.spill() > 0); + assertEquals(0, iter.spill()); + assertEquals(n - 1, Platform.getLong(iter.getBaseObject(), iter.getBaseOffset())); + assertFalse(iter.hasNext()); + + sorter.cleanupResources(); + assertSpillFilesWereCleanedUp(); + } + @Test public void forcedSpillingWithNotReadIterator() throws Exception { final UnsafeExternalSorter sorter = newSorter(); @@ -518,40 +579,28 @@ public void testGetIterator() throws Exception { } @Test - public void testOOMDuringSpill() throws Exception { + public void testNoOOMDuringSpill() throws Exception { final UnsafeExternalSorter sorter = newSorter(); - // we assume that given default configuration, - // the size of the data we insert to the sorter (ints) - // and assuming we shouldn't spill before pointers array is exhausted - // (memory manager is not configured to throw at this point) - // - so this loop runs a reasonable number of iterations (<2000). - // test indeed completed within <30ms (on a quad i7 laptop). - for (int i = 0; sorter.hasSpaceForAnotherRecord(); ++i) { + for (int i = 0; i < 100; i++) { insertNumber(sorter, i); } - // we expect the next insert to attempt growing the pointerssArray first - // allocation is expected to fail, then a spill is triggered which - // attempts another allocation which also fails and we expect to see this - // OOM here. the original code messed with a released array within the - // spill code and ended up with a failed assertion. we also expect the - // location of the OOM to be - // org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset - memoryManager.markconsequentOOM(2); - try { - insertNumber(sorter, 1024); - fail("expected OutOfMmoryError but it seems operation surprisingly succeeded"); - } - // we expect an SparkOutOfMemoryError here, anything else (i.e the original NPE is a failure) - catch (SparkOutOfMemoryError oom){ - String oomStackTrace = Utils.exceptionString(oom); - assertThat("expected SparkOutOfMemoryError in " + - "org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset", - oomStackTrace, - Matchers.containsString( - "org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset")); + + // Check that spilling still succeeds when the task is starved for memory. + memoryManager.markconsequentOOM(Integer.MAX_VALUE); + sorter.spill(); + memoryManager.resetConsequentOOM(); + + // Ensure that records can be appended after spilling, i.e. check that the sorter will allocate + // the new pointer array that it could not allocate while spilling. + for (int i = 0; i < 100; ++i) { + insertNumber(sorter, i); } + + sorter.cleanupResources(); + assertSpillFilesWereCleanedUp(); } + private void verifyIntIterator(UnsafeSorterIterator iter, int start, int end) throws IOException { for (int i = start; i < end; i++) { diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java index 2b8a0602730e1..9d4909ddce792 100644 --- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java +++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java @@ -20,6 +20,7 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; +import org.apache.spark.unsafe.array.LongArray; import org.junit.Assert; import org.junit.Test; @@ -27,7 +28,6 @@ import org.apache.spark.SparkConf; import org.apache.spark.memory.TestMemoryConsumer; import org.apache.spark.memory.TestMemoryManager; -import org.apache.spark.memory.SparkOutOfMemoryError; import org.apache.spark.memory.TaskMemoryManager; import org.apache.spark.unsafe.Platform; import org.apache.spark.unsafe.memory.MemoryBlock; @@ -37,7 +37,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.isIn; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; public class UnsafeInMemorySorterSuite { @@ -147,7 +146,7 @@ public int compare( } @Test - public void freeAfterOOM() { + public void testNoOOMDuringReset() { final SparkConf sparkConf = new SparkConf(); sparkConf.set(package$.MODULE$.MEMORY_OFFHEAP_ENABLED(), false); @@ -156,12 +155,7 @@ public void freeAfterOOM() { final TaskMemoryManager memoryManager = new TaskMemoryManager( testMemoryManager, 0); final TestMemoryConsumer consumer = new TestMemoryConsumer(memoryManager); - final MemoryBlock dataPage = memoryManager.allocatePage(2048, consumer); - final Object baseObject = dataPage.getBaseObject(); - // Write the records into the data page: - long position = dataPage.getBaseOffset(); - final HashPartitioner hashPartitioner = new HashPartitioner(4); // Use integer comparison for comparing prefixes (which are partition ids, in this case) final PrefixComparator prefixComparator = PrefixComparators.LONG; final RecordComparator recordComparator = new RecordComparator() { @@ -179,18 +173,24 @@ public int compare( UnsafeInMemorySorter sorter = new UnsafeInMemorySorter(consumer, memoryManager, recordComparator, prefixComparator, 100, shouldUseRadixSort()); - testMemoryManager.markExecutionAsOutOfMemoryOnce(); - try { - sorter.reset(); - fail("expected SparkOutOfMemoryError but it seems operation surprisingly succeeded"); - } catch (SparkOutOfMemoryError oom) { - // as expected - } - // [SPARK-21907] this failed on NPE at - // org.apache.spark.memory.MemoryConsumer.freeArray(MemoryConsumer.java:108) - sorter.free(); - // simulate a 'back to back' free. - sorter.free(); + // Ensure that the sorter does not OOM while freeing its memory. + testMemoryManager.markconsequentOOM(Integer.MAX_VALUE); + sorter.freeMemory(); + testMemoryManager.resetConsequentOOM(); + Assert.assertFalse(sorter.hasSpaceForAnotherRecord()); + + // Get the sorter in an usable state again by allocating a new pointer array. + LongArray array = consumer.allocateArray(1000); + sorter.expandPointerArray(array); + + // Ensure that it is safe to call freeMemory() multiple times. + testMemoryManager.markconsequentOOM(Integer.MAX_VALUE); + sorter.freeMemory(); + sorter.freeMemory(); + testMemoryManager.resetConsequentOOM(); + Assert.assertFalse(sorter.hasSpaceForAnotherRecord()); + + assertEquals(0L, memoryManager.cleanUpAllAllocatedMemory()); } } diff --git a/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java b/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java index 0f489fb219010..b188ee16b97d0 100644 --- a/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java +++ b/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java @@ -28,6 +28,7 @@ import org.apache.spark.api.java.*; import org.apache.spark.*; +import org.apache.spark.util.Utils; /** * Java apps can use both Java-friendly JavaSparkContext and Scala SparkContext. @@ -35,14 +36,16 @@ public class JavaSparkContextSuite implements Serializable { @Test - public void javaSparkContext() { + public void javaSparkContext() throws IOException { + File tempDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark"); + String dummyJarFile = File.createTempFile(tempDir.toString(), "jarFile").toString(); String[] jars = new String[] {}; java.util.Map environment = new java.util.HashMap<>(); new JavaSparkContext(new SparkConf().setMaster("local").setAppName("name")).stop(); new JavaSparkContext("local", "name", new SparkConf()).stop(); new JavaSparkContext("local", "name").stop(); - new JavaSparkContext("local", "name", "sparkHome", "jarFile").stop(); + new JavaSparkContext("local", "name", "sparkHome", dummyJarFile).stop(); new JavaSparkContext("local", "name", "sparkHome", jars).stop(); new JavaSparkContext("local", "name", "sparkHome", jars, environment).stop(); } diff --git a/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala b/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala index 435b927068e60..1ba13c2ef1897 100644 --- a/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala +++ b/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala @@ -19,9 +19,12 @@ package org.apache.spark import scala.concurrent.duration._ +import org.apache.spark.TestUtils.createTempScriptWithExpectedOutput import org.apache.spark.internal.config._ import org.apache.spark.rdd.{PartitionPruningRDD, RDD} +import org.apache.spark.resource.TestResourceIDs.{EXECUTOR_GPU_ID, TASK_GPU_ID, WORKER_GPU_ID} import org.apache.spark.scheduler.BarrierJobAllocationFailed._ +import org.apache.spark.scheduler.BarrierJobSlotsNumberCheckFailed import org.apache.spark.util.ThreadUtils /** @@ -259,4 +262,37 @@ class BarrierStageOnSubmittedSuite extends SparkFunSuite with LocalSparkContext testSubmitJob(sc, rdd, message = ERROR_MESSAGE_BARRIER_REQUIRE_MORE_SLOTS_THAN_CURRENT_TOTAL_NUMBER) } + + test("SPARK-32518: CoarseGrainedSchedulerBackend.maxNumConcurrentTasks should " + + "consider all kinds of resources for the barrier stage") { + withTempDir { dir => + val discoveryScript = createTempScriptWithExpectedOutput( + dir, "gpuDiscoveryScript", """{"name": "gpu","addresses":["0"]}""") + + val conf = new SparkConf() + // Setup a local cluster which would only has one executor with 2 CPUs and 1 GPU. + .setMaster("local-cluster[1, 2, 1024]") + .setAppName("test-cluster") + .set(WORKER_GPU_ID.amountConf, "1") + .set(WORKER_GPU_ID.discoveryScriptConf, discoveryScript) + .set(EXECUTOR_GPU_ID.amountConf, "1") + .set(TASK_GPU_ID.amountConf, "1") + // disable barrier stage retry to fail the application as soon as possible + .set(BARRIER_MAX_CONCURRENT_TASKS_CHECK_MAX_FAILURES, 1) + sc = new SparkContext(conf) + TestUtils.waitUntilExecutorsUp(sc, 1, 60000) + + val exception = intercept[BarrierJobSlotsNumberCheckFailed] { + // Setup a barrier stage which contains 2 tasks and each task requires 1 CPU and 1 GPU. + // Therefore, the total resources requirement (2 CPUs and 2 GPUs) of this barrier stage + // can not be satisfied since the cluster only has 2 CPUs and 1 GPU in total. + sc.parallelize(Range(1, 10), 2) + .barrier() + .mapPartitions { iter => iter } + .collect() + } + assert(exception.getMessage.contains("[SPARK-24819]: Barrier execution " + + "mode does not allow run a barrier stage that requires more slots")) + } + } } diff --git a/core/src/test/scala/org/apache/spark/DistributedSuite.scala b/core/src/test/scala/org/apache/spark/DistributedSuite.scala index 3478b16325dd6..ce1df3adf6352 100644 --- a/core/src/test/scala/org/apache/spark/DistributedSuite.scala +++ b/core/src/test/scala/org/apache/spark/DistributedSuite.scala @@ -38,7 +38,7 @@ class DistributedSuite extends SparkFunSuite with Matchers with LocalSparkContex // Necessary to make ScalaTest 3.x interrupt a thread on the JVM like ScalaTest 2.2.x implicit val defaultSignaler: Signaler = ThreadSignaler - val clusterUrl = "local-cluster[2,1,1024]" + val clusterUrl = "local-cluster[3,1,1024]" test("task throws not serializable exception") { // Ensures that executors do not crash when an exn is not serializable. If executors crash, @@ -174,7 +174,7 @@ class DistributedSuite extends SparkFunSuite with Matchers with LocalSparkContex private def testCaching(conf: SparkConf, storageLevel: StorageLevel): Unit = { sc = new SparkContext(conf.setMaster(clusterUrl).setAppName("test")) - TestUtils.waitUntilExecutorsUp(sc, 2, 60000) + TestUtils.waitUntilExecutorsUp(sc, 3, 60000) val data = sc.parallelize(1 to 1000, 10) val cachedData = data.persist(storageLevel) assert(cachedData.count === 1000) @@ -206,7 +206,8 @@ class DistributedSuite extends SparkFunSuite with Matchers with LocalSparkContex "caching on disk" -> StorageLevel.DISK_ONLY, "caching in memory, replicated" -> StorageLevel.MEMORY_ONLY_2, "caching in memory, serialized, replicated" -> StorageLevel.MEMORY_ONLY_SER_2, - "caching on disk, replicated" -> StorageLevel.DISK_ONLY_2, + "caching on disk, replicated 2" -> StorageLevel.DISK_ONLY_2, + "caching on disk, replicated 3" -> StorageLevel.DISK_ONLY_3, "caching in memory and disk, replicated" -> StorageLevel.MEMORY_AND_DISK_2, "caching in memory and disk, serialized, replicated" -> StorageLevel.MEMORY_AND_DISK_SER_2 ).foreach { case (testName, storageLevel) => diff --git a/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala b/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala index 5b367d2fb01d4..6a38bba5dd0e5 100644 --- a/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala @@ -27,7 +27,8 @@ import org.scalatest.PrivateMethodTester import org.apache.spark.executor.ExecutorMetrics import org.apache.spark.internal.config -import org.apache.spark.internal.config.Tests.TEST_SCHEDULE_INTERVAL +import org.apache.spark.internal.config.DECOMMISSION_ENABLED +import org.apache.spark.internal.config.Tests.TEST_DYNAMIC_ALLOCATION_SCHEDULE_ENABLED import org.apache.spark.metrics.MetricsSystem import org.apache.spark.resource._ import org.apache.spark.resource.ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID @@ -1270,6 +1271,68 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite { assert(executorsPendingToRemove(manager).size === 6) // limit reached (1 executor remaining) } + test("mock polling loop remove with decommissioning") { + val clock = new ManualClock(2020L) + val manager = createManager(createConf(1, 20, 1, true), clock = clock) + + // Remove idle executors on timeout + onExecutorAddedDefaultProfile(manager, "executor-1") + onExecutorAddedDefaultProfile(manager, "executor-2") + onExecutorAddedDefaultProfile(manager, "executor-3") + assert(executorsDecommissioning(manager).isEmpty) + assert(executorsPendingToRemove(manager).isEmpty) + + // idle threshold not reached yet + clock.advance(executorIdleTimeout * 1000 / 2) + schedule(manager) + assert(manager.executorMonitor.timedOutExecutors().isEmpty) + assert(executorsPendingToRemove(manager).isEmpty) + assert(executorsDecommissioning(manager).isEmpty) + + // idle threshold exceeded + clock.advance(executorIdleTimeout * 1000) + assert(manager.executorMonitor.timedOutExecutors().size === 3) + schedule(manager) + assert(executorsPendingToRemove(manager).isEmpty) // limit reached (1 executor remaining) + assert(executorsDecommissioning(manager).size === 2) // limit reached (1 executor remaining) + + // Mark a subset as busy - only idle executors should be removed + onExecutorAddedDefaultProfile(manager, "executor-4") + onExecutorAddedDefaultProfile(manager, "executor-5") + onExecutorAddedDefaultProfile(manager, "executor-6") + onExecutorAddedDefaultProfile(manager, "executor-7") + assert(manager.executorMonitor.executorCount === 7) + assert(executorsPendingToRemove(manager).isEmpty) // no pending to be removed + assert(executorsDecommissioning(manager).size === 2) // 2 decommissioning + onExecutorBusy(manager, "executor-4") + onExecutorBusy(manager, "executor-5") + onExecutorBusy(manager, "executor-6") // 3 busy and 2 idle (of the 5 active ones) + + // after scheduling, the previously timed out executor should be removed, since + // there are new active ones. + schedule(manager) + assert(executorsDecommissioning(manager).size === 3) + + // advance the clock so that idle executors should time out and move to the pending list + clock.advance(executorIdleTimeout * 1000) + schedule(manager) + assert(executorsPendingToRemove(manager).size === 0) + assert(executorsDecommissioning(manager).size === 4) + assert(!executorsDecommissioning(manager).contains("executor-4")) + assert(!executorsDecommissioning(manager).contains("executor-5")) + assert(!executorsDecommissioning(manager).contains("executor-6")) + + // Busy executors are now idle and should be removed + onExecutorIdle(manager, "executor-4") + onExecutorIdle(manager, "executor-5") + onExecutorIdle(manager, "executor-6") + schedule(manager) + assert(executorsDecommissioning(manager).size === 4) + clock.advance(executorIdleTimeout * 1000) + schedule(manager) + assert(executorsDecommissioning(manager).size === 6) // limit reached (1 executor remaining) + } + test("listeners trigger add executors correctly") { val manager = createManager(createConf(1, 20, 1)) assert(addTime(manager) === NOT_SET) @@ -1588,7 +1651,8 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite { private def createConf( minExecutors: Int = 1, maxExecutors: Int = 5, - initialExecutors: Int = 1): SparkConf = { + initialExecutors: Int = 1, + decommissioningEnabled: Boolean = false): SparkConf = { val sparkConf = new SparkConf() .set(config.DYN_ALLOCATION_ENABLED, true) .set(config.DYN_ALLOCATION_MIN_EXECUTORS, minExecutors) @@ -1601,9 +1665,11 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite { .set(config.DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT.key, s"${executorIdleTimeout.toString}s") .set(config.SHUFFLE_SERVICE_ENABLED, true) .set(config.DYN_ALLOCATION_TESTING, true) - // SPARK-22864: effectively disable the allocation schedule by setting the period to a - // really long value. - .set(TEST_SCHEDULE_INTERVAL, 30000L) + // SPARK-22864/SPARK-32287: effectively disable the allocation schedule for the tests so that + // we won't result in the race condition between thread "spark-dynamic-executor-allocation" + // and thread "pool-1-thread-1-ScalaTest-running". + .set(TEST_DYNAMIC_ALLOCATION_SCHEDULE_ENABLED, false) + .set(DECOMMISSION_ENABLED, decommissioningEnabled) sparkConf } @@ -1670,6 +1736,10 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite { private def executorsPendingToRemove(manager: ExecutorAllocationManager): Set[String] = { manager.executorMonitor.executorsPendingToRemove() } + + private def executorsDecommissioning(manager: ExecutorAllocationManager): Set[String] = { + manager.executorMonitor.executorsDecommissioning() + } } /** diff --git a/core/src/test/scala/org/apache/spark/ExternalShuffleServiceSuite.scala b/core/src/test/scala/org/apache/spark/ExternalShuffleServiceSuite.scala index 9026447e5a98c..48c1cc5906f30 100644 --- a/core/src/test/scala/org/apache/spark/ExternalShuffleServiceSuite.scala +++ b/core/src/test/scala/org/apache/spark/ExternalShuffleServiceSuite.scala @@ -100,50 +100,6 @@ class ExternalShuffleServiceSuite extends ShuffleSuite with BeforeAndAfterAll wi e.getMessage should include ("Fetch failure will not retry stage due to testing config") } - test("SPARK-27651: read host local shuffle blocks from disk and avoid network remote fetches") { - val confWithHostLocalRead = - conf.clone.set(config.SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED, true) - confWithHostLocalRead.set(config.STORAGE_LOCAL_DISK_BY_EXECUTORS_CACHE_SIZE, 5) - sc = new SparkContext("local-cluster[2,1,1024]", "test", confWithHostLocalRead) - sc.getConf.get(config.SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED) should equal(true) - sc.env.blockManager.externalShuffleServiceEnabled should equal(true) - sc.env.blockManager.hostLocalDirManager.isDefined should equal(true) - sc.env.blockManager.blockStoreClient.getClass should equal(classOf[ExternalBlockStoreClient]) - - // In a slow machine, one executor may register hundreds of milliseconds ahead of the other one. - // If we don't wait for all executors, it's possible that only one executor runs all jobs. Then - // all shuffle blocks will be in this executor, ShuffleBlockFetcherIterator will directly fetch - // local blocks from the local BlockManager and won't send requests to ExternalShuffleService. - // In this case, we won't receive FetchFailed. And it will make this test fail. - // Therefore, we should wait until all executors are up - TestUtils.waitUntilExecutorsUp(sc, 2, 60000) - - val rdd = sc.parallelize(0 until 1000, 10) - .map { i => (i, 1) } - .reduceByKey(_ + _) - - rdd.count() - rdd.count() - - val cachedExecutors = rdd.mapPartitions { _ => - SparkEnv.get.blockManager.hostLocalDirManager.map { localDirManager => - localDirManager.getCachedHostLocalDirs().keySet.iterator - }.getOrElse(Iterator.empty) - }.collect().toSet - - // both executors are caching the dirs of the other one - cachedExecutors should equal(sc.getExecutorIds().toSet) - - // Invalidate the registered executors, disallowing access to their shuffle blocks (without - // deleting the actual shuffle files, so we could access them without the shuffle service). - // As directories are already cached there is no request to external shuffle service. - rpcHandler.applicationRemoved(sc.conf.getAppId, false /* cleanupLocalDirs */) - - // Now Spark will not receive FetchFailed as host local blocks are read from the cached local - // disk directly - rdd.collect().map(_._2).sum should equal(1000) - } - test("SPARK-25888: using external shuffle service fetching disk persisted blocks") { val confWithRddFetchEnabled = conf.clone.set(config.SHUFFLE_SERVICE_FETCH_RDD_ENABLED, true) sc = new SparkContext("local-cluster[1,1,1024]", "test", confWithRddFetchEnabled) diff --git a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala index 581786ce0a70c..552b0d8e49a03 100644 --- a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala @@ -24,6 +24,7 @@ import java.util.{Locale, TimeZone} import org.apache.log4j.spi.LoggingEvent import scala.annotation.tailrec +import org.apache.commons.io.FileUtils import org.apache.log4j.{Appender, AppenderSkeleton, Level, Logger} import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, BeforeAndAfterEach, Outcome} import org.scalatest.funsuite.AnyFunSuite @@ -65,6 +66,12 @@ abstract class SparkFunSuite with Logging { // scalastyle:on + // Initialize the logger forcibly to let the logger log timestamp + // based on the local time zone depending on environments. + // The default time zone will be set to America/Los_Angeles later + // so this initialization is necessary here. + log + // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*) TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")) // Add Locale setting @@ -101,6 +108,17 @@ abstract class SparkFunSuite getTestResourceFile(file).getCanonicalPath } + protected final def copyAndGetResourceFile(fileName: String, suffix: String): File = { + val url = Thread.currentThread().getContextClassLoader.getResource(fileName) + // To avoid illegal accesses to a resource file inside jar + // (URISyntaxException might be thrown when accessing it), + // copy it into a temporary one for accessing it from the dependent module. + val file = File.createTempFile("test-resource", suffix) + file.deleteOnExit() + FileUtils.copyURLToFile(url, file) + file + } + /** * Note: this method doesn't support `BeforeAndAfter`. You must use `BeforeAndAfterEach` to * set up and tear down resources. diff --git a/core/src/test/scala/org/apache/spark/TempLocalSparkContext.scala b/core/src/test/scala/org/apache/spark/TempLocalSparkContext.scala new file mode 100644 index 0000000000000..6d5fcd1edfb03 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/TempLocalSparkContext.scala @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark + +import _root_.io.netty.util.internal.logging.{InternalLoggerFactory, Slf4JLoggerFactory} +import org.scalatest.BeforeAndAfterAll +import org.scalatest.BeforeAndAfterEach +import org.scalatest.Suite + +import org.apache.spark.internal.Logging +import org.apache.spark.resource.ResourceProfile + +/** + * Manages a local `sc` `SparkContext` variable, correctly stopping it after each test. + * + * Note: this class is a copy of [[LocalSparkContext]]. Why copy it? Reduce conflict. Because + * many test suites use [[LocalSparkContext]] and overwrite some variable or function (e.g. + * sc of LocalSparkContext), there occurs conflict when we refactor the `sc` as a new function. + * After migrating all test suites that use [[LocalSparkContext]] to use + * [[TempLocalSparkContext]], we will delete the original [[LocalSparkContext]] and rename + * [[TempLocalSparkContext]] to [[LocalSparkContext]]. + */ +trait TempLocalSparkContext extends BeforeAndAfterEach + with BeforeAndAfterAll with Logging { self: Suite => + + private var _conf: SparkConf = defaultSparkConf + + @transient private var _sc: SparkContext = _ + + def conf: SparkConf = _conf + + /** + * Currently, we are focusing on the reconstruction of LocalSparkContext, so this method + * was created temporarily. When the migration work is completed, this method will be + * renamed to `sc` and the variable `sc` will be deleted. + */ + def sc: SparkContext = { + if (_sc == null) { + _sc = new SparkContext(_conf) + } + _sc + } + + override def beforeAll(): Unit = { + super.beforeAll() + InternalLoggerFactory.setDefaultFactory(Slf4JLoggerFactory.INSTANCE) + } + + override def afterEach(): Unit = { + try { + resetSparkContext() + } finally { + super.afterEach() + } + } + + def resetSparkContext(): Unit = { + TempLocalSparkContext.stop(_sc) + ResourceProfile.clearDefaultProfile() + _sc = null + _conf = defaultSparkConf + } + + private def defaultSparkConf: SparkConf = new SparkConf() + .setMaster("local[2]").setAppName(s"${this.getClass.getSimpleName}") +} + +object TempLocalSparkContext { + def stop(sc: SparkContext): Unit = { + if (sc != null) { + sc.stop() + } + // To avoid RPC rebinding to the same port, since it doesn't unbind immediately on shutdown + System.clearProperty("spark.driver.port") + } + + /** Runs `f` by passing in `sc` and ensures that `sc` is stopped. */ + def withSpark[T](sc: SparkContext)(f: SparkContext => T): T = { + try { + f(sc) + } finally { + stop(sc) + } + } +} diff --git a/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala b/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala index 72c05a92848ff..5511852ca176e 100644 --- a/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala +++ b/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala @@ -26,7 +26,6 @@ import scala.util.Try import org.apache.commons.io.output.TeeOutputStream import org.apache.commons.lang3.SystemUtils -import org.scalatest.Assertions._ import org.apache.spark.util.Utils diff --git a/core/src/test/scala/org/apache/spark/deploy/DecommissionWorkerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/DecommissionWorkerSuite.scala index ee9a6be03868f..abe5b7a71ca63 100644 --- a/core/src/test/scala/org/apache/spark/deploy/DecommissionWorkerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/DecommissionWorkerSuite.scala @@ -28,7 +28,7 @@ import org.scalatest.BeforeAndAfterEach import org.scalatest.concurrent.Eventually._ import org.apache.spark._ -import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, RequestMasterState, WorkerDecommission} +import org.apache.spark.deploy.DeployMessages.{DecommissionWorkers, MasterStateResponse, RequestMasterState} import org.apache.spark.deploy.master.{ApplicationInfo, Master, WorkerInfo} import org.apache.spark.deploy.worker.Worker import org.apache.spark.internal.{config, Logging} @@ -57,7 +57,7 @@ class DecommissionWorkerSuite override def beforeEach(): Unit = { super.beforeEach() masterAndWorkerConf = new SparkConf() - .set(config.Worker.WORKER_DECOMMISSION_ENABLED, true) + .set(config.DECOMMISSION_ENABLED, true) masterAndWorkerSecurityManager = new SecurityManager(masterAndWorkerConf) masterRpcEnv = RpcEnv.create( Master.SYSTEM_NAME, @@ -84,6 +84,19 @@ class DecommissionWorkerSuite } } + // Unlike TestUtils.withListener, it also waits for the job to be done + def withListener(sc: SparkContext, listener: RootStageAwareListener) + (body: SparkListener => Unit): Unit = { + sc.addSparkListener(listener) + try { + body(listener) + sc.listenerBus.waitUntilEmpty() + listener.waitForJobDone() + } finally { + sc.listenerBus.removeListener(listener) + } + } + test("decommission workers should not result in job failure") { val maxTaskFailures = 2 val numTimesToKillWorkers = maxTaskFailures + 1 @@ -109,7 +122,7 @@ class DecommissionWorkerSuite } } } - TestUtils.withListener(sc, listener) { _ => + withListener(sc, listener) { _ => val jobResult = sc.parallelize(1 to 1, 1).map { _ => Thread.sleep(5 * 1000L); 1 }.count() @@ -164,7 +177,7 @@ class DecommissionWorkerSuite } } } - TestUtils.withListener(sc, listener) { _ => + withListener(sc, listener) { _ => val jobResult = sc.parallelize(1 to 2, 2).mapPartitionsWithIndex((pid, _) => { val sleepTimeSeconds = if (pid == 0) 1 else 10 Thread.sleep(sleepTimeSeconds * 1000L) @@ -190,10 +203,11 @@ class DecommissionWorkerSuite } } - test("decommission workers ensure that fetch failures lead to rerun") { + def testFetchFailures(initialSleepMillis: Int): Unit = { createWorkers(2) sc = createSparkContext( config.Tests.TEST_NO_STAGE_RETRY.key -> "false", + "spark.test.executor.decommission.initial.sleep.millis" -> initialSleepMillis.toString, config.UNREGISTER_OUTPUT_ON_HOST_ON_FETCH_FAILURE.key -> "true") val executorIdToWorkerInfo = getExecutorToWorkerAssignments val executorToDecom = executorIdToWorkerInfo.keysIterator.next @@ -212,22 +226,29 @@ class DecommissionWorkerSuite override def handleRootTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = { val taskInfo = taskEnd.taskInfo if (taskInfo.executorId == executorToDecom && taskInfo.attemptNumber == 0 && - taskEnd.stageAttemptId == 0) { + taskEnd.stageAttemptId == 0 && taskEnd.stageId == 0) { decommissionWorkerOnMaster(workerToDecom, "decommission worker after task on it is done") } } } - TestUtils.withListener(sc, listener) { _ => + withListener(sc, listener) { _ => val jobResult = sc.parallelize(1 to 2, 2).mapPartitionsWithIndex((_, _) => { val executorId = SparkEnv.get.executorId - val sleepTimeSeconds = if (executorId == executorToDecom) 10 else 1 - Thread.sleep(sleepTimeSeconds * 1000L) + val context = TaskContext.get() + // Only sleep in the first attempt to create the required window for decommissioning. + // Subsequent attempts don't need to be delayed to speed up the test. + if (context.attemptNumber() == 0 && context.stageAttemptNumber() == 0) { + val sleepTimeSeconds = if (executorId == executorToDecom) 10 else 1 + Thread.sleep(sleepTimeSeconds * 1000L) + } List(1).iterator }, preservesPartitioning = true) .repartition(1).mapPartitions(iter => { val context = TaskContext.get() if (context.attemptNumber == 0 && context.stageAttemptNumber() == 0) { + // Wait a bit for the decommissioning to be triggered in the listener + Thread.sleep(5000) // MapIndex is explicitly -1 to force the entire host to be decommissioned // However, this will cause both the tasks in the preceding stage since the host here is // "localhost" (shortcoming of this single-machine unit test in that all the workers @@ -246,6 +267,14 @@ class DecommissionWorkerSuite assert(tasksSeen.size === 6, s"Expected 6 tasks but got $tasksSeen") } + test("decommission stalled workers ensure that fetch failures lead to rerun") { + testFetchFailures(3600 * 1000) + } + + test("decommission eager workers ensure that fetch failures lead to rerun") { + testFetchFailures(0) + } + private abstract class RootStageAwareListener extends SparkListener { private var rootStageId: Option[Int] = None private val tasksFinished = new ConcurrentLinkedQueue[String]() @@ -265,6 +294,7 @@ class DecommissionWorkerSuite override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = { jobEnd.jobResult match { case JobSucceeded => jobDone.set(true) + case JobFailed(exception) => logError(s"Job failed", exception) } } @@ -272,7 +302,15 @@ class DecommissionWorkerSuite protected def handleRootTaskStart(start: SparkListenerTaskStart) = {} + private def getSignature(taskInfo: TaskInfo, stageId: Int, stageAttemptId: Int): + String = { + s"${stageId}:${stageAttemptId}:" + + s"${taskInfo.index}:${taskInfo.attemptNumber}-${taskInfo.status}" + } + override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = { + val signature = getSignature(taskStart.taskInfo, taskStart.stageId, taskStart.stageAttemptId) + logInfo(s"Task started: $signature") if (isRootStageId(taskStart.stageId)) { rootTasksStarted.add(taskStart.taskInfo) handleRootTaskStart(taskStart) @@ -280,8 +318,7 @@ class DecommissionWorkerSuite } override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = { - val taskSignature = s"${taskEnd.stageId}:${taskEnd.stageAttemptId}:" + - s"${taskEnd.taskInfo.index}:${taskEnd.taskInfo.attemptNumber}" + val taskSignature = getSignature(taskEnd.taskInfo, taskEnd.stageId, taskEnd.stageAttemptId) logInfo(s"Task End $taskSignature") tasksFinished.add(taskSignature) if (isRootStageId(taskEnd.stageId)) { @@ -291,8 +328,13 @@ class DecommissionWorkerSuite } def getTasksFinished(): Seq[String] = { - assert(jobDone.get(), "Job isn't successfully done yet") - tasksFinished.asScala.toSeq + tasksFinished.asScala.toList + } + + def waitForJobDone(): Unit = { + eventually(timeout(10.seconds), interval(100.milliseconds)) { + assert(jobDone.get(), "Job isn't successfully done yet") + } } } @@ -372,7 +414,7 @@ class DecommissionWorkerSuite def decommissionWorkerOnMaster(workerInfo: WorkerInfo, reason: String): Unit = { logInfo(s"Trying to decommission worker ${workerInfo.id} for reason `$reason`") - master.self.send(WorkerDecommission(workerInfo.id, workerInfo.endpoint)) + master.self.send(DecommissionWorkers(Seq(workerInfo.id))) } def killWorkerAfterTimeout(workerInfo: WorkerInfo, secondsToWait: Int): Unit = { diff --git a/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala b/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala index d681c13337e0d..ea4d252f0dbae 100644 --- a/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala @@ -61,7 +61,8 @@ class ExternalShuffleServiceMetricsSuite extends SparkFunSuite { "registeredExecutorsSize", "registerExecutorRequestLatencyMillis", "shuffle-server.usedDirectMemory", - "shuffle-server.usedHeapMemory") + "shuffle-server.usedHeapMemory", + "finalizeShuffleMergeLatencyMillis") ) } } diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 35311d372e478..b5b3751439750 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -570,7 +570,8 @@ class SparkSubmitSuite } } - val clArgs2 = Seq("--class", "org.SomeClass", "thejar.jar") + val dummyJarFile = TestUtils.createJarWithClasses(Seq.empty) + val clArgs2 = Seq("--class", "org.SomeClass", dummyJarFile.toString) val appArgs2 = new SparkSubmitArguments(clArgs2) val (_, _, conf2, _) = submit.prepareSubmitEnvironment(appArgs2) assert(!conf2.contains(UI_SHOW_CONSOLE_PROGRESS)) @@ -1220,6 +1221,86 @@ class SparkSubmitSuite testRemoteResources(enableHttpFs = true, forceDownloadSchemes = Seq("*")) } + test("SPARK-32119: Jars and files should be loaded when Executors launch for plugins") { + val tempDir = Utils.createTempDir() + val tempFileName = "test.txt" + val tempFile = new File(tempDir, tempFileName) + + // scalastyle:off println + Utils.tryWithResource { + new PrintWriter(tempFile) + } { writer => + writer.println("SparkPluginTest") + } + // scalastyle:on println + + val sparkPluginCodeBody = + """ + |@Override + |public org.apache.spark.api.plugin.ExecutorPlugin executorPlugin() { + | return new TestExecutorPlugin(); + |} + | + |@Override + |public org.apache.spark.api.plugin.DriverPlugin driverPlugin() { return null; } + """.stripMargin + val executorPluginCodeBody = + s""" + |@Override + |public void init( + | org.apache.spark.api.plugin.PluginContext ctx, + | java.util.Map extraConf) { + | String str = null; + | try (java.io.BufferedReader reader = + | new java.io.BufferedReader(new java.io.InputStreamReader( + | new java.io.FileInputStream("$tempFileName")))) { + | str = reader.readLine(); + | } catch (java.io.IOException e) { + | throw new RuntimeException(e); + | } finally { + | assert str == "SparkPluginTest"; + | } + |} + """.stripMargin + + val compiledExecutorPlugin = TestUtils.createCompiledClass( + "TestExecutorPlugin", + tempDir, + "", + null, + Seq.empty, + Seq("org.apache.spark.api.plugin.ExecutorPlugin"), + executorPluginCodeBody) + + val thisClassPath = + sys.props("java.class.path").split(File.pathSeparator).map(p => new File(p).toURI.toURL) + val compiledSparkPlugin = TestUtils.createCompiledClass( + "TestSparkPlugin", + tempDir, + "", + null, + Seq(tempDir.toURI.toURL) ++ thisClassPath, + Seq("org.apache.spark.api.plugin.SparkPlugin"), + sparkPluginCodeBody) + + val jarUrl = TestUtils.createJar( + Seq(compiledSparkPlugin, compiledExecutorPlugin), + new File(tempDir, "testplugin.jar")) + + val unusedJar = TestUtils.createJarWithClasses(Seq.empty) + val unusedFile = Files.createTempFile(tempDir.toPath, "unused", null) + val args = Seq( + "--class", SimpleApplicationTest.getClass.getName.stripSuffix("$"), + "--name", "testApp", + "--master", "local-cluster[1,1,1024]", + "--conf", "spark.plugins=TestSparkPlugin", + "--conf", "spark.ui.enabled=false", + "--jars", jarUrl.toString + "," + unusedJar.toString, + "--files", tempFile.toString + "," + unusedFile.toString, + unusedJar.toString) + runSparkSubmit(args) + } + private def testRemoteResources( enableHttpFs: Boolean, forceDownloadSchemes: Seq[String] = Nil): Unit = { diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala index 31e6c730eadc0..2a37f75d86a41 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala @@ -79,7 +79,7 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll { test("create additional resolvers") { val repos = "a/1,b/2,c/3" - val settings = SparkSubmitUtils.buildIvySettings(Option(repos), None) + val settings = SparkSubmitUtils.buildIvySettings(Option(repos), Some(tempIvyPath)) val resolver = settings.getDefaultResolver.asInstanceOf[ChainResolver] assert(resolver.getResolvers.size() === 4) val expected = repos.split(",").map(r => s"$r/") @@ -134,7 +134,7 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll { // end to end val jarPath = SparkSubmitUtils.resolveMavenCoordinates( main.toString, - SparkSubmitUtils.buildIvySettings(Option(repo), Option(tempIvyPath)), + SparkSubmitUtils.buildIvySettings(Option(repo), Some(tempIvyPath)), isTest = true) assert(jarPath.indexOf(tempIvyPath) >= 0, "should use non-default ivy path") } @@ -147,7 +147,7 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll { IvyTestUtils.withRepository(main, Some(dep), Some(SparkSubmitUtils.m2Path)) { repo => val jarPath = SparkSubmitUtils.resolveMavenCoordinates( main.toString, - SparkSubmitUtils.buildIvySettings(None, None), + SparkSubmitUtils.buildIvySettings(None, Some(tempIvyPath)), isTest = true) assert(jarPath.indexOf("mylib") >= 0, "should find artifact") assert(jarPath.indexOf("mydep") >= 0, "should find dependency") @@ -158,7 +158,7 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll { IvyTestUtils.withRepository(main, Some(dep), Some(ivyLocal), useIvyLayout = true) { repo => val jarPath = SparkSubmitUtils.resolveMavenCoordinates( main.toString, - SparkSubmitUtils.buildIvySettings(None, None), + SparkSubmitUtils.buildIvySettings(None, Some(tempIvyPath)), isTest = true) assert(jarPath.indexOf("mylib") >= 0, "should find artifact") assert(jarPath.indexOf("mydep") >= 0, "should find dependency") @@ -182,7 +182,7 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll { intercept[RuntimeException] { SparkSubmitUtils.resolveMavenCoordinates( "a:b:c", - SparkSubmitUtils.buildIvySettings(None, None), + SparkSubmitUtils.buildIvySettings(None, Some(tempIvyPath)), isTest = true) } } @@ -194,14 +194,14 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll { val path = SparkSubmitUtils.resolveMavenCoordinates( coordinates, - SparkSubmitUtils.buildIvySettings(None, None), + SparkSubmitUtils.buildIvySettings(None, Some(tempIvyPath)), isTest = true) assert(path === "", "should return empty path") val main = MavenCoordinate("org.apache.spark", "spark-streaming-kafka-assembly_2.12", "1.2.0") IvyTestUtils.withRepository(main, None, None) { repo => val files = SparkSubmitUtils.resolveMavenCoordinates( coordinates + "," + main.toString, - SparkSubmitUtils.buildIvySettings(Some(repo), None), + SparkSubmitUtils.buildIvySettings(Some(repo), Some(tempIvyPath)), isTest = true) assert(files.indexOf(main.artifactId) >= 0, "Did not return artifact") } @@ -213,7 +213,7 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll { IvyTestUtils.withRepository(main, Some(dep), None) { repo => val files = SparkSubmitUtils.resolveMavenCoordinates( main.toString, - SparkSubmitUtils.buildIvySettings(Some(repo), None), + SparkSubmitUtils.buildIvySettings(Some(repo), Some(tempIvyPath)), Seq("my.great.dep:mydep"), isTest = true) assert(files.indexOf(main.artifactId) >= 0, "Did not return artifact") diff --git a/core/src/test/scala/org/apache/spark/deploy/client/AppClientSuite.scala b/core/src/test/scala/org/apache/spark/deploy/client/AppClientSuite.scala index e091bd05c2dc8..93c0aa000e207 100644 --- a/core/src/test/scala/org/apache/spark/deploy/client/AppClientSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/client/AppClientSuite.scala @@ -27,7 +27,7 @@ import org.scalatest.concurrent.{Eventually, ScalaFutures} import org.apache.spark._ import org.apache.spark.deploy.{ApplicationDescription, Command} -import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, RequestMasterState} +import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, RequestMasterState, WorkerDecommissioning} import org.apache.spark.deploy.master.{ApplicationInfo, Master} import org.apache.spark.deploy.worker.Worker import org.apache.spark.internal.{config, Logging} @@ -59,7 +59,7 @@ class AppClientSuite */ override def beforeAll(): Unit = { super.beforeAll() - conf = new SparkConf().set(config.Worker.WORKER_DECOMMISSION_ENABLED.key, "true") + conf = new SparkConf().set(config.DECOMMISSION_ENABLED.key, "true") securityManager = new SecurityManager(conf) masterRpcEnv = RpcEnv.create(Master.SYSTEM_NAME, "localhost", 0, conf, securityManager) workerRpcEnvs = (0 until numWorkers).map { i => @@ -122,14 +122,18 @@ class AppClientSuite // Send a decommission self to all the workers // Note: normally the worker would send this on their own. - workers.foreach(worker => worker.decommissionSelf()) + workers.foreach { worker => + worker.decommissionSelf() + // send the notice to Master to tell the decommission of Workers + master.self.send(WorkerDecommissioning(worker.workerId, worker.self)) + } // Decommissioning is async. eventually(timeout(1.seconds), interval(10.millis)) { // We only record decommissioning for the executor we've requested assert(ci.listener.execDecommissionedMap.size === 1) val decommissionInfo = ci.listener.execDecommissionedMap.get(executorId) - assert(decommissionInfo != null && decommissionInfo.isHostDecommissioned, + assert(decommissionInfo != null && decommissionInfo.workerHost.isDefined, s"$executorId should have been decommissioned along with its worker") } @@ -245,7 +249,7 @@ class AppClientSuite } def executorRemoved( - id: String, message: String, exitStatus: Option[Int], workerLost: Boolean): Unit = { + id: String, message: String, exitStatus: Option[Int], workerHost: Option[String]): Unit = { execRemovedList.add(id) } diff --git a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala index 060b878fb8ef2..e9b739ce7a4c6 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala @@ -213,7 +213,7 @@ class SingleEventLogFileWriterSuite extends EventLogFileWritersSuite { compressionCodecShortName) val finalLogPath = new Path(logPath) - assert(fileSystem.exists(finalLogPath) && fileSystem.isFile(finalLogPath)) + assert(fileSystem.exists(finalLogPath) && fileSystem.getFileStatus(finalLogPath).isFile) assert(expectedLines === readLinesFromEventLogFile(finalLogPath, fileSystem)) } } @@ -357,10 +357,10 @@ class RollingEventLogFilesWriterSuite extends EventLogFileWritersSuite { expectedLines: Seq[String]): Unit = { val logDirPath = getAppEventLogDirPath(logBaseDir, appId, appAttemptId) - assert(fileSystem.exists(logDirPath) && fileSystem.isDirectory(logDirPath)) + assert(fileSystem.exists(logDirPath) && fileSystem.getFileStatus(logDirPath).isDirectory) val appStatusFile = getAppStatusFilePath(logDirPath, appId, appAttemptId, inProgress = false) - assert(fileSystem.exists(appStatusFile) && fileSystem.isFile(appStatusFile)) + assert(fileSystem.exists(appStatusFile) && fileSystem.getFileStatus(appStatusFile).isFile) val eventLogFiles = listEventLogFiles(logDirPath) val allLines = mutable.ArrayBuffer[String]() diff --git a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala index 21a99a462aa1e..0b0754be2f56f 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala @@ -44,7 +44,7 @@ import org.apache.spark.deploy.history.EventLogTestHelper._ import org.apache.spark.internal.Logging import org.apache.spark.internal.config.DRIVER_LOG_DFS_DIR import org.apache.spark.internal.config.History._ -import org.apache.spark.internal.config.UI.{ADMIN_ACLS, ADMIN_ACLS_GROUPS, USER_GROUPS_MAPPING} +import org.apache.spark.internal.config.UI.{ADMIN_ACLS, ADMIN_ACLS_GROUPS, UI_VIEW_ACLS, UI_VIEW_ACLS_GROUPS, USER_GROUPS_MAPPING} import org.apache.spark.io._ import org.apache.spark.scheduler._ import org.apache.spark.scheduler.cluster.ExecutorInfo @@ -90,9 +90,13 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging { } } - private def testAppLogParsing(inMemory: Boolean): Unit = { + test("SPARK-31608: parse application logs with HybridStore") { + testAppLogParsing(false, true) + } + + private def testAppLogParsing(inMemory: Boolean, useHybridStore: Boolean = false): Unit = { val clock = new ManualClock(12345678) - val conf = createTestConf(inMemory = inMemory) + val conf = createTestConf(inMemory = inMemory, useHybridStore = useHybridStore) val provider = new FsHistoryProvider(conf, clock) // Write a new-style application log. @@ -1471,6 +1475,107 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging { } } + test("SPARK-33146: don't let one bad rolling log folder prevent loading other applications") { + withTempDir { dir => + val conf = createTestConf(true) + conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath) + val hadoopConf = SparkHadoopUtil.newConfiguration(conf) + val fs = new Path(dir.getAbsolutePath).getFileSystem(hadoopConf) + + val provider = new FsHistoryProvider(conf) + + val writer = new RollingEventLogFilesWriter("app", None, dir.toURI, conf, hadoopConf) + writer.start() + + writeEventsToRollingWriter(writer, Seq( + SparkListenerApplicationStart("app", Some("app"), 0, "user", None), + SparkListenerJobStart(1, 0, Seq.empty)), rollFile = false) + provider.checkForLogs() + provider.cleanLogs() + assert(dir.listFiles().size === 1) + assert(provider.getListing.length === 1) + + // Manually delete the appstatus file to make an invalid rolling event log + val appStatusPath = RollingEventLogFilesWriter.getAppStatusFilePath(new Path(writer.logPath), + "app", None, true) + fs.delete(appStatusPath, false) + provider.checkForLogs() + provider.cleanLogs() + assert(provider.getListing.length === 0) + + // Create a new application + val writer2 = new RollingEventLogFilesWriter("app2", None, dir.toURI, conf, hadoopConf) + writer2.start() + writeEventsToRollingWriter(writer2, Seq( + SparkListenerApplicationStart("app2", Some("app2"), 0, "user", None), + SparkListenerJobStart(1, 0, Seq.empty)), rollFile = false) + + // Both folders exist but only one application found + provider.checkForLogs() + provider.cleanLogs() + assert(provider.getListing.length === 1) + assert(dir.listFiles().size === 2) + + // Make sure a new provider sees the valid application + provider.stop() + val newProvider = new FsHistoryProvider(conf) + newProvider.checkForLogs() + assert(newProvider.getListing.length === 1) + } + } + + test("SPARK-33215: check ui view permissions without retrieving ui") { + val conf = createTestConf() + .set(HISTORY_SERVER_UI_ACLS_ENABLE, true) + .set(HISTORY_SERVER_UI_ADMIN_ACLS, Seq("user1", "user2")) + .set(HISTORY_SERVER_UI_ADMIN_ACLS_GROUPS, Seq("group1")) + .set(USER_GROUPS_MAPPING, classOf[TestGroupsMappingProvider].getName) + + val provider = new FsHistoryProvider(conf) + val log = newLogFile("app1", Some("attempt1"), inProgress = false) + writeFile(log, None, + SparkListenerApplicationStart("app1", Some("app1"), System.currentTimeMillis(), + "test", Some("attempt1")), + SparkListenerEnvironmentUpdate(Map( + "Spark Properties" -> List((UI_VIEW_ACLS.key, "user"), (UI_VIEW_ACLS_GROUPS.key, "group")), + "Hadoop Properties" -> Seq.empty, + "JVM Information" -> Seq.empty, + "System Properties" -> Seq.empty, + "Classpath Entries" -> Seq.empty + )), + SparkListenerApplicationEnd(System.currentTimeMillis())) + + provider.checkForLogs() + + // attempt2 doesn't exist + intercept[NoSuchElementException] { + provider.checkUIViewPermissions("app1", Some("attempt2"), "user1") + } + // app2 doesn't exist + intercept[NoSuchElementException] { + provider.checkUIViewPermissions("app2", Some("attempt1"), "user1") + } + + // user1 and user2 are admins + assert(provider.checkUIViewPermissions("app1", Some("attempt1"), "user1")) + assert(provider.checkUIViewPermissions("app1", Some("attempt1"), "user2")) + // user3 is a member of admin group "group1" + assert(provider.checkUIViewPermissions("app1", Some("attempt1"), "user3")) + // test is the app owner + assert(provider.checkUIViewPermissions("app1", Some("attempt1"), "test")) + // user is in the app's view acls + assert(provider.checkUIViewPermissions("app1", Some("attempt1"), "user")) + // user5 is a member of the app's view acls group "group" + assert(provider.checkUIViewPermissions("app1", Some("attempt1"), "user5")) + + // abc, user6, user7 don't have permissions + assert(!provider.checkUIViewPermissions("app1", Some("attempt1"), "abc")) + assert(!provider.checkUIViewPermissions("app1", Some("attempt1"), "user6")) + assert(!provider.checkUIViewPermissions("app1", Some("attempt1"), "user7")) + + provider.stop() + } + /** * Asks the provider to check for logs and calls a function to perform checks on the updated * app list. Example: @@ -1509,7 +1614,9 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging { new FileOutputStream(file).close() } - private def createTestConf(inMemory: Boolean = false): SparkConf = { + private def createTestConf( + inMemory: Boolean = false, + useHybridStore: Boolean = false): SparkConf = { val conf = new SparkConf() .set(HISTORY_LOG_DIR, testDir.getAbsolutePath()) .set(FAST_IN_PROGRESS_PARSING, true) @@ -1517,6 +1624,7 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging { if (!inMemory) { conf.set(LOCAL_STORE_DIR, Utils.createTempDir().getAbsolutePath()) } + conf.set(HYBRID_STORE_ENABLED, useHybridStore) conf } diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerMemoryManagerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerMemoryManagerSuite.scala new file mode 100644 index 0000000000000..697f8f72624e5 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerMemoryManagerSuite.scala @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.deploy.history + +import org.apache.spark.{SparkConf, SparkFunSuite} +import org.apache.spark.internal.config.History._ + +class HistoryServerMemoryManagerSuite extends SparkFunSuite { + + private val MAX_USAGE = 3L + + test("lease and release memory") { + val conf = new SparkConf().set(MAX_IN_MEMORY_STORE_USAGE, MAX_USAGE) + val manager = new HistoryServerMemoryManager(conf) + + // Memory usage estimation for non-compressed log file is filesize / 2 + manager.lease("app1", None, 2L, None) + manager.lease("app2", None, 2L, None) + manager.lease("app3", None, 2L, None) + assert(manager.currentUsage.get === 3L) + assert(manager.active.size === 3) + assert(manager.active.get(("app1", None)) === Some(1L)) + + intercept[RuntimeException] { + manager.lease("app4", None, 2L, None) + } + + // Releasing a non-existent app is a no-op + manager.release("app4", None) + assert(manager.currentUsage.get === 3L) + + manager.release("app1", None) + assert(manager.currentUsage.get === 2L) + assert(manager.active.size === 2) + + manager.lease("app4", None, 2L, None) + assert(manager.currentUsage.get === 3L) + assert(manager.active.size === 3) + } +} diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index 51e38f9cdcd2d..e4c23d3d1b1c3 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -584,6 +584,24 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers } } + test("SPARK-33215: speed up event log download by skipping UI rebuild") { + val appId = "local-1430917381535" + + stop() + init() + + val port = server.boundPort + val testUrls = Seq( + s"http://localhost:$port/api/v1/applications/$appId/logs", + s"http://localhost:$port/api/v1/applications/$appId/1/logs", + s"http://localhost:$port/api/v1/applications/$appId/2/logs") + + testUrls.foreach { url => + TestUtils.httpResponseCode(new URL(url)) + } + assert(server.cacheMetrics.loadCount.getCount === 0, "downloading event log shouldn't load ui") + } + test("access history application defaults to the last attempt id") { def getRedirectUrl(url: URL): (Int, String) = { diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala new file mode 100644 index 0000000000000..fa57049b1a770 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.deploy.history + +import java.io.File +import java.util.NoSuchElementException +import java.util.concurrent.LinkedBlockingQueue + +import org.apache.commons.io.FileUtils +import org.scalatest.BeforeAndAfter +import org.scalatest.concurrent.TimeLimits +import org.scalatest.time.SpanSugar._ + +import org.apache.spark.SparkFunSuite +import org.apache.spark.status.KVUtils._ +import org.apache.spark.util.kvstore._ + +class HybridStoreSuite extends SparkFunSuite with BeforeAndAfter with TimeLimits { + + private var db: LevelDB = _ + private var dbpath: File = _ + + before { + dbpath = File.createTempFile("test.", ".ldb") + dbpath.delete() + db = new LevelDB(dbpath, new KVStoreScalaSerializer()) + } + + after { + if (db != null) { + db.close() + } + if (dbpath != null) { + FileUtils.deleteQuietly(dbpath) + } + } + + test("test multiple objects write read delete") { + val store = createHybridStore() + + val t1 = createCustomType1(1) + val t2 = createCustomType1(2) + + intercept[NoSuchElementException] { + store.read(t1.getClass(), t1.key) + } + + store.write(t1) + store.write(t2) + store.delete(t2.getClass(), t2.key) + + Seq(false, true).foreach { switch => + if (switch) switchHybridStore(store) + + intercept[NoSuchElementException] { + store.read(t2.getClass(), t2.key) + } + assert(store.read(t1.getClass(), t1.key) === t1) + assert(store.count(t1.getClass()) === 1L) + } + } + + test("test metadata") { + val store = createHybridStore() + assert(store.getMetadata(classOf[CustomType1]) === null) + + val t1 = createCustomType1(1) + store.setMetadata(t1) + assert(store.getMetadata(classOf[CustomType1]) === t1) + + // Switch to LevelDB and set a new metadata + switchHybridStore(store) + + val t2 = createCustomType1(2) + store.setMetadata(t2) + assert(store.getMetadata(classOf[CustomType1]) === t2) + } + + test("test update") { + val store = createHybridStore() + val t = createCustomType1(1) + + store.write(t) + t.name = "name2" + store.write(t) + + Seq(false, true).foreach { switch => + if (switch) switchHybridStore(store) + + assert(store.count(t.getClass()) === 1L) + assert(store.read(t.getClass(), t.key) === t) + } + } + + test("test basic iteration") { + val store = createHybridStore() + + val t1 = createCustomType1(1) + store.write(t1) + val t2 = createCustomType1(2) + store.write(t2) + + Seq(false, true).foreach { switch => + if (switch) switchHybridStore(store) + + assert(store.view(t1.getClass()).iterator().next().id === t1.id) + assert(store.view(t1.getClass()).skip(1).iterator().next().id === t2.id) + assert(store.view(t1.getClass()).skip(1).max(1).iterator().next().id === t2.id) + assert(store.view(t1.getClass()).first(t1.key).max(1).iterator().next().id === t1.id) + assert(store.view(t1.getClass()).first(t2.key).max(1).iterator().next().id === t2.id) + } + } + + test("test delete after switch") { + val store = createHybridStore() + val t = createCustomType1(1) + store.write(t) + switchHybridStore(store) + intercept[IllegalStateException] { + store.delete(t.getClass(), t.key) + } + } + + test("test klassMap") { + val store = createHybridStore() + val t1 = createCustomType1(1) + store.write(t1) + assert(store.klassMap.size === 1) + val t2 = new CustomType2("key2") + store.write(t2) + assert(store.klassMap.size === 2) + + switchHybridStore(store) + val t3 = new CustomType3("key3") + store.write(t3) + // Cannot put new klass to klassMap after the switching starts + assert(store.klassMap.size === 2) + } + + private def createHybridStore(): HybridStore = { + val store = new HybridStore() + store.setLevelDB(db) + store + } + + private def createCustomType1(i: Int): CustomType1 = { + new CustomType1("key" + i, "id" + i, "name" + i, i, "child" + i) + } + + private def switchHybridStore(store: HybridStore): Unit = { + assert(store.getStore().isInstanceOf[InMemoryStore]) + val listener = new SwitchListener() + store.switchToLevelDB(listener, "test", None) + failAfter(2.seconds) { + assert(listener.waitUntilDone()) + } + while (!store.getStore().isInstanceOf[LevelDB]) { + Thread.sleep(10) + } + } + + private class SwitchListener extends HybridStore.SwitchToLevelDBListener { + + // Put true to the queue when switch succeeds, and false when fails. + private val results = new LinkedBlockingQueue[Boolean]() + + override def onSwitchToLevelDBSuccess(): Unit = { + try { + results.put(true) + } catch { + case _: InterruptedException => + // no-op + } + } + + override def onSwitchToLevelDBFail(e: Exception): Unit = { + try { + results.put(false) + } catch { + case _: InterruptedException => + // no-op + } + } + + def waitUntilDone(): Boolean = { + results.take() + } + } +} + +class CustomType1( + @KVIndexParam var key: String, + @KVIndexParam("id") var id: String, + @KVIndexParam(value = "name", copy = true) var name: String, + @KVIndexParam("int") var num: Int, + @KVIndexParam(value = "child", parent = "id") var child: String) { + + override def equals(o: Any): Boolean = { + o match { + case t: CustomType1 => + id.equals(t.id) && name.equals(t.name) + case _ => false + } + } + + override def hashCode: Int = { + id.hashCode + } + + override def toString: String = { + "CustomType1[key=" + key + ",id=" + id + ",name=" + name + ",num=" + num; + } +} + +class CustomType2(@KVIndexParam var key: String) {} + +class CustomType3(@KVIndexParam var key: String) {} diff --git a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala index 8898d68664f36..3329300b64d13 100644 --- a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala @@ -72,6 +72,7 @@ class MockWorker(master: RpcEndpointRef, conf: SparkConf = new SparkConf) extend }) } + var decommissioned = false var appDesc = DeployTestUtils.createAppDesc() val drivers = mutable.HashSet[String]() val driverResources = new mutable.HashMap[String, Map[String, Set[String]]] @@ -96,6 +97,8 @@ class MockWorker(master: RpcEndpointRef, conf: SparkConf = new SparkConf) extend case None => } driverIdToAppId.remove(driverId) + case DecommissionWorker => + decommissioned = true } } @@ -687,12 +690,20 @@ class MasterSuite extends SparkFunSuite } } - // TODO(SPARK-32250): Enable the test back. It is flaky in GitHub Actions. - ignore("SPARK-27510: Master should avoid dead loop while launching executor failed in Worker") { + test("SPARK-27510: Master should avoid dead loop while launching executor failed in Worker") { val master = makeAliveMaster() var worker: MockExecutorLaunchFailWorker = null try { - worker = new MockExecutorLaunchFailWorker(master) + val conf = new SparkConf() + // SPARK-32250: When running test on Github Action machine, the available processors in JVM + // is only 2, while on Jenkins it's 32. For this specific test, 2 available processors, which + // also decides number of threads in Dispatcher, is not enough to consume the messages. In + // the worst situation, MockExecutorLaunchFailWorker would occupy these 2 threads for + // handling messages LaunchDriver, LaunchExecutor at the same time but leave no thread for + // the driver to handle the message RegisteredApplication. At the end, it results in the dead + // lock situation. Therefore, we need to set more threads to avoid the dead lock. + conf.set(Network.RPC_NETTY_DISPATCHER_NUM_THREADS, 6) + worker = new MockExecutorLaunchFailWorker(master, conf) worker.rpcEnv.setupEndpoint("worker", worker) val workerRegMsg = RegisterWorker( worker.id, @@ -734,9 +745,9 @@ class MasterSuite extends SparkFunSuite hostnames: Seq[String]): Unit = { val conf = new SparkConf() val master = makeAliveMaster(conf) - val workerRegs = (1 to numWorkers).map{idx => + val workers = (1 to numWorkers).map { idx => val worker = new MockWorker(master.self, conf) - worker.rpcEnv.setupEndpoint("worker", worker) + worker.rpcEnv.setupEndpoint(s"worker-$idx", worker) val workerReg = RegisterWorker( worker.id, "localhost", @@ -747,14 +758,14 @@ class MasterSuite extends SparkFunSuite "http://localhost:8080", RpcAddress("localhost", 10000)) master.self.send(workerReg) - workerReg + worker } eventually(timeout(10.seconds)) { val masterState = master.self.askSync[MasterStateResponse](RequestMasterState) assert(masterState.workers.length === numWorkers) assert(masterState.workers.forall(_.state == WorkerState.ALIVE)) - assert(masterState.workers.map(_.id).toSet == workerRegs.map(_.id).toSet) + assert(masterState.workers.map(_.id).toSet == workers.map(_.id).toSet) } val decomWorkersCount = master.self.askSync[Integer](DecommissionWorkersOnHosts(hostnames)) @@ -765,8 +776,11 @@ class MasterSuite extends SparkFunSuite eventually(timeout(30.seconds)) { val masterState = master.self.askSync[MasterStateResponse](RequestMasterState) assert(masterState.workers.length === numWorkers) - val workersActuallyDecomed = masterState.workers.count(_.state == WorkerState.DECOMMISSIONED) - assert(workersActuallyDecomed === numWorkersExpectedToDecom) + val workersActuallyDecomed = masterState.workers + .filter(_.state == WorkerState.DECOMMISSIONED).map(_.id) + val decommissionedWorkers = workers.filter(w => workersActuallyDecomed.contains(w.id)) + assert(workersActuallyDecomed.length === numWorkersExpectedToDecom) + assert(decommissionedWorkers.forall(_.decommissioned)) } // Decommissioning a worker again should return the same answer since we want this call to be diff --git a/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala b/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala index 7888796dd55e6..15966e2744491 100644 --- a/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala +++ b/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala @@ -129,6 +129,40 @@ class PluginContainerSuite extends SparkFunSuite with BeforeAndAfterEach with Lo assert(TestSparkPlugin.driverPlugin != null) } + test("SPARK-33088: executor tasks trigger plugin calls") { + val conf = new SparkConf() + .setAppName(getClass().getName()) + .set(SparkLauncher.SPARK_MASTER, "local[1]") + .set(PLUGINS, Seq(classOf[TestSparkPlugin].getName())) + + sc = new SparkContext(conf) + sc.parallelize(1 to 10, 2).count() + + assert(TestSparkPlugin.executorPlugin.numOnTaskStart == 2) + assert(TestSparkPlugin.executorPlugin.numOnTaskSucceeded == 2) + assert(TestSparkPlugin.executorPlugin.numOnTaskFailed == 0) + } + + test("SPARK-33088: executor failed tasks trigger plugin calls") { + val conf = new SparkConf() + .setAppName(getClass().getName()) + .set(SparkLauncher.SPARK_MASTER, "local[1]") + .set(PLUGINS, Seq(classOf[TestSparkPlugin].getName())) + + sc = new SparkContext(conf) + try { + sc.parallelize(1 to 10, 2).foreach(i => throw new RuntimeException) + } catch { + case t: Throwable => // ignore exception + } + + eventually(timeout(10.seconds), interval(100.millis)) { + assert(TestSparkPlugin.executorPlugin.numOnTaskStart == 2) + assert(TestSparkPlugin.executorPlugin.numOnTaskSucceeded == 0) + assert(TestSparkPlugin.executorPlugin.numOnTaskFailed == 2) + } + } + test("plugin initialization in non-local mode") { val path = Utils.createTempDir() @@ -309,6 +343,10 @@ private class TestDriverPlugin extends DriverPlugin { private class TestExecutorPlugin extends ExecutorPlugin { + var numOnTaskStart: Int = 0 + var numOnTaskSucceeded: Int = 0 + var numOnTaskFailed: Int = 0 + override def init(ctx: PluginContext, extraConf: JMap[String, String]): Unit = { ctx.metricRegistry().register("executorMetric", new Gauge[Int] { override def getValue(): Int = 84 @@ -316,6 +354,17 @@ private class TestExecutorPlugin extends ExecutorPlugin { TestSparkPlugin.executorContext = ctx } + override def onTaskStart(): Unit = { + numOnTaskStart += 1 + } + + override def onTaskSucceeded(): Unit = { + numOnTaskSucceeded += 1 + } + + override def onTaskFailed(failureReason: TaskFailedReason): Unit = { + numOnTaskFailed += 1 + } } private object TestSparkPlugin { diff --git a/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala b/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala index 60f67699f81be..987f383c9c4fa 100644 --- a/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala +++ b/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala @@ -119,6 +119,14 @@ class TestMemoryManager(conf: SparkConf) consequentOOM += n } + /** + * Undos the effects of [[markExecutionAsOutOfMemoryOnce]] and [[markconsequentOOM]] and lets + * calls to [[acquireExecutionMemory()]] (if there is enough memory available). + */ + def resetConsequentOOM(): Unit = synchronized { + consequentOOM = 0 + } + def limit(avail: Long): Unit = synchronized { require(avail >= 0) available = avail diff --git a/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala b/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala index 0e21a36071c42..3d4b8c868d6fc 100644 --- a/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala +++ b/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala @@ -35,12 +35,27 @@ class StatsdSinkSuite extends SparkFunSuite { STATSD_KEY_UNIT -> "seconds", STATSD_KEY_HOST -> "127.0.0.1" ) - private val socketTimeout = 30000 // milliseconds - private val socketBufferSize = 8192 + // The maximum size of a single datagram packet payload. Payloads + // larger than this will be truncated. + private val maxPayloadSize = 256 // bytes + + // The receive buffer must be large enough to hold all inflight + // packets. This includes any kernel and protocol overhead. + // This value was determined experimentally and should be + // increased if timeouts are seen. + private val socketMinRecvBufferSize = 16384 // bytes + private val socketTimeout = 30000 // milliseconds private def withSocketAndSink(testCode: (DatagramSocket, StatsdSink) => Any): Unit = { val socket = new DatagramSocket - socket.setReceiveBufferSize(socketBufferSize) + + // Leave the receive buffer size untouched unless it is too + // small. If the receive buffer is too small packets will be + // silently dropped and receive operations will timeout. + if (socket.getReceiveBufferSize() < socketMinRecvBufferSize) { + socket.setReceiveBufferSize(socketMinRecvBufferSize) + } + socket.setSoTimeout(socketTimeout) val props = new Properties defaultProps.foreach(e => props.put(e._1, e._2)) @@ -61,7 +76,7 @@ class StatsdSinkSuite extends SparkFunSuite { sink.registry.register("counter", counter) sink.report() - val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize) + val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize) socket.receive(p) val result = new String(p.getData, 0, p.getLength, UTF_8) @@ -77,7 +92,7 @@ class StatsdSinkSuite extends SparkFunSuite { sink.registry.register("gauge", gauge) sink.report() - val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize) + val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize) socket.receive(p) val result = new String(p.getData, 0, p.getLength, UTF_8) @@ -87,7 +102,7 @@ class StatsdSinkSuite extends SparkFunSuite { test("metrics StatsD sink with Histogram") { withSocketAndSink { (socket, sink) => - val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize) + val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize) val histogram = new Histogram(new UniformReservoir) histogram.update(10) histogram.update(20) @@ -121,7 +136,7 @@ class StatsdSinkSuite extends SparkFunSuite { test("metrics StatsD sink with Timer") { withSocketAndSink { (socket, sink) => - val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize) + val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize) val timer = new Timer() timer.update(1, SECONDS) timer.update(2, SECONDS) diff --git a/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferServiceSuite.scala b/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferServiceSuite.scala index baa878eb14047..fa1a75d076051 100644 --- a/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferServiceSuite.scala +++ b/core/src/test/scala/org/apache/spark/network/netty/NettyBlockTransferServiceSuite.scala @@ -118,8 +118,8 @@ class NettyBlockTransferServiceSuite .thenAnswer(_ => {hitExecutorDeadException = true}) service0 = createService(port, driverEndpointRef) - val clientFactoryField = service0.getClass.getField( - "org$apache$spark$network$netty$NettyBlockTransferService$$clientFactory") + val clientFactoryField = service0.getClass + .getSuperclass.getSuperclass.getDeclaredField("clientFactory") clientFactoryField.setAccessible(true) clientFactoryField.set(service0, clientFactory) diff --git a/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala b/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala index c942328acc8c1..9e3f27911019c 100644 --- a/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala +++ b/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala @@ -44,6 +44,7 @@ class LocalCheckpointSuite extends SparkFunSuite with LocalSparkContext { assert(transform(StorageLevel.MEMORY_ONLY_SER_2) === StorageLevel.MEMORY_AND_DISK_SER_2) assert(transform(StorageLevel.DISK_ONLY) === StorageLevel.DISK_ONLY) assert(transform(StorageLevel.DISK_ONLY_2) === StorageLevel.DISK_ONLY_2) + assert(transform(StorageLevel.DISK_ONLY_3) === StorageLevel.DISK_ONLY_3) assert(transform(StorageLevel.MEMORY_AND_DISK) === StorageLevel.MEMORY_AND_DISK) assert(transform(StorageLevel.MEMORY_AND_DISK_SER) === StorageLevel.MEMORY_AND_DISK_SER) assert(transform(StorageLevel.MEMORY_AND_DISK_2) === StorageLevel.MEMORY_AND_DISK_2) diff --git a/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala b/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala index c74c728b3e3f3..8b1c602cd8e58 100644 --- a/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala +++ b/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala @@ -136,4 +136,17 @@ class InboxSuite extends SparkFunSuite { endpoint.verifySingleOnNetworkErrorMessage(cause, remoteAddress) } + + test("SPARK-32738: should reduce the number of active threads when fatal error happens") { + val endpoint = mock(classOf[TestRpcEndpoint]) + when(endpoint.receive).thenThrow(new OutOfMemoryError()) + + val dispatcher = mock(classOf[Dispatcher]) + val inbox = new Inbox("name", endpoint) + inbox.post(OneWayMessage(null, "hi")) + intercept[OutOfMemoryError] { + inbox.process(dispatcher) + } + assert(inbox.getNumActiveThreads == 0) + } } diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala index d18ca36f1fa60..e4ec62f8efc5b 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala @@ -189,7 +189,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with test("throw exception if the number of barrier() calls are not the same on every task") { initLocalClusterSparkContext() - sc.conf.set("spark.barrier.sync.timeout", "1") + sc.conf.set("spark.barrier.sync.timeout", "5") val rdd = sc.makeRDD(1 to 10, 4) val rdd2 = rdd.barrier().mapPartitions { it => val context = BarrierTaskContext.get() @@ -212,7 +212,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with rdd2.collect() }.getMessage assert(error.contains("The coordinator didn't get all barrier sync requests")) - assert(error.contains("within 1 second(s)")) + assert(error.contains("within 5 second(s)")) } def testBarrierTaskKilled(interruptOnKill: Boolean): Unit = { diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala index c829006923c4f..99be1faab8b85 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.scheduler import java.util.Properties import java.util.concurrent.{CountDownLatch, TimeUnit} -import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicLong, AtomicReference} +import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong, AtomicReference} import scala.annotation.meta.param import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet, Map} @@ -125,14 +125,14 @@ class MyRDD( class DAGSchedulerSuiteDummyException extends Exception -class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLimits { +class DAGSchedulerSuite extends SparkFunSuite with TempLocalSparkContext with TimeLimits { import DAGSchedulerSuite._ // Necessary to make ScalaTest 3.x interrupt a thread on the JVM like ScalaTest 2.2.x implicit val defaultSignaler: Signaler = ThreadSignaler - val conf = new SparkConf + private var firstInit: Boolean = _ /** Set of TaskSets the DAGScheduler has requested executed. */ val taskSets = scala.collection.mutable.Buffer[TaskSet]() @@ -178,8 +178,8 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi override def executorDecommission( executorId: String, decommissionInfo: ExecutorDecommissionInfo): Unit = {} - override def getExecutorDecommissionInfo( - executorId: String): Option[ExecutorDecommissionInfo] = None + override def getExecutorDecommissionState( + executorId: String): Option[ExecutorDecommissionState] = None } /** @@ -297,11 +297,19 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi override def beforeEach(): Unit = { super.beforeEach() - init(new SparkConf()) + firstInit = true } - private def init(testConf: SparkConf): Unit = { - sc = new SparkContext("local[2]", "DAGSchedulerSuite", testConf) + override def sc: SparkContext = { + val sc = super.sc + if (firstInit) { + init(sc) + firstInit = false + } + sc + } + + private def init(sc: SparkContext): Unit = { sparkListener = new EventInfoRecordingListener failure = null sc.addSparkListener(sparkListener) @@ -310,10 +318,10 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi cancelledStages.clear() cacheLocations.clear() results.clear() - securityMgr = new SecurityManager(conf) - broadcastManager = new BroadcastManager(true, conf, securityMgr) - mapOutputTracker = spy(new MyMapOutputTrackerMaster(conf, broadcastManager)) - blockManagerMaster = spy(new MyBlockManagerMaster(conf)) + securityMgr = new SecurityManager(sc.getConf) + broadcastManager = new BroadcastManager(true, sc.getConf, securityMgr) + mapOutputTracker = spy(new MyMapOutputTrackerMaster(sc.getConf, broadcastManager)) + blockManagerMaster = spy(new MyBlockManagerMaster(sc.getConf)) scheduler = new DAGScheduler( sc, taskScheduler, @@ -353,6 +361,8 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi * DAGScheduler event loop. */ private def runEvent(event: DAGSchedulerEvent): Unit = { + // Ensure the initialization of various components + sc dagEventProcessLoopTester.post(event) } @@ -491,12 +501,8 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi } test("All shuffle files on the storage endpoint should be cleaned up when it is lost") { - // reset the test context with the right shuffle service config - afterEach() - val conf = new SparkConf() conf.set(config.SHUFFLE_SERVICE_ENABLED.key, "true") conf.set("spark.files.fetchFailure.unRegisterOutputOnHost", "true") - init(conf) runEvent(ExecutorAdded("hostA-exec1", "hostA")) runEvent(ExecutorAdded("hostA-exec2", "hostA")) runEvent(ExecutorAdded("hostB-exec", "hostB")) @@ -565,11 +571,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi } test("SPARK-32003: All shuffle files for executor should be cleaned up on fetch failure") { - // reset the test context with the right shuffle service config - afterEach() - val conf = new SparkConf() conf.set(config.SHUFFLE_SERVICE_ENABLED.key, "true") - init(conf) val shuffleMapRdd = new MyRDD(sc, 3, Nil) val shuffleDep = new ShuffleDependency(shuffleMapRdd, new HashPartitioner(3)) @@ -787,8 +789,8 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi override def executorDecommission( executorId: String, decommissionInfo: ExecutorDecommissionInfo): Unit = {} - override def getExecutorDecommissionInfo( - executorId: String): Option[ExecutorDecommissionInfo] = None + override def getExecutorDecommissionState( + executorId: String): Option[ExecutorDecommissionState] = None } val noKillScheduler = new DAGScheduler( sc, @@ -848,9 +850,9 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi } private val shuffleFileLossTests = Seq( - ("executor process lost with shuffle service", ExecutorProcessLost("", false), true, false), - ("worker lost with shuffle service", ExecutorProcessLost("", true), true, true), - ("worker lost without shuffle service", ExecutorProcessLost("", true), false, true), + ("executor process lost with shuffle service", ExecutorProcessLost("", None), true, false), + ("worker lost with shuffle service", ExecutorProcessLost("", Some("hostA")), true, true), + ("worker lost without shuffle service", ExecutorProcessLost("", Some("hostA")), false, true), ("executor failure with shuffle service", ExecutorKilled, true, false), ("executor failure without shuffle service", ExecutorKilled, false, true)) @@ -861,11 +863,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi "not lost" } test(s"shuffle files $maybeLost when $eventDescription") { - // reset the test context with the right shuffle service config - afterEach() - val conf = new SparkConf() conf.set(config.SHUFFLE_SERVICE_ENABLED.key, shuffleServiceOn.toString) - init(conf) assert(sc.env.blockManager.externalShuffleServiceEnabled == shuffleServiceOn) val shuffleMapRdd = new MyRDD(sc, 2, Nil) @@ -874,10 +872,18 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi val reduceRdd = new MyRDD(sc, 1, List(shuffleDep), tracker = mapOutputTracker) submit(reduceRdd, Array(0)) completeShuffleMapStageSuccessfully(0, 0, 1) + val expectHostFileLoss = event match { + case ExecutorProcessLost(_, workerHost, _) => workerHost.isDefined + case _ => false + } runEvent(ExecutorLost("hostA-exec", event)) verify(blockManagerMaster, times(1)).removeExecutor("hostA-exec") if (expectFileLoss) { - verify(mapOutputTracker, times(1)).removeOutputsOnExecutor("hostA-exec") + if (expectHostFileLoss) { + verify(mapOutputTracker, times(1)).removeOutputsOnHost("hostA") + } else { + verify(mapOutputTracker, times(1)).removeOutputsOnExecutor("hostA-exec") + } intercept[MetadataFetchFailedException] { mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0) } @@ -2880,11 +2886,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi } test("SPARK-25341: abort stage while using old fetch protocol") { - // reset the test context with using old fetch protocol - afterEach() - val conf = new SparkConf() conf.set(config.SHUFFLE_USE_OLD_FETCH_PROTOCOL.key, "true") - init(conf) // Construct the scenario of indeterminate stage fetch failed. constructIndeterminateStageFetchFailed() // The job should fail because Spark can't rollback the shuffle map stage while @@ -3212,10 +3214,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi } test("test 2 resource profile with merge conflict config true") { - afterEach() - val conf = new SparkConf() conf.set(config.RESOURCE_PROFILE_MERGE_CONFLICTS.key, "true") - init(conf) val ereqs = new ExecutorResourceRequests().cores(4) val treqs = new TaskResourceRequests().cpus(1) @@ -3233,10 +3232,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi } test("test multiple resource profiles created from merging use same rp") { - afterEach() - val conf = new SparkConf() conf.set(config.RESOURCE_PROFILE_MERGE_CONFLICTS.key, "true") - init(conf) val ereqs = new ExecutorResourceRequests().cores(4) val treqs = new TaskResourceRequests().cpus(1) @@ -3330,10 +3326,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi } test("test merge 3 resource profiles") { - afterEach() - val conf = new SparkConf() conf.set(config.RESOURCE_PROFILE_MERGE_CONFLICTS.key, "true") - init(conf) val ereqs = new ExecutorResourceRequests().cores(4) val treqs = new TaskResourceRequests().cpus(1) val rp1 = new ResourceProfile(ereqs.requests, treqs.requests) diff --git a/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala index 07d88672290fc..08191d09a9f2d 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/ExternalClusterManagerSuite.scala @@ -101,6 +101,6 @@ private class DummyTaskScheduler extends TaskScheduler { override def executorDecommission( executorId: String, decommissionInfo: ExecutorDecommissionInfo): Unit = {} - override def getExecutorDecommissionInfo( - executorId: String): Option[ExecutorDecommissionInfo] = None + override def getExecutorDecommissionState( + executorId: String): Option[ExecutorDecommissionState] = None } diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala index 394a2a9fbf7cb..8a7ff9eb6dcd3 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskContextSuite.scala @@ -70,7 +70,7 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark 0, 0, taskBinary, rdd.partitions(0), Seq.empty, 0, new Properties, closureSerializer.serialize(TaskMetrics.registered).array()) intercept[RuntimeException] { - task.run(0, 0, null, null) + task.run(0, 0, null, null, Option.empty) } assert(TaskContextSuite.completed) } @@ -92,7 +92,7 @@ class TaskContextSuite extends SparkFunSuite with BeforeAndAfter with LocalSpark 0, 0, taskBinary, rdd.partitions(0), Seq.empty, 0, new Properties, closureSerializer.serialize(TaskMetrics.registered).array()) intercept[RuntimeException] { - task.run(0, 0, null, null) + task.run(0, 0, null, null, Option.empty) } assert(TaskContextSuite.lastError.getMessage == "damn error") } diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala index e5836458e7f91..f29eb70eb3628 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala @@ -34,7 +34,7 @@ import org.apache.spark.internal.config import org.apache.spark.resource.{ExecutorResourceRequests, ResourceProfile, TaskResourceRequests} import org.apache.spark.resource.ResourceUtils._ import org.apache.spark.resource.TestResourceIDs._ -import org.apache.spark.util.ManualClock +import org.apache.spark.util.{Clock, ManualClock, SystemClock} class FakeSchedulerBackend extends SchedulerBackend { def start(): Unit = {} @@ -91,7 +91,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B val conf = new SparkConf().setMaster(master).setAppName("TaskSchedulerImplSuite") confs.foreach { case (k, v) => conf.set(k, v) } sc = new SparkContext(conf) - taskScheduler = new TaskSchedulerImpl(sc) + taskScheduler = new TaskSchedulerImpl(sc, sc.conf.get(config.TASK_MAX_FAILURES)) setupHelper() } @@ -140,6 +140,33 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B taskScheduler } + test("SPARK-32653: Decommissioned host/executor should be considered as inactive") { + val scheduler = setupScheduler() + val exec0 = "exec0" + val exec1 = "exec1" + val exec2 = "exec2" + val host0 = "host0" + val host1 = "host1" + val workerOffers = IndexedSeq( + WorkerOffer(exec0, host0, 1), + WorkerOffer(exec1, host0, 1), + WorkerOffer(exec2, host1, 1)) + scheduler.resourceOffers(workerOffers) + assert(Seq(exec0, exec1, exec2).forall(scheduler.isExecutorAlive)) + assert(Seq(host0, host1).forall(scheduler.hasExecutorsAliveOnHost)) + assert(scheduler.getExecutorsAliveOnHost(host0) + .exists(s => s.contains(exec0) && s.contains(exec1))) + assert(scheduler.getExecutorsAliveOnHost(host1).exists(_.contains(exec2))) + + scheduler.executorDecommission(exec1, ExecutorDecommissionInfo("test", None)) + scheduler.executorDecommission(exec2, ExecutorDecommissionInfo("test", Some(host1))) + + assert(scheduler.isExecutorAlive(exec0)) + assert(!Seq(exec1, exec2).exists(scheduler.isExecutorAlive)) + assert(scheduler.hasExecutorsAliveOnHost(host0)) + assert(!scheduler.hasExecutorsAliveOnHost(host1)) + } + test("Scheduler does not always schedule tasks on the same workers") { val taskScheduler = setupScheduler() val numFreeCores = 1 @@ -1802,51 +1829,93 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B assert(2 == taskDescriptions.head.resources(GPU).addresses.size) } - private def setupSchedulerForDecommissionTests(): TaskSchedulerImpl = { - val taskScheduler = setupSchedulerWithMaster( - s"local[2]", - config.CPUS_PER_TASK.key -> 1.toString) - taskScheduler.submitTasks(FakeTask.createTaskSet(2)) - val multiCoreWorkerOffers = IndexedSeq(WorkerOffer("executor0", "host0", 1), - WorkerOffer("executor1", "host1", 1)) - val taskDescriptions = taskScheduler.resourceOffers(multiCoreWorkerOffers).flatten - assert(taskDescriptions.map(_.executorId).sorted === Seq("executor0", "executor1")) + private def setupSchedulerForDecommissionTests(clock: Clock, numTasks: Int): TaskSchedulerImpl = { + // one task per host + val numHosts = numTasks + val conf = new SparkConf() + .setMaster(s"local[$numHosts]") + .setAppName("TaskSchedulerImplSuite") + .set(config.CPUS_PER_TASK.key, "1") + sc = new SparkContext(conf) + val maxTaskFailures = sc.conf.get(config.TASK_MAX_FAILURES) + taskScheduler = new TaskSchedulerImpl(sc, maxTaskFailures, clock = clock) { + override def createTaskSetManager(taskSet: TaskSet, maxFailures: Int): TaskSetManager = { + val tsm = super.createTaskSetManager(taskSet, maxFailures) + // we need to create a spied tsm so that we can see the copies running + val tsmSpy = spy(tsm) + stageToMockTaskSetManager(taskSet.stageId) = tsmSpy + tsmSpy + } + } + setupHelper() + // Spawn the tasks on different executors/hosts + taskScheduler.submitTasks(FakeTask.createTaskSet(numTasks)) + for (i <- 0 until numTasks) { + val executorId = s"executor$i" + val taskDescriptions = taskScheduler.resourceOffers(IndexedSeq(WorkerOffer( + executorId, s"host$i", 1))).flatten + assert(taskDescriptions.size === 1) + assert(taskDescriptions(0).executorId == executorId) + assert(taskDescriptions(0).index === i) + } taskScheduler } - test("scheduler should keep the decommission info where host was decommissioned") { - val scheduler = setupSchedulerForDecommissionTests() - - scheduler.executorDecommission("executor0", ExecutorDecommissionInfo("0", false)) - scheduler.executorDecommission("executor1", ExecutorDecommissionInfo("1", true)) - scheduler.executorDecommission("executor0", ExecutorDecommissionInfo("0 new", false)) - scheduler.executorDecommission("executor1", ExecutorDecommissionInfo("1 new", false)) - - assert(scheduler.getExecutorDecommissionInfo("executor0") - === Some(ExecutorDecommissionInfo("0 new", false))) - assert(scheduler.getExecutorDecommissionInfo("executor1") - === Some(ExecutorDecommissionInfo("1", true))) - assert(scheduler.getExecutorDecommissionInfo("executor2").isEmpty) + test("scheduler should keep the decommission state where host was decommissioned") { + val clock = new ManualClock(10000L) + val scheduler = setupSchedulerForDecommissionTests(clock, 2) + val decomTime = clock.getTimeMillis() + scheduler.executorDecommission("executor0", ExecutorDecommissionInfo("0", None)) + scheduler.executorDecommission("executor1", ExecutorDecommissionInfo("1", Some("host1"))) + + assert(scheduler.getExecutorDecommissionState("executor0") + === Some(ExecutorDecommissionState(decomTime, None))) + assert(scheduler.getExecutorDecommissionState("executor1") + === Some(ExecutorDecommissionState(decomTime, Some("host1")))) + assert(scheduler.getExecutorDecommissionState("executor2").isEmpty) } - test("scheduler should ignore decommissioning of removed executors") { - val scheduler = setupSchedulerForDecommissionTests() + test("test full decommissioning flow") { + val clock = new ManualClock(10000L) + val scheduler = setupSchedulerForDecommissionTests(clock, 2) + val manager = stageToMockTaskSetManager(0) + // The task started should be running. + assert(manager.copiesRunning.take(2) === Array(1, 1)) // executor 0 is decommissioned after loosing - assert(scheduler.getExecutorDecommissionInfo("executor0").isEmpty) + assert(scheduler.getExecutorDecommissionState("executor0").isEmpty) scheduler.executorLost("executor0", ExecutorExited(0, false, "normal")) - assert(scheduler.getExecutorDecommissionInfo("executor0").isEmpty) - scheduler.executorDecommission("executor0", ExecutorDecommissionInfo("", false)) - assert(scheduler.getExecutorDecommissionInfo("executor0").isEmpty) + assert(scheduler.getExecutorDecommissionState("executor0").isEmpty) + scheduler.executorDecommission("executor0", ExecutorDecommissionInfo("", None)) + assert(scheduler.getExecutorDecommissionState("executor0").isEmpty) + + // 0th task just died above + assert(manager.copiesRunning.take(2) === Array(0, 1)) + + assert(scheduler.executorsPendingDecommission.isEmpty) + clock.advance(5000) + + // executor1 hasn't been decommissioned yet + assert(scheduler.getExecutorDecommissionState("executor1").isEmpty) // executor 1 is decommissioned before loosing - assert(scheduler.getExecutorDecommissionInfo("executor1").isEmpty) - scheduler.executorDecommission("executor1", ExecutorDecommissionInfo("", false)) - assert(scheduler.getExecutorDecommissionInfo("executor1").isDefined) + scheduler.executorDecommission("executor1", ExecutorDecommissionInfo("", None)) + assert(scheduler.getExecutorDecommissionState("executor1").isDefined) + clock.advance(2000) + + // executor1 is eventually lost scheduler.executorLost("executor1", ExecutorExited(0, false, "normal")) - assert(scheduler.getExecutorDecommissionInfo("executor1").isEmpty) - scheduler.executorDecommission("executor1", ExecutorDecommissionInfo("", false)) - assert(scheduler.getExecutorDecommissionInfo("executor1").isEmpty) + assert(scheduler.executorsPendingDecommission.isEmpty) + // So now both the tasks are no longer running + assert(manager.copiesRunning.take(2) === Array(0, 0)) + clock.advance(2000) + + // Now give it some resources and both tasks should be rerun + val taskDescriptions = taskScheduler.resourceOffers(IndexedSeq( + WorkerOffer("executor2", "host2", 1), WorkerOffer("executor3", "host3", 1))).flatten + assert(taskDescriptions.size === 2) + assert(taskDescriptions.map(_.index).sorted == Seq(0, 1)) + assert(manager.copiesRunning.take(2) === Array(1, 1)) } /** diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala index 0a98030a56edc..c389fd2ffa8b1 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala @@ -41,7 +41,7 @@ import org.apache.spark.resource.TestResourceIDs._ import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend import org.apache.spark.serializer.SerializerInstance import org.apache.spark.storage.BlockManagerId -import org.apache.spark.util.{AccumulatorV2, ManualClock} +import org.apache.spark.util.{AccumulatorV2, Clock, ManualClock, SystemClock} class FakeDAGScheduler(sc: SparkContext, taskScheduler: FakeTaskScheduler) extends DAGScheduler(sc) { @@ -109,8 +109,11 @@ object FakeRackUtil { * a list of "live" executors and their hostnames for isExecutorAlive and hasExecutorsAliveOnHost * to work, and these are required for locality in TaskSetManager. */ -class FakeTaskScheduler(sc: SparkContext, liveExecutors: (String, String)* /* execId, host */) - extends TaskSchedulerImpl(sc) +class FakeTaskScheduler( + sc: SparkContext, + clock: Clock, + liveExecutors: (String, String)* /* execId, host */) + extends TaskSchedulerImpl(sc, sc.conf.get(config.TASK_MAX_FAILURES), clock = clock) { val startedTasks = new ArrayBuffer[Long] val endedTasks = new mutable.HashMap[Long, TaskEndReason] @@ -120,6 +123,10 @@ class FakeTaskScheduler(sc: SparkContext, liveExecutors: (String, String)* /* ex val executors = new mutable.HashMap[String, String] + def this(sc: SparkContext, liveExecutors: (String, String)*) = { + this(sc, new SystemClock, liveExecutors: _*) + } + // this must be initialized before addExecutor override val defaultRackValue: Option[String] = Some("default") for ((execId, host) <- liveExecutors) { @@ -149,13 +156,12 @@ class FakeTaskScheduler(sc: SparkContext, liveExecutors: (String, String)* /* ex override def taskSetFinished(manager: TaskSetManager): Unit = finishedManagers += manager - override def isExecutorAlive(execId: String): Boolean = executors.contains(execId) + override def isExecutorAlive(execId: String): Boolean = + executors.contains(execId) && !isExecutorDecommissioned(execId) - override def hasExecutorsAliveOnHost(host: String): Boolean = executors.values.exists(_ == host) - - override def hasHostAliveOnRack(rack: String): Boolean = { - hostsByRack.get(rack) != None - } + override def hasExecutorsAliveOnHost(host: String): Boolean = + !isHostDecommissioned(host) && executors + .exists { case (e, h) => h == host && !isExecutorDecommissioned(e) } def addExecutor(execId: String, host: String): Unit = { executors.put(execId, host) @@ -653,6 +659,59 @@ class TaskSetManagerSuite assert(manager.resourceOffer("execA", "host1", ANY)._1.isDefined) } + test("SPARK-32653: Decommissioned host should not be used to calculate locality levels") { + sc = new SparkContext("local", "test") + sched = new FakeTaskScheduler(sc) + val backend = mock(classOf[SchedulerBackend]) + doNothing().when(backend).reviveOffers() + sched.initialize(backend) + + val exec0 = "exec0" + val exec1 = "exec1" + val host0 = "host0" + sched.addExecutor(exec0, host0) + sched.addExecutor(exec1, host0) + + val taskSet = FakeTask.createTaskSet(2, + Seq(ExecutorCacheTaskLocation(host0, exec0)), + Seq(ExecutorCacheTaskLocation(host0, exec1))) + sched.submitTasks(taskSet) + val manager = sched.taskSetManagerForAttempt(0, 0).get + + assert(manager.myLocalityLevels === Array(PROCESS_LOCAL, NODE_LOCAL, ANY)) + + // Decommission all executors on host0, to mimic CoarseGrainedSchedulerBackend. + sched.executorDecommission(exec0, ExecutorDecommissionInfo("test", Some(host0))) + sched.executorDecommission(exec1, ExecutorDecommissionInfo("test", Some(host0))) + + assert(manager.myLocalityLevels === Array(ANY)) + } + + test("SPARK-32653: Decommissioned executor should not be used to calculate locality levels") { + sc = new SparkContext("local", "test") + sched = new FakeTaskScheduler(sc) + val backend = mock(classOf[SchedulerBackend]) + doNothing().when(backend).reviveOffers() + sched.initialize(backend) + + val exec0 = "exec0" + val exec1 = "exec1" + val host0 = "host0" + sched.addExecutor(exec0, host0) + sched.addExecutor(exec1, host0) + + val taskSet = FakeTask.createTaskSet(1, Seq(ExecutorCacheTaskLocation(host0, exec0))) + sched.submitTasks(taskSet) + val manager = sched.taskSetManagerForAttempt(0, 0).get + + assert(manager.myLocalityLevels === Array(PROCESS_LOCAL, NODE_LOCAL, ANY)) + + // Decommission the only executor (without the host) that the task is interested in running on. + sched.executorDecommission(exec0, ExecutorDecommissionInfo("test", None)) + + assert(manager.myLocalityLevels === Array(NODE_LOCAL, ANY)) + } + test("test RACK_LOCAL tasks") { // Assign host1 to rack1 FakeRackUtil.assignHostToRack("host1", "rack1") @@ -750,6 +809,14 @@ class TaskSetManagerSuite assert(thrown2.getMessage().contains("bigger than spark.driver.maxResultSize")) } + test("SPARK-32470: do not check total size of intermediate stages") { + val conf = new SparkConf().set(config.MAX_RESULT_SIZE.key, "20k") + sc = new SparkContext("local", "test", conf) + // final result is below limit. + val r = sc.makeRDD(0 until 2000, 2000).distinct(10).filter(_ == 0).collect() + assert(1 === r.size) + } + test("[SPARK-13931] taskSetManager should not send Resubmitted tasks after being a zombie") { val conf = new SparkConf().set(config.SPECULATION_ENABLED, true) sc = new SparkContext("local", "test", conf) @@ -1914,14 +1981,16 @@ class TaskSetManagerSuite test("SPARK-21040: Check speculative tasks are launched when an executor is decommissioned" + " and the tasks running on it cannot finish within EXECUTOR_DECOMMISSION_KILL_INTERVAL") { sc = new SparkContext("local", "test") - sched = new FakeTaskScheduler(sc, ("exec1", "host1"), ("exec2", "host2"), ("exec3", "host3")) + val clock = new ManualClock() + sched = new FakeTaskScheduler(sc, clock, + ("exec1", "host1"), ("exec2", "host2"), ("exec3", "host3")) + sched.backend = mock(classOf[SchedulerBackend]) val taskSet = FakeTask.createTaskSet(4) sc.conf.set(config.SPECULATION_ENABLED, true) sc.conf.set(config.SPECULATION_MULTIPLIER, 1.5) sc.conf.set(config.SPECULATION_QUANTILE, 0.5) sc.conf.set(config.EXECUTOR_DECOMMISSION_KILL_INTERVAL.key, "5s") - val clock = new ManualClock() - val manager = new TaskSetManager(sched, taskSet, MAX_TASK_FAILURES, clock = clock) + val manager = sched.createTaskSetManager(taskSet, MAX_TASK_FAILURES) val accumUpdatesByTask: Array[Seq[AccumulatorV2[_, _]]] = taskSet.tasks.map { task => task.metrics.internalAccums } @@ -1957,13 +2026,12 @@ class TaskSetManagerSuite assert(!manager.checkSpeculatableTasks(0)) assert(sched.speculativeTasks.toSet === Set()) - // decommission exec-2. All tasks running on exec-2 (i.e. TASK 2,3) will be added to - // executorDecommissionSpeculationTriggerTimeoutOpt + // decommission exec-2. All tasks running on exec-2 (i.e. TASK 2,3) will be now + // checked if they should be speculated. // (TASK 2 -> 15, TASK 3 -> 15) - manager.executorDecommission("exec2") - assert(manager.tidToExecutorKillTimeMapping.keySet === Set(2, 3)) - assert(manager.tidToExecutorKillTimeMapping(2) === 15*1000) - assert(manager.tidToExecutorKillTimeMapping(3) === 15*1000) + sched.executorDecommission("exec2", ExecutorDecommissionInfo("decom", None)) + assert(sched.getExecutorDecommissionState("exec2").map(_.startTime) === + Some(clock.getTimeMillis())) assert(manager.checkSpeculatableTasks(0)) // TASK 2 started at t=0s, so it can still finish before t=15s (Median task runtime = 10s) diff --git a/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionExtendedSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionExtendedSuite.scala index d95deb1f5f327..129eb8bf91051 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionExtendedSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionExtendedSuite.scala @@ -23,8 +23,7 @@ import org.scalatest.concurrent.Eventually.{eventually, interval, timeout} import org.apache.spark.{LocalSparkContext, SparkContext, SparkFunSuite, TestUtils} import org.apache.spark.LocalSparkContext.withSpark -import org.apache.spark.internal.config.{DYN_ALLOCATION_ENABLED, DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT, DYN_ALLOCATION_INITIAL_EXECUTORS, DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED} -import org.apache.spark.internal.config.Worker.WORKER_DECOMMISSION_ENABLED +import org.apache.spark.internal.config.{DECOMMISSION_ENABLED, DYN_ALLOCATION_ENABLED, DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT, DYN_ALLOCATION_INITIAL_EXECUTORS, DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED} import org.apache.spark.launcher.SparkLauncher.{EXECUTOR_MEMORY, SPARK_MASTER} import org.apache.spark.scheduler.cluster.StandaloneSchedulerBackend @@ -37,7 +36,7 @@ class WorkerDecommissionExtendedSuite extends SparkFunSuite with LocalSparkConte .set(DYN_ALLOCATION_ENABLED, true) .set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true) .set(DYN_ALLOCATION_INITIAL_EXECUTORS, 5) - .set(WORKER_DECOMMISSION_ENABLED, true) + .set(DECOMMISSION_ENABLED, true) test("Worker decommission and executor idle timeout") { sc = new SparkContext(conf.set(DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT.key, "10s")) @@ -65,7 +64,8 @@ class WorkerDecommissionExtendedSuite extends SparkFunSuite with LocalSparkConte val sched = sc.schedulerBackend.asInstanceOf[StandaloneSchedulerBackend] sc.getExecutorIds().tail.foreach { id => - sched.decommissionExecutor(id, ExecutorDecommissionInfo("", false)) + sched.decommissionExecutor(id, ExecutorDecommissionInfo("", None), + adjustTargetNumExecutors = false) assert(rdd3.sortByKey().collect().length === 100) } } diff --git a/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionSuite.scala index 3c34070e8bb97..4a92cbcb85847 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/WorkerDecommissionSuite.scala @@ -31,8 +31,8 @@ import org.apache.spark.util.{RpcUtils, SerializableBuffer, ThreadUtils} class WorkerDecommissionSuite extends SparkFunSuite with LocalSparkContext { override def beforeEach(): Unit = { - val conf = new SparkConf().setAppName("test").setMaster("local") - .set(config.Worker.WORKER_DECOMMISSION_ENABLED, true) + val conf = new SparkConf().setAppName("test") + .set(config.DECOMMISSION_ENABLED, true) sc = new SparkContext("local-cluster[2, 1, 1024]", "test", conf) } @@ -47,7 +47,12 @@ class WorkerDecommissionSuite extends SparkFunSuite with LocalSparkContext { assert(sleepyRdd.count() === 10) } - test("verify a task with all workers decommissioned succeeds") { + test("verify a running task with all workers decommissioned succeeds") { + // Wait for the executors to come up + TestUtils.waitUntilExecutorsUp(sc = sc, + numExecutors = 2, + timeout = 30000) // 30s + val input = sc.parallelize(1 to 10) // Listen for the job val sem = new Semaphore(0) @@ -56,9 +61,7 @@ class WorkerDecommissionSuite extends SparkFunSuite with LocalSparkContext { sem.release() } }) - TestUtils.waitUntilExecutorsUp(sc = sc, - numExecutors = 2, - timeout = 30000) // 30s + val sleepyRdd = input.mapPartitions{ x => Thread.sleep(5000) // 5s x @@ -73,16 +76,13 @@ class WorkerDecommissionSuite extends SparkFunSuite with LocalSparkContext { // decom.sh message passing is tested manually. val sched = sc.schedulerBackend.asInstanceOf[StandaloneSchedulerBackend] val execs = sched.getExecutorIds() - execs.foreach(execId => sched.decommissionExecutor(execId, ExecutorDecommissionInfo("", false))) + // Make the executors decommission, finish, exit, and not be replaced. + val execsAndDecomInfo = execs.map((_, ExecutorDecommissionInfo("", None))).toArray + sched.decommissionExecutors( + execsAndDecomInfo, + adjustTargetNumExecutors = true, + triggeredByExecutor = false) val asyncCountResult = ThreadUtils.awaitResult(asyncCount, 20.seconds) assert(asyncCountResult === 10) - // Try and launch task after decommissioning, this should fail - val postDecommissioned = input.map(x => x) - val postDecomAsyncCount = postDecommissioned.countAsync() - val thrown = intercept[java.util.concurrent.TimeoutException]{ - val result = ThreadUtils.awaitResult(postDecomAsyncCount, 20.seconds) - } - assert(postDecomAsyncCount.isCompleted === false, - "After exec decommission new task could not launch") } } diff --git a/core/src/test/scala/org/apache/spark/shuffle/HostLocalShuffleReadingSuite.scala b/core/src/test/scala/org/apache/spark/shuffle/HostLocalShuffleReadingSuite.scala new file mode 100644 index 0000000000000..8f0c4da88feb2 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/shuffle/HostLocalShuffleReadingSuite.scala @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.shuffle + +import org.scalatest.matchers.must.Matchers +import org.scalatest.matchers.should.Matchers._ + +import org.apache.spark._ +import org.apache.spark.internal.config._ +import org.apache.spark.network.TransportContext +import org.apache.spark.network.netty.{NettyBlockTransferService, SparkTransportConf} +import org.apache.spark.network.server.TransportServer +import org.apache.spark.network.shuffle.{ExternalBlockHandler, ExternalBlockStoreClient} +import org.apache.spark.util.Utils + +/** + * This's an end to end test suite used to test the host local shuffle reading. + */ +class HostLocalShuffleReadingSuite extends SparkFunSuite with Matchers with LocalSparkContext { + var rpcHandler: ExternalBlockHandler = _ + var server: TransportServer = _ + var transportContext: TransportContext = _ + + override def afterEach(): Unit = { + Option(rpcHandler).foreach { handler => + Utils.tryLogNonFatalError{ + server.close() + } + Utils.tryLogNonFatalError{ + handler.close() + } + Utils.tryLogNonFatalError{ + transportContext.close() + } + server = null + rpcHandler = null + transportContext = null + } + super.afterEach() + } + + Seq(true, false).foreach { isESSEnabled => /* ESS: external shuffle service */ + val conf = new SparkConf() + .set(SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED, true) + + import scala.language.existentials + val (essStatus, blockStoreClientClass) = if (isESSEnabled) { + // LocalSparkCluster will disable the ExternalShuffleService by default. Therefore, + // we have to manually setup an server which embedded with ExternalBlockHandler to + // mimic a ExternalShuffleService. Then, executors on the Worker can successfully + // find a ExternalShuffleService to connect. + val transportConf = SparkTransportConf.fromSparkConf(conf, "shuffle", numUsableCores = 2) + rpcHandler = new ExternalBlockHandler(transportConf, null) + transportContext = new TransportContext(transportConf, rpcHandler) + server = transportContext.createServer() + conf.set(SHUFFLE_SERVICE_PORT, server.getPort) + + ("enabled (SPARK-27651)", classOf[ExternalBlockStoreClient]) + } else { + ("disabled (SPARK-32077)", classOf[NettyBlockTransferService]) + } + + test(s"host local shuffle reading with external shuffle service $essStatus") { + conf.set(SHUFFLE_SERVICE_ENABLED, isESSEnabled) + .set(STORAGE_LOCAL_DISK_BY_EXECUTORS_CACHE_SIZE, 5) + sc = new SparkContext("local-cluster[2,1,1024]", "test-host-local-shuffle-reading", conf) + // In a slow machine, one executor may register hundreds of milliseconds ahead of the other + // one. If we don't wait for all executors, it's possible that only one executor runs all + // jobs. Then all shuffle blocks will be in this executor, ShuffleBlockFetcherIterator will + // directly fetch local blocks from the local BlockManager and won't send requests to + // BlockStoreClient. In this case, we won't receive FetchFailed. And it will make this + // test fail. Therefore, we should wait until all executors are up + TestUtils.waitUntilExecutorsUp(sc, 2, 60000) + + sc.getConf.get(SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED) should equal(true) + sc.env.blockManager.externalShuffleServiceEnabled should equal(isESSEnabled) + sc.env.blockManager.hostLocalDirManager.isDefined should equal(true) + sc.env.blockManager.blockStoreClient.getClass should equal(blockStoreClientClass) + + val rdd = sc.parallelize(0 until 1000, 10) + .map { i => + SparkEnv.get.blockManager.hostLocalDirManager.map { localDirManager => + // No shuffle fetch yet. So the cache must be empty + assert(localDirManager.getCachedHostLocalDirs.isEmpty) + } + (i, 1) + }.reduceByKey(_ + _) + + // raise a job and trigger the shuffle fetching during the job + assert(rdd.count() === 1000) + + val cachedExecutors = rdd.mapPartitions { _ => + SparkEnv.get.blockManager.hostLocalDirManager.map { localDirManager => + localDirManager.getCachedHostLocalDirs.keySet.iterator + }.getOrElse(Iterator.empty) + }.collect().toSet + + // both executors are caching the dirs of the other one + cachedExecutors should equal(sc.getExecutorIds().toSet) + + Option(rpcHandler).foreach { handler => + // Invalidate the registered executors, disallowing access to their shuffle blocks (without + // deleting the actual shuffle files, so we could access them without the shuffle service). + // As directories are already cached there is no request to external shuffle service. + handler.applicationRemoved(sc.conf.getAppId, false /* cleanupLocalDirs */) + } + + val (local, remote) = rdd.map { case (_, _) => + val shuffleReadMetrics = TaskContext.get().taskMetrics().shuffleReadMetrics + ((shuffleReadMetrics.localBytesRead, shuffleReadMetrics.localBlocksFetched), + (shuffleReadMetrics.remoteBytesRead, shuffleReadMetrics.remoteBlocksFetched)) + }.collect().unzip + // Spark should read the shuffle data locally from the cached directories on the same host, + // so there's no remote fetching at all. + val (localBytesRead, localBlocksFetched) = local.unzip + val (remoteBytesRead, remoteBlocksFetched) = remote.unzip + assert(localBytesRead.sum > 0 && localBlocksFetched.sum > 0) + assert(remoteBytesRead.sum === 0 && remoteBlocksFetched.sum === 0) + } + } +} diff --git a/core/src/test/scala/org/apache/spark/shuffle/sort/IndexShuffleBlockResolverSuite.scala b/core/src/test/scala/org/apache/spark/shuffle/sort/IndexShuffleBlockResolverSuite.scala index 725a1d90557a2..91260d01eb8b6 100644 --- a/core/src/test/scala/org/apache/spark/shuffle/sort/IndexShuffleBlockResolverSuite.scala +++ b/core/src/test/scala/org/apache/spark/shuffle/sort/IndexShuffleBlockResolverSuite.scala @@ -156,4 +156,9 @@ class IndexShuffleBlockResolverSuite extends SparkFunSuite with BeforeAndAfterEa indexIn2.close() } } + + test("SPARK-33198 getMigrationBlocks should not fail at missing files") { + val resolver = new IndexShuffleBlockResolver(conf, blockManager) + assert(resolver.getMigrationBlocks(ShuffleBlockInfo(Int.MaxValue, Long.MaxValue)).isEmpty) + } } diff --git a/core/src/test/scala/org/apache/spark/status/api/v1/ExecutorSummarySuite.scala b/core/src/test/scala/org/apache/spark/status/api/v1/ExecutorSummarySuite.scala new file mode 100644 index 0000000000000..286911bdfc19a --- /dev/null +++ b/core/src/test/scala/org/apache/spark/status/api/v1/ExecutorSummarySuite.scala @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.status.api.v1 + +import java.util.Date + +import com.fasterxml.jackson.core.`type`.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule + +import org.apache.spark.SparkFunSuite + +class ExecutorSummarySuite extends SparkFunSuite { + + test("Check ExecutorSummary serialize and deserialize with empty peakMemoryMetrics") { + val mapper = new ObjectMapper().registerModule(DefaultScalaModule) + val executorSummary = new ExecutorSummary("id", "host:port", true, 1, + 10, 10, 1, 1, 1, + 0, 0, 1, 100, + 1, 100, 100, + 10, false, 20, new Date(1600984336352L), + Option.empty, Option.empty, Map(), Option.empty, Set(), Option.empty, Map(), Map(), 1) + val expectedJson = "{\"id\":\"id\",\"hostPort\":\"host:port\",\"isActive\":true," + + "\"rddBlocks\":1,\"memoryUsed\":10,\"diskUsed\":10,\"totalCores\":1,\"maxTasks\":1," + + "\"activeTasks\":1,\"failedTasks\":0,\"completedTasks\":0,\"totalTasks\":1," + + "\"totalDuration\":100,\"totalGCTime\":1,\"totalInputBytes\":100," + + "\"totalShuffleRead\":100,\"totalShuffleWrite\":10,\"isBlacklisted\":false," + + "\"maxMemory\":20,\"addTime\":1600984336352,\"removeTime\":null,\"removeReason\":null," + + "\"executorLogs\":{},\"memoryMetrics\":null,\"blacklistedInStages\":[]," + + "\"peakMemoryMetrics\":null,\"attributes\":{},\"resources\":{},\"resourceProfileId\":1}" + val json = mapper.writeValueAsString(executorSummary) + assert(expectedJson.equals(json)) + val deserializeExecutorSummary = mapper.readValue(json, new TypeReference[ExecutorSummary] {}) + assert(deserializeExecutorSummary.peakMemoryMetrics == None) + } + +} diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionIntegrationSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionIntegrationSuite.scala index 6a52f72938c6c..bb685cd353ddc 100644 --- a/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionIntegrationSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionIntegrationSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.storage -import java.util.concurrent.{ConcurrentHashMap, ConcurrentLinkedQueue, Semaphore} +import java.util.concurrent.{ConcurrentHashMap, ConcurrentLinkedQueue, Semaphore, TimeUnit} import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer @@ -40,7 +40,47 @@ class BlockManagerDecommissionIntegrationSuite extends SparkFunSuite with LocalS val TaskEnded = "TASK_ENDED" val JobEnded = "JOB_ENDED" - test(s"verify that an already running task which is going to cache data succeeds " + + Seq(false, true).foreach { isEnabled => + test(s"SPARK-32850: BlockManager decommission should respect the configuration " + + s"(enabled=${isEnabled})") { + val conf = new SparkConf() + .setAppName("test-blockmanager-decommissioner") + .setMaster("local-cluster[2, 1, 1024]") + .set(config.DECOMMISSION_ENABLED, true) + .set(config.STORAGE_DECOMMISSION_ENABLED, isEnabled) + sc = new SparkContext(conf) + TestUtils.waitUntilExecutorsUp(sc, 2, 6000) + val executors = sc.getExecutorIds().toArray + val decommissionListener = new SparkListener { + override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = { + // ensure Tasks launched at executors before they're marked as decommissioned by driver + Thread.sleep(3000) + sc.schedulerBackend.asInstanceOf[StandaloneSchedulerBackend] + .decommissionExecutors( + executors.map { id => (id, ExecutorDecommissionInfo("test")) }, + true, + false) + } + } + sc.addSparkListener(decommissionListener) + + val decommissionStatus: Seq[Boolean] = sc.parallelize(1 to 100, 2).mapPartitions { _ => + val startTime = System.currentTimeMillis() + while (SparkEnv.get.blockManager.decommissioner.isEmpty && + // wait at most 6 seconds for BlockManager to start to decommission (if enabled) + System.currentTimeMillis() - startTime < 6000) { + Thread.sleep(300) + } + val blockManagerDecommissionStatus = + if (SparkEnv.get.blockManager.decommissioner.isEmpty) false else true + Iterator.single(blockManagerDecommissionStatus) + }.collect() + assert(decommissionStatus.forall(_ == isEnabled)) + sc.removeSparkListener(decommissionListener) + } + } + + testRetry(s"verify that an already running task which is going to cache data succeeds " + s"on a decommissioned executor after task start") { runDecomTest(true, false, TaskStarted) } @@ -65,13 +105,15 @@ class BlockManagerDecommissionIntegrationSuite extends SparkFunSuite with LocalS val migrateDuring = whenToDecom != JobEnded val master = s"local-cluster[${numExecs}, 1, 1024]" val conf = new SparkConf().setAppName("test").setMaster(master) - .set(config.Worker.WORKER_DECOMMISSION_ENABLED, true) + .set(config.DECOMMISSION_ENABLED, true) .set(config.STORAGE_DECOMMISSION_ENABLED, true) .set(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED, persist) .set(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED, shuffle) - // Just replicate blocks as fast as we can during testing, there isn't another + // Since we use the bus for testing we don't want to drop any messages + .set(config.LISTENER_BUS_EVENT_QUEUE_CAPACITY, 1000000) + // Just replicate blocks quickly during testing, there isn't another // workload we need to worry about. - .set(config.STORAGE_DECOMMISSION_REPLICATION_REATTEMPT_INTERVAL, 1L) + .set(config.STORAGE_DECOMMISSION_REPLICATION_REATTEMPT_INTERVAL, 10L) if (whenToDecom == TaskStarted) { // We are using accumulators below, make sure those are reported frequently. @@ -89,7 +131,7 @@ class BlockManagerDecommissionIntegrationSuite extends SparkFunSuite with LocalS val sleepIntervalMs = whenToDecom match { // Increase the window of time b/w task started and ended so that we can decom within that. - case TaskStarted => 2000 + case TaskStarted => 10000 // Make one task take a really short time so that we can decommission right after it is // done but before its peers are done. case TaskEnded => @@ -137,7 +179,7 @@ class BlockManagerDecommissionIntegrationSuite extends SparkFunSuite with LocalS taskEndEvents.add(taskEnd) } - override def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit = { + override def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit = synchronized { blocksUpdated.append(blockUpdated) } @@ -176,11 +218,11 @@ class BlockManagerDecommissionIntegrationSuite extends SparkFunSuite with LocalS } else { 10.milliseconds } - eventually(timeout(6.seconds), interval(intervalMs)) { + eventually(timeout(20.seconds), interval(intervalMs)) { assert(getCandidateExecutorToDecom.isDefined) } } else { - ThreadUtils.awaitResult(asyncCount, 15.seconds) + ThreadUtils.awaitResult(asyncCount, 1.minute) } // Decommission one of the executors. @@ -188,13 +230,16 @@ class BlockManagerDecommissionIntegrationSuite extends SparkFunSuite with LocalS val execToDecommission = getCandidateExecutorToDecom.get logInfo(s"Decommissioning executor ${execToDecommission}") + + // Decommission executor and ensure it is not relaunched by setting adjustTargetNumExecutors sched.decommissionExecutor( execToDecommission, - ExecutorDecommissionInfo("", isHostDecommissioned = false)) + ExecutorDecommissionInfo("", None), + adjustTargetNumExecutors = true) val decomTime = new SystemClock().getTimeMillis() // Wait for job to finish. - val asyncCountResult = ThreadUtils.awaitResult(asyncCount, 15.seconds) + val asyncCountResult = ThreadUtils.awaitResult(asyncCount, 1.minute) assert(asyncCountResult === numParts) // All tasks finished, so accum should have been increased numParts times. assert(accum.value === numParts) @@ -226,7 +271,7 @@ class BlockManagerDecommissionIntegrationSuite extends SparkFunSuite with LocalS } // Wait for our respective blocks to have migrated - eventually(timeout(30.seconds), interval(10.milliseconds)) { + eventually(timeout(1.minute), interval(10.milliseconds)) { if (persist) { // One of our blocks should have moved. val rddUpdates = blocksUpdated.filter { update => @@ -266,18 +311,19 @@ class BlockManagerDecommissionIntegrationSuite extends SparkFunSuite with LocalS val execIdToBlocksMapping = storageStatus.map( status => (status.blockManagerId.executorId, status.blocks)).toMap // No cached blocks should be present on executor which was decommissioned - assert(execIdToBlocksMapping(execToDecommission).keys.filter(_.isRDD).toSeq === Seq(), + assert( + !execIdToBlocksMapping.contains(execToDecommission) || + execIdToBlocksMapping(execToDecommission).keys.filter(_.isRDD).toSeq === Seq(), "Cache blocks should be migrated") if (persist) { // There should still be all the RDD blocks cached assert(execIdToBlocksMapping.values.flatMap(_.keys).count(_.isRDD) === numParts) } - // Make the executor we decommissioned exit - sched.client.killExecutors(List(execToDecommission)) - - // Wait for the executor to be removed - executorRemovedSem.acquire(1) + // Wait for the executor to be removed automatically after migration. + // This is set to a high value since github actions is sometimes high latency + // but I've never seen this go for more than a minute. + assert(executorRemovedSem.tryAcquire(1, 5L, TimeUnit.MINUTES)) // Since the RDD is cached or shuffled so further usage of same RDD should use the // cached data. Original RDD partitions should not be recomputed i.e. accum diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionUnitSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionUnitSuite.scala index 41b68d5978d16..b7ac378b4c6cd 100644 --- a/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionUnitSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerDecommissionUnitSuite.scala @@ -20,7 +20,7 @@ package org.apache.spark.storage import scala.concurrent.duration._ import org.mockito.{ArgumentMatchers => mc} -import org.mockito.Mockito.{mock, times, verify, when} +import org.mockito.Mockito.{atLeast => least, mock, times, verify, when} import org.scalatest.concurrent.Eventually._ import org.scalatest.matchers.must.Matchers @@ -38,6 +38,9 @@ class BlockManagerDecommissionUnitSuite extends SparkFunSuite with Matchers { private val sparkConf = new SparkConf(false) .set(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED, true) .set(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED, true) + // Just replicate blocks quickly during testing, as there isn't another + // workload we need to worry about. + .set(config.STORAGE_DECOMMISSION_REPLICATION_REATTEMPT_INTERVAL, 10L) private def registerShuffleBlocks( mockMigratableShuffleResolver: MigratableResolver, @@ -54,6 +57,195 @@ class BlockManagerDecommissionUnitSuite extends SparkFunSuite with Matchers { } } + /** + * Validate a given configuration with the mocks. + * The fail variable controls if we expect migration to fail, in which case we expect + * a constant Long.MaxValue timestamp. + */ + private def validateDecommissionTimestamps(conf: SparkConf, bm: BlockManager, + fail: Boolean = false, assertDone: Boolean = true) = { + // Verify the decommissioning manager timestamps and status + val bmDecomManager = new BlockManagerDecommissioner(conf, bm) + validateDecommissionTimestampsOnManager(bmDecomManager, fail, assertDone) + } + + private def validateDecommissionTimestampsOnManager(bmDecomManager: BlockManagerDecommissioner, + fail: Boolean = false, assertDone: Boolean = true, numShuffles: Option[Int] = None) = { + var previousTime: Option[Long] = None + try { + bmDecomManager.start() + eventually(timeout(100.second), interval(10.milliseconds)) { + val (currentTime, done) = bmDecomManager.lastMigrationInfo() + assert(!assertDone || done) + // Make sure the time stamp starts moving forward. + if (!fail) { + previousTime match { + case None => + previousTime = Some(currentTime) + assert(false) + case Some(t) => + assert(t < currentTime) + } + } else { + // If we expect migration to fail we should get the max value quickly. + assert(currentTime === Long.MaxValue) + } + numShuffles.foreach { s => + assert(bmDecomManager.numMigratedShuffles.get() === s) + } + } + if (!fail) { + // Wait 5 seconds and assert times keep moving forward. + Thread.sleep(5000) + val (currentTime, done) = bmDecomManager.lastMigrationInfo() + assert((!assertDone || done) && currentTime > previousTime.get) + } + } finally { + bmDecomManager.stop() + } + } + + test("test that with no blocks we finish migration") { + // Set up the mocks so we return empty + val bm = mock(classOf[BlockManager]) + val migratableShuffleBlockResolver = mock(classOf[MigratableResolver]) + when(migratableShuffleBlockResolver.getStoredShuffles()) + .thenReturn(Seq()) + when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver) + when(bm.getMigratableRDDBlocks()) + .thenReturn(Seq()) + when(bm.getPeers(mc.any())) + .thenReturn(Seq(BlockManagerId("exec2", "host2", 12345))) + + // Verify the decom manager handles this correctly + validateDecommissionTimestamps(sparkConf, bm) + } + + test("block decom manager with no migrations configured") { + val bm = mock(classOf[BlockManager]) + val migratableShuffleBlockResolver = mock(classOf[MigratableResolver]) + registerShuffleBlocks(migratableShuffleBlockResolver, Set((1, 1L, 1))) + when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver) + when(bm.getMigratableRDDBlocks()) + .thenReturn(Seq()) + when(bm.getPeers(mc.any())) + .thenReturn(Seq(BlockManagerId("exec2", "host2", 12345))) + + val badConf = new SparkConf(false) + .set(config.STORAGE_DECOMMISSION_SHUFFLE_BLOCKS_ENABLED, false) + .set(config.STORAGE_DECOMMISSION_RDD_BLOCKS_ENABLED, false) + .set(config.STORAGE_DECOMMISSION_REPLICATION_REATTEMPT_INTERVAL, 10L) + // Verify the decom manager handles this correctly + validateDecommissionTimestamps(badConf, bm, fail = true) + } + + test("block decom manager with no peers") { + // Set up the mocks so we return one shuffle block + val bm = mock(classOf[BlockManager]) + val migratableShuffleBlockResolver = mock(classOf[MigratableResolver]) + registerShuffleBlocks(migratableShuffleBlockResolver, Set((1, 1L, 1))) + when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver) + when(bm.getMigratableRDDBlocks()) + .thenReturn(Seq()) + when(bm.getPeers(mc.any())) + .thenReturn(Seq()) + + // Verify the decom manager handles this correctly + validateDecommissionTimestamps(sparkConf, bm, fail = true) + } + + + test("block decom manager with only shuffle files time moves forward") { + // Set up the mocks so we return one shuffle block + val bm = mock(classOf[BlockManager]) + val migratableShuffleBlockResolver = mock(classOf[MigratableResolver]) + registerShuffleBlocks(migratableShuffleBlockResolver, Set((1, 1L, 1))) + when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver) + when(bm.getMigratableRDDBlocks()) + .thenReturn(Seq()) + when(bm.getPeers(mc.any())) + .thenReturn(Seq(BlockManagerId("exec2", "host2", 12345))) + + // Verify the decom manager handles this correctly + validateDecommissionTimestamps(sparkConf, bm) + } + + test("block decom manager does not re-add removed shuffle files") { + // Set up the mocks so we return one shuffle block + val bm = mock(classOf[BlockManager]) + val migratableShuffleBlockResolver = mock(classOf[MigratableResolver]) + registerShuffleBlocks(migratableShuffleBlockResolver, Set()) + when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver) + when(bm.getMigratableRDDBlocks()) + .thenReturn(Seq()) + when(bm.getPeers(mc.any())) + .thenReturn(Seq(BlockManagerId("exec2", "host2", 12345))) + val bmDecomManager = new BlockManagerDecommissioner(sparkConf, bm) + bmDecomManager.migratingShuffles += ShuffleBlockInfo(10, 10) + + validateDecommissionTimestampsOnManager(bmDecomManager, fail = false, assertDone = false) + } + + test("block decom manager handles IO failures") { + // Set up the mocks so we return one shuffle block + val bm = mock(classOf[BlockManager]) + val migratableShuffleBlockResolver = mock(classOf[MigratableResolver]) + registerShuffleBlocks(migratableShuffleBlockResolver, Set((1, 1L, 1))) + when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver) + when(bm.getMigratableRDDBlocks()) + .thenReturn(Seq()) + when(bm.getPeers(mc.any())) + .thenReturn(Seq(BlockManagerId("exec2", "host2", 12345))) + + val blockTransferService = mock(classOf[BlockTransferService]) + // Simulate an ambiguous IO error (e.g. block could be gone, connection failed, etc.) + when(blockTransferService.uploadBlockSync( + mc.any(), mc.any(), mc.any(), mc.any(), mc.any(), mc.any(), mc.isNull())).thenThrow( + new java.io.IOException("boop") + ) + + when(bm.blockTransferService).thenReturn(blockTransferService) + + // Verify the decom manager handles this correctly + val bmDecomManager = new BlockManagerDecommissioner(sparkConf, bm) + validateDecommissionTimestampsOnManager(bmDecomManager, fail = false) + } + + test("block decom manager short circuits removed blocks") { + // Set up the mocks so we return one shuffle block + val bm = mock(classOf[BlockManager]) + val migratableShuffleBlockResolver = mock(classOf[MigratableResolver]) + // First call get blocks, then empty list simulating a delete. + when(migratableShuffleBlockResolver.getStoredShuffles()) + .thenReturn(Seq(ShuffleBlockInfo(1, 1))) + .thenReturn(Seq()) + when(migratableShuffleBlockResolver.getMigrationBlocks(mc.any())) + .thenReturn(List( + (ShuffleIndexBlockId(1, 1, 1), mock(classOf[ManagedBuffer])), + (ShuffleDataBlockId(1, 1, 1), mock(classOf[ManagedBuffer])))) + .thenReturn(List()) + + when(bm.migratableResolver).thenReturn(migratableShuffleBlockResolver) + when(bm.getMigratableRDDBlocks()) + .thenReturn(Seq()) + when(bm.getPeers(mc.any())) + .thenReturn(Seq(BlockManagerId("exec2", "host2", 12345))) + + val blockTransferService = mock(classOf[BlockTransferService]) + // Simulate an ambiguous IO error (e.g. block could be gone, connection failed, etc.) + when(blockTransferService.uploadBlockSync( + mc.any(), mc.any(), mc.any(), mc.any(), mc.any(), mc.any(), mc.isNull())).thenThrow( + new java.io.IOException("boop") + ) + + when(bm.blockTransferService).thenReturn(blockTransferService) + + // Verify the decom manager handles this correctly + val bmDecomManager = new BlockManagerDecommissioner(sparkConf, bm) + validateDecommissionTimestampsOnManager(bmDecomManager, fail = false, + numShuffles = Some(1)) + } + test("test shuffle and cached rdd migration without any error") { val blockTransferService = mock(classOf[BlockTransferService]) val bm = mock(classOf[BlockManager]) @@ -77,13 +269,37 @@ class BlockManagerDecommissionUnitSuite extends SparkFunSuite with Matchers { try { bmDecomManager.start() - eventually(timeout(5.second), interval(10.milliseconds)) { - assert(bmDecomManager.shufflesToMigrate.isEmpty == true) - verify(bm, times(1)).replicateBlock( + var previousRDDTime: Option[Long] = None + var previousShuffleTime: Option[Long] = None + + // We don't check that all blocks are migrated because out mock is always returning an RDD. + eventually(timeout(100.second), interval(10.milliseconds)) { + assert(bmDecomManager.shufflesToMigrate.isEmpty === true) + assert(bmDecomManager.numMigratedShuffles.get() === 1) + verify(bm, least(1)).replicateBlock( mc.eq(storedBlockId1), mc.any(), mc.any(), mc.eq(Some(3))) verify(blockTransferService, times(2)) .uploadBlockSync(mc.eq("host2"), mc.eq(bmPort), mc.eq("exec2"), mc.any(), mc.any(), mc.eq(StorageLevel.DISK_ONLY), mc.isNull()) + // Since we never "finish" the RDD blocks, make sure the time is always moving forward. + assert(bmDecomManager.rddBlocksLeft) + previousRDDTime match { + case None => + previousRDDTime = Some(bmDecomManager.lastRDDMigrationTime) + assert(false) + case Some(t) => + assert(bmDecomManager.lastRDDMigrationTime > t) + } + // Since we do eventually finish the shuffle blocks make sure the shuffle blocks complete + // and that the time keeps moving forward. + assert(!bmDecomManager.shuffleBlocksLeft) + previousShuffleTime match { + case None => + previousShuffleTime = Some(bmDecomManager.lastShuffleMigrationTime) + assert(false) + case Some(t) => + assert(bmDecomManager.lastShuffleMigrationTime > t) + } } } finally { bmDecomManager.stop() diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala index 2a3d2d7f86a7e..5450a4b67c00b 100644 --- a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala @@ -144,10 +144,28 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE blockManager } + // Save modified system properties so that we can restore them after tests. + val originalArch = System.getProperty("os.arch") + val originalCompressedOops = System.getProperty(TEST_USE_COMPRESSED_OOPS_KEY) + + def reinitializeSizeEstimator(arch: String, useCompressedOops: String): Unit = { + def set(k: String, v: String): Unit = { + if (v == null) { + System.clearProperty(k) + } else { + System.setProperty(k, v) + } + } + set("os.arch", arch) + set(TEST_USE_COMPRESSED_OOPS_KEY, useCompressedOops) + val initialize = PrivateMethod[Unit](Symbol("initialize")) + SizeEstimator invokePrivate initialize() + } + override def beforeEach(): Unit = { super.beforeEach() // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case - System.setProperty("os.arch", "amd64") + reinitializeSizeEstimator("amd64", "true") conf = new SparkConf(false) init(conf) @@ -168,12 +186,12 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE liveListenerBus, None, blockManagerInfo, mapOutputTracker)), rpcEnv.setupEndpoint("blockmanagerHeartbeat", new BlockManagerMasterHeartbeatEndpoint(rpcEnv, true, blockManagerInfo)), conf, true)) - - val initialize = PrivateMethod[Unit](Symbol("initialize")) - SizeEstimator invokePrivate initialize() } override def afterEach(): Unit = { + // Restore system properties and SizeEstimator to their original states. + reinitializeSizeEstimator(originalArch, originalCompressedOops) + try { conf = null allStores.foreach(_.stop()) @@ -222,7 +240,7 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE val driverEndpoint = rpcEnv.setupEndpoint(CoarseGrainedSchedulerBackend.ENDPOINT_NAME, new RpcEndpoint { private val executorSet = mutable.HashSet[String]() - override val rpcEnv: RpcEnv = this.rpcEnv + override val rpcEnv: RpcEnv = BlockManagerSuite.this.rpcEnv override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = { case CoarseGrainedClusterMessages.RegisterExecutor(executorId, _, _, _, _, _, _, _) => executorSet += executorId @@ -236,7 +254,7 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE def createAndRegisterBlockManager(timeout: Boolean): BlockManagerId = { val id = if (timeout) "timeout" else "normal" val bmRef = rpcEnv.setupEndpoint(s"bm-$id", new RpcEndpoint { - override val rpcEnv: RpcEnv = this.rpcEnv + override val rpcEnv: RpcEnv = BlockManagerSuite.this.rpcEnv private def reply[T](context: RpcCallContext, response: T): Unit = { if (timeout) { Thread.sleep(conf.getTimeAsMs(Network.RPC_ASK_TIMEOUT.key) + 1000) diff --git a/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala b/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala index ccd7e4b62ad9e..c46ab2d199f0b 100644 --- a/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala @@ -26,6 +26,7 @@ import org.scalatest._ import org.apache.spark._ import org.apache.spark.internal.config._ +import org.apache.spark.internal.config.Tests.TEST_USE_COMPRESSED_OOPS_KEY import org.apache.spark.memory.{MemoryMode, UnifiedMemoryManager} import org.apache.spark.serializer.{KryoSerializer, SerializerManager} import org.apache.spark.storage.memory.{BlockEvictionHandler, MemoryStore, PartiallySerializedBlock, PartiallyUnrolledIterator} @@ -51,12 +52,34 @@ class MemoryStoreSuite implicit def StringToBlockId(value: String): BlockId = new TestBlockId(value) def rdd(rddId: Int, splitId: Int): RDDBlockId = RDDBlockId(rddId, splitId) + // Save modified system properties so that we can restore them after tests. + val originalArch = System.getProperty("os.arch") + val originalCompressedOops = System.getProperty(TEST_USE_COMPRESSED_OOPS_KEY) + + def reinitializeSizeEstimator(arch: String, useCompressedOops: String): Unit = { + def set(k: String, v: String): Unit = { + if (v == null) { + System.clearProperty(k) + } else { + System.setProperty(k, v) + } + } + set("os.arch", arch) + set(TEST_USE_COMPRESSED_OOPS_KEY, useCompressedOops) + val initialize = PrivateMethod[Unit](Symbol("initialize")) + SizeEstimator invokePrivate initialize() + } + override def beforeEach(): Unit = { super.beforeEach() // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case - System.setProperty("os.arch", "amd64") - val initialize = PrivateMethod[Unit](Symbol("initialize")) - SizeEstimator invokePrivate initialize() + reinitializeSizeEstimator("amd64", "true") + } + + override def afterEach(): Unit = { + super.afterEach() + // Restore system properties and SizeEstimator to their original states. + reinitializeSizeEstimator(originalArch, originalCompressedOops) } def makeMemoryStore(maxMem: Long): (MemoryStore, BlockInfoManager) = { diff --git a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala index bf1379ceb89a8..99c43b12d6553 100644 --- a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala @@ -66,6 +66,16 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT transfer } + private def createMockBlockManager(): BlockManager = { + val blockManager = mock(classOf[BlockManager]) + val localBmId = BlockManagerId("test-client", "test-local-host", 1) + doReturn(localBmId).when(blockManager).blockManagerId + // By default, the mock BlockManager returns None for hostLocalDirManager. One could + // still use initHostLocalDirManager() to specify a custom hostLocalDirManager. + doReturn(None).when(blockManager).hostLocalDirManager + blockManager + } + private def initHostLocalDirManager( blockManager: BlockManager, hostLocalDirs: Map[String, Array[String]]): Unit = { @@ -73,9 +83,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT val hostLocalDirManager = new HostLocalDirManager( futureExecutionContext = global, cacheSize = 1, - externalBlockStoreClient = mockExternalBlockStoreClient, - host = "localhost", - externalShuffleServicePort = 7337) + blockStoreClient = mockExternalBlockStoreClient) when(blockManager.hostLocalDirManager).thenReturn(Some(hostLocalDirManager)) when(mockExternalBlockStoreClient.getHostLocalDirs(any(), any(), any(), any())) @@ -116,9 +124,8 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT } test("successful 3 local + 4 host local + 2 remote reads") { - val blockManager = mock(classOf[BlockManager]) - val localBmId = BlockManagerId("test-local-client", "test-local-host", 1) - doReturn(localBmId).when(blockManager).blockManagerId + val blockManager = createMockBlockManager() + val localBmId = blockManager.blockManagerId // Make sure blockManager.getBlockData would return the blocks val localBlocks = Map[BlockId, ManagedBuffer]( @@ -197,13 +204,11 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT // 2 remote blocks are read from the same block manager verify(transfer, times(1)).fetchBlocks(any(), any(), any(), any(), any(), any()) - assert(blockManager.hostLocalDirManager.get.getCachedHostLocalDirs().size === 1) + assert(blockManager.hostLocalDirManager.get.getCachedHostLocalDirs.size === 1) } test("error during accessing host local dirs for executors") { - val blockManager = mock(classOf[BlockManager]) - val localBmId = BlockManagerId("test-local-client", "test-local-host", 1) - doReturn(localBmId).when(blockManager).blockManagerId + val blockManager = createMockBlockManager() val hostLocalBlocks = Map[BlockId, ManagedBuffer]( ShuffleBlockId(0, 1, 0) -> createMockManagedBuffer()) @@ -218,9 +223,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT val hostLocalDirManager = new HostLocalDirManager( futureExecutionContext = global, cacheSize = 1, - externalBlockStoreClient = mockExternalBlockStoreClient, - host = "localhost", - externalShuffleServicePort = 7337) + blockStoreClient = mockExternalBlockStoreClient) when(blockManager.hostLocalDirManager).thenReturn(Some(hostLocalDirManager)) when(mockExternalBlockStoreClient.getHostLocalDirs(any(), any(), any(), any())) @@ -256,10 +259,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT } test("Hit maxBytesInFlight limitation before maxBlocksInFlightPerAddress") { - val blockManager = mock(classOf[BlockManager]) - val localBmId = BlockManagerId("test-client", "test-local-host", 1) - doReturn(localBmId).when(blockManager).blockManagerId - + val blockManager = createMockBlockManager() val remoteBmId1 = BlockManagerId("test-remote-client-1", "test-remote-host1", 1) val remoteBmId2 = BlockManagerId("test-remote-client-2", "test-remote-host2", 2) val blockId1 = ShuffleBlockId(0, 1, 0) @@ -301,10 +301,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT } test("Hit maxBlocksInFlightPerAddress limitation before maxBytesInFlight") { - val blockManager = mock(classOf[BlockManager]) - val localBmId = BlockManagerId("test-client", "test-local-host", 1) - doReturn(localBmId).when(blockManager).blockManagerId - + val blockManager = createMockBlockManager() val remoteBmId = BlockManagerId("test-remote-client-1", "test-remote-host", 2) val blockId1 = ShuffleBlockId(0, 1, 0) val blockId2 = ShuffleBlockId(0, 2, 0) @@ -348,10 +345,8 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT } test("fetch continuous blocks in batch successful 3 local + 4 host local + 2 remote reads") { - val blockManager = mock(classOf[BlockManager]) - val localBmId = BlockManagerId("test-client", "test-local-host", 1) - doReturn(localBmId).when(blockManager).blockManagerId - + val blockManager = createMockBlockManager() + val localBmId = blockManager.blockManagerId // Make sure blockManager.getBlockData would return the merged block val localBlocks = Seq[BlockId]( ShuffleBlockId(0, 0, 0), @@ -431,14 +426,11 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT verify(blockManager, times(1)) .getHostLocalShuffleData(any(), meq(Array("local-dir"))) - assert(blockManager.hostLocalDirManager.get.getCachedHostLocalDirs().size === 1) + assert(blockManager.hostLocalDirManager.get.getCachedHostLocalDirs.size === 1) } test("fetch continuous blocks in batch should respect maxBytesInFlight") { - val blockManager = mock(classOf[BlockManager]) - val localBmId = BlockManagerId("test-client", "test-local-host", 1) - doReturn(localBmId).when(blockManager).blockManagerId - + val blockManager = createMockBlockManager() // Make sure remote blocks would return the merged block val remoteBmId1 = BlockManagerId("test-client-1", "test-client-1", 1) val remoteBmId2 = BlockManagerId("test-client-2", "test-client-2", 2) @@ -494,10 +486,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT } test("fetch continuous blocks in batch should respect maxBlocksInFlightPerAddress") { - val blockManager = mock(classOf[BlockManager]) - val localBmId = BlockManagerId("test-client", "test-local-host", 1) - doReturn(localBmId).when(blockManager).blockManagerId - + val blockManager = createMockBlockManager() // Make sure remote blocks would return the merged block val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 1) val remoteBlocks = Seq( @@ -549,10 +538,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT } test("release current unexhausted buffer in case the task completes early") { - val blockManager = mock(classOf[BlockManager]) - val localBmId = BlockManagerId("test-client", "test-client", 1) - doReturn(localBmId).when(blockManager).blockManagerId - + val blockManager = createMockBlockManager() // Make sure remote blocks would return val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 2) val blocks = Map[BlockId, ManagedBuffer]( @@ -617,10 +603,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT } test("fail all blocks if any of the remote request fails") { - val blockManager = mock(classOf[BlockManager]) - val localBmId = BlockManagerId("test-client", "test-client", 1) - doReturn(localBmId).when(blockManager).blockManagerId - + val blockManager = createMockBlockManager() // Make sure remote blocks would return val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 2) val blocks = Map[BlockId, ManagedBuffer]( @@ -707,10 +690,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT } test("retry corrupt blocks") { - val blockManager = mock(classOf[BlockManager]) - val localBmId = BlockManagerId("test-client", "test-client", 1) - doReturn(localBmId).when(blockManager).blockManagerId - + val blockManager = createMockBlockManager() // Make sure remote blocks would return val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 2) val blocks = Map[BlockId, ManagedBuffer]( @@ -785,9 +765,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT test("big blocks are also checked for corruption") { val streamLength = 10000L - val blockManager = mock(classOf[BlockManager]) - val localBlockManagerId = BlockManagerId("local-client", "local-client", 1) - doReturn(localBlockManagerId).when(blockManager).blockManagerId + val blockManager = createMockBlockManager() // This stream will throw IOException when the first byte is read val corruptBuffer1 = mockCorruptBuffer(streamLength, 0) @@ -906,10 +884,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT } test("retry corrupt blocks (disabled)") { - val blockManager = mock(classOf[BlockManager]) - val localBmId = BlockManagerId("test-client", "test-client", 1) - doReturn(localBmId).when(blockManager).blockManagerId - + val blockManager = createMockBlockManager() // Make sure remote blocks would return val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 2) val blocks = Map[BlockId, ManagedBuffer]( @@ -971,10 +946,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT test("Blocks should be shuffled to disk when size of the request is above the" + " threshold(maxReqSizeShuffleToMem).") { - val blockManager = mock(classOf[BlockManager]) - val localBmId = BlockManagerId("test-client", "test-client", 1) - doReturn(localBmId).when(blockManager).blockManagerId - + val blockManager = createMockBlockManager() val diskBlockManager = mock(classOf[DiskBlockManager]) val tmpDir = Utils.createTempDir() doReturn{ @@ -1036,10 +1008,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT } test("fail zero-size blocks") { - val blockManager = mock(classOf[BlockManager]) - val localBmId = BlockManagerId("test-client", "test-client", 1) - doReturn(localBmId).when(blockManager).blockManagerId - + val blockManager = createMockBlockManager() // Make sure remote blocks would return val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 2) val blocks = Map[BlockId, ManagedBuffer]( diff --git a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala index 9ba705c4abd75..459af6748e0e0 100644 --- a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.ui -import org.openqa.selenium.WebDriver +import org.openqa.selenium.{JavascriptExecutor, WebDriver} import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions} import org.apache.spark.tags.ChromeUITest @@ -28,7 +28,7 @@ import org.apache.spark.tags.ChromeUITest @ChromeUITest class ChromeUISeleniumSuite extends RealBrowserUISeleniumSuite("webdriver.chrome.driver") { - override var webDriver: WebDriver = _ + override var webDriver: WebDriver with JavascriptExecutor = _ override def beforeAll(): Unit = { super.beforeAll() diff --git a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala index 2c135850b5a21..3f296acdeb326 100644 --- a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.ui -import org.openqa.selenium.{By, WebDriver} +import org.openqa.selenium.{By, JavascriptExecutor, WebDriver} import org.scalatest.BeforeAndAfterAll import org.scalatest.concurrent.Eventually._ import org.scalatest.matchers.must.Matchers @@ -37,7 +37,7 @@ import org.apache.spark.util.CallSite abstract class RealBrowserUISeleniumSuite(val driverProp: String) extends SparkFunSuite with WebBrowser with Matchers with BeforeAndAfterAll { - implicit var webDriver: WebDriver + implicit var webDriver: WebDriver with JavascriptExecutor private val driverPropPrefix = "spark.test." override def beforeAll(): Unit = { @@ -130,6 +130,58 @@ abstract class RealBrowserUISeleniumSuite(val driverProp: String) } } + test("Search text for paged tables should not be saved") { + withSpark(newSparkContext()) { sc => + sc.parallelize(1 to 10).collect + + eventually(timeout(10.seconds), interval(1.seconds)) { + val taskSearchBox = "$(\"input[aria-controls='active-tasks-table']\")" + goToUi(sc, "/stages/stage/?id=0&attempt=0") + // Wait for ajax loading done. + Thread.sleep(20) + setValueToSearchBox(taskSearchBox, "task1") + val taskSearchText = getTextFromSearchBox(taskSearchBox) + assert(taskSearchText === "task1") + + val executorSearchBox = "$(\"input[aria-controls='active-executors-table']\")" + goToUi(sc, "/executors") + Thread.sleep(20) + setValueToSearchBox(executorSearchBox, "executor1") + val executorSearchText = getTextFromSearchBox(executorSearchBox) + assert(executorSearchText === "executor1") + + goToUi(sc, "/stages/stage/?id=0&attempt=0") + Thread.sleep(20) + val revisitTaskSearchText = getTextFromSearchBox(taskSearchBox) + assert(revisitTaskSearchText === "") + + goToUi(sc, "/executors") + Thread.sleep(20) + val revisitExecutorSearchText = getTextFromSearchBox(executorSearchBox) + assert(revisitExecutorSearchText === "") + } + } + + def setValueToSearchBox(searchBox: String, text: String): Unit = { + webDriver.executeScript(s"$searchBox.val('$text');") + fireDataTable(searchBox) + } + + def getTextFromSearchBox(searchBox: String): String = { + webDriver.executeScript(s"return $searchBox.val();").toString + } + + def fireDataTable(searchBox: String): Unit = { + webDriver.executeScript( + s""" + |var keyEvent = $$.Event('keyup'); + |// 13 means enter key. + |keyEvent.keyCode = keyEvent.which = 13; + |$searchBox.trigger(keyEvent); + """.stripMargin) + } + } + /** * Create a test SparkContext with the SparkUI enabled. * It is safe to `get` the SparkUI directly from the SparkContext returned here. diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala index 6817fccd10305..d7caeaa39ce55 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala @@ -46,7 +46,6 @@ import org.apache.spark.internal.config.Status._ import org.apache.spark.internal.config.UI._ import org.apache.spark.shuffle.FetchFailedException import org.apache.spark.status.api.v1.{JacksonMessageWriter, RDDDataDistribution, StageStatus} -import org.apache.spark.util.CallSite private[spark] class SparkUICssErrorHandler extends DefaultCssErrorHandler { diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala index f3ed233da7122..2ae51f425dcb5 100644 --- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala @@ -170,6 +170,7 @@ class JsonProtocolSuite extends SparkFunSuite { testStorageLevel(StorageLevel.NONE) testStorageLevel(StorageLevel.DISK_ONLY) testStorageLevel(StorageLevel.DISK_ONLY_2) + testStorageLevel(StorageLevel.DISK_ONLY_3) testStorageLevel(StorageLevel.MEMORY_ONLY) testStorageLevel(StorageLevel.MEMORY_ONLY_2) testStorageLevel(StorageLevel.MEMORY_ONLY_SER) diff --git a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala index d4f2053e0b2f4..6183ba9faa6b4 100644 --- a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala @@ -73,15 +73,35 @@ class SizeEstimatorSuite with PrivateMethodTester with ResetSystemProperties { + // Save modified system properties so that we can restore them after tests. + val originalArch = System.getProperty("os.arch") + val originalCompressedOops = System.getProperty(TEST_USE_COMPRESSED_OOPS_KEY) + + def reinitializeSizeEstimator(arch: String, useCompressedOops: String): Unit = { + def set(k: String, v: String): Unit = { + if (v == null) { + System.clearProperty(k) + } else { + System.setProperty(k, v) + } + } + set("os.arch", arch) + set(TEST_USE_COMPRESSED_OOPS_KEY, useCompressedOops) + val initialize = PrivateMethod[Unit](Symbol("initialize")) + SizeEstimator invokePrivate initialize() + } + override def beforeEach(): Unit = { - // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case super.beforeEach() - System.setProperty("os.arch", "amd64") - System.setProperty(TEST_USE_COMPRESSED_OOPS_KEY, "true") + // Set the arch to 64-bit and compressedOops to true so that SizeEstimator + // provides identical results accross all systems in these tests. + reinitializeSizeEstimator("amd64", "true") } override def afterEach(): Unit = { super.afterEach() + // Restore system properties and SizeEstimator to their original states. + reinitializeSizeEstimator(originalArch, originalCompressedOops) } test("simple classes") { @@ -178,11 +198,7 @@ class SizeEstimatorSuite } test("32-bit arch") { - System.setProperty("os.arch", "x86") - - val initialize = PrivateMethod[Unit](Symbol("initialize")) - SizeEstimator invokePrivate initialize() - + reinitializeSizeEstimator("x86", "true") assertResult(40)(SizeEstimator.estimate(DummyString(""))) assertResult(48)(SizeEstimator.estimate(DummyString("a"))) assertResult(48)(SizeEstimator.estimate(DummyString("ab"))) @@ -192,11 +208,7 @@ class SizeEstimatorSuite // NOTE: The String class definition varies across JDK versions (1.6 vs. 1.7) and vendors // (Sun vs IBM). Use a DummyString class to make tests deterministic. test("64-bit arch with no compressed oops") { - System.setProperty("os.arch", "amd64") - System.setProperty(TEST_USE_COMPRESSED_OOPS_KEY, "false") - val initialize = PrivateMethod[Unit](Symbol("initialize")) - SizeEstimator invokePrivate initialize() - + reinitializeSizeEstimator("amd64", "false") assertResult(56)(SizeEstimator.estimate(DummyString(""))) assertResult(64)(SizeEstimator.estimate(DummyString("a"))) assertResult(64)(SizeEstimator.estimate(DummyString("ab"))) @@ -214,14 +226,13 @@ class SizeEstimatorSuite } test("class field blocks rounding on 64-bit VM without useCompressedOops") { + reinitializeSizeEstimator("amd64", "false") assertResult(24)(SizeEstimator.estimate(new DummyClass5)) assertResult(32)(SizeEstimator.estimate(new DummyClass6)) } test("check 64-bit detection for s390x arch") { - System.setProperty("os.arch", "s390x") - val initialize = PrivateMethod[Unit](Symbol("initialize")) - SizeEstimator invokePrivate initialize() + reinitializeSizeEstimator("s390x", "true") // Class should be 32 bytes on s390x if recognised as 64 bit platform assertResult(32)(SizeEstimator.estimate(new DummyClass7)) } diff --git a/dev/.rat-excludes b/dev/.rat-excludes index df1dd51a7c519..0e892a927906a 100644 --- a/dev/.rat-excludes +++ b/dev/.rat-excludes @@ -25,7 +25,7 @@ bootstrap.bundle.min.js bootstrap.bundle.min.js.map bootstrap.min.css bootstrap.min.css.map -jquery-3.4.1.min.js +jquery-3.5.1.min.js d3.min.js dagre-d3.min.js graphlib-dot.min.js @@ -124,3 +124,4 @@ GangliaReporter.java application_1578436911597_0052 config.properties app-20200706201101-0003 +py.typed diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1 index e344a7fc23191..fb4cc22de35f4 100644 --- a/dev/appveyor-install-dependencies.ps1 +++ b/dev/appveyor-install-dependencies.ps1 @@ -19,7 +19,7 @@ $CRAN = "https://cloud.r-project.org" Function InstallR { if ( -not(Test-Path Env:\R_ARCH) ) { - $arch = "i386" + $arch = "x64" } Else { $arch = $env:R_ARCH @@ -68,7 +68,7 @@ Function InstallRtools { $gccPath = $env:GCC_PATH } $env:PATH = $RtoolsDrive + '\Rtools40\bin;' + $RtoolsDrive + '\Rtools40\mingw64\bin;' + $RtoolsDrive + '\Rtools40\' + $gccPath + '\bin;' + $env:PATH - $env:BINPREF=$RtoolsDrive + '/Rtools40/mingw64/bin/' + $env:BINPREF=$RtoolsDrive + '/Rtools40/mingw$(WIN)/bin/' } # create tools directory outside of Spark directory diff --git a/dev/create-release/generate-contributors.py b/dev/create-release/generate-contributors.py index 57775dde9dd67..4e07bd79f8ac3 100755 --- a/dev/create-release/generate-contributors.py +++ b/dev/create-release/generate-contributors.py @@ -22,7 +22,9 @@ import re import sys -from releaseutils import * +from releaseutils import tag_exists, raw_input, get_commits, yesOrNoPrompt, get_date, \ + is_valid_author, capitalize_author, JIRA, find_components, translate_issue_type, \ + translate_component, CORE_COMPONENT, contributors_file_name, nice_join # You must set the following before use! JIRA_API_BASE = os.environ.get("JIRA_API_BASE", "https://issues.apache.org/jira") diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index 31633456a6590..240f4c8dfd371 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -182,8 +182,7 @@ if [[ "$1" == "package" ]]; then tar cvzf spark-$SPARK_VERSION.tgz --exclude spark-$SPARK_VERSION/.git spark-$SPARK_VERSION echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --armour --output spark-$SPARK_VERSION.tgz.asc \ --detach-sig spark-$SPARK_VERSION.tgz - echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --print-md \ - SHA512 spark-$SPARK_VERSION.tgz > spark-$SPARK_VERSION.tgz.sha512 + shasum -a 512 spark-$SPARK_VERSION.tgz > spark-$SPARK_VERSION.tgz.sha512 rm -rf spark-$SPARK_VERSION ZINC_PORT=3035 @@ -275,6 +274,9 @@ if [[ "$1" == "package" ]]; then # In dry run mode, only build the first one. The keys in BINARY_PKGS_ARGS are used as the # list of packages to be built, so it's ok for things to be missing in BINARY_PKGS_EXTRA. + # NOTE: Don't forget to update the valid combinations of distributions at + # 'python/pyspark/install.py' and 'python/docs/source/getting_started/install.rst' + # if you're changing them. declare -A BINARY_PKGS_ARGS BINARY_PKGS_ARGS["hadoop3.2"]="-Phadoop-3.2 $HIVE_PROFILES" if ! is_dry_run; then @@ -282,13 +284,12 @@ if [[ "$1" == "package" ]]; then if [[ $SPARK_VERSION < "3.0." ]]; then BINARY_PKGS_ARGS["hadoop2.6"]="-Phadoop-2.6 $HIVE_PROFILES" else - BINARY_PKGS_ARGS["hadoop2.7-hive1.2"]="-Phadoop-2.7 -Phive-1.2 $HIVE_PROFILES" BINARY_PKGS_ARGS["hadoop2.7"]="-Phadoop-2.7 $HIVE_PROFILES" fi fi declare -A BINARY_PKGS_EXTRA - BINARY_PKGS_EXTRA["hadoop2.7"]="withpip,withr" + BINARY_PKGS_EXTRA["hadoop3.2"]="withpip,withr" if [[ $PUBLISH_SCALA_2_11 = 1 ]]; then key="without-hadoop-scala-2.11" diff --git a/dev/create-release/release-tag.sh b/dev/create-release/release-tag.sh index 39856a9955955..a9a518f9e10d7 100755 --- a/dev/create-release/release-tag.sh +++ b/dev/create-release/release-tag.sh @@ -64,8 +64,12 @@ init_maven_sbt ASF_SPARK_REPO="gitbox.apache.org/repos/asf/spark.git" +function uriencode { jq -nSRr --arg v "$1" '$v|@uri'; } + +declare -r ENCODED_ASF_PASSWORD=$(uriencode "$ASF_PASSWORD") + rm -rf spark -git clone "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_SPARK_REPO" -b $GIT_BRANCH +git clone "https://$ASF_USERNAME:$ENCODED_ASF_PASSWORD@$ASF_SPARK_REPO" -b $GIT_BRANCH cd spark git config user.name "$GIT_NAME" diff --git a/dev/create-release/release-util.sh b/dev/create-release/release-util.sh index af9ed201b3b47..7961eed850891 100755 --- a/dev/create-release/release-util.sh +++ b/dev/create-release/release-util.sh @@ -228,7 +228,7 @@ function init_maven_sbt { if [[ $JAVA_VERSION < "1.8." ]]; then # Needed for maven central when using Java 7. SBT_OPTS="-Dhttps.protocols=TLSv1.1,TLSv1.2" - MVN_EXTRA_OPTS="-Dhttps.protocols=TLSv1.1,TLSv1.2" + MVN_EXTRA_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dhttps.protocols=TLSv1.1,TLSv1.2" MVN="$MVN $MVN_EXTRA_OPTS" fi export MVN MVN_EXTRA_OPTS SBT_OPTS diff --git a/dev/create-release/releaseutils.py b/dev/create-release/releaseutils.py index 9f1dffbd09ac4..cc7ad931198a2 100755 --- a/dev/create-release/releaseutils.py +++ b/dev/create-release/releaseutils.py @@ -23,7 +23,7 @@ from subprocess import Popen, PIPE try: - from jira.client import JIRA + from jira.client import JIRA # noqa: F401 # Old versions have JIRAError in exceptions package, new (0.5+) in utils. try: from jira.exceptions import JIRAError @@ -31,11 +31,11 @@ from jira.utils import JIRAError except ImportError: print("This tool requires the jira-python library") - print("Install using 'sudo pip install jira'") + print("Install using 'sudo pip3 install jira'") sys.exit(-1) try: - from github import Github + from github import Github # noqa: F401 from github import GithubException except ImportError: print("This tool requires the PyGithub library") diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile index a02a6b7bccf27..6b32f10490719 100644 --- a/dev/create-release/spark-rm/Dockerfile +++ b/dev/create-release/spark-rm/Dockerfile @@ -36,7 +36,7 @@ ARG APT_INSTALL="apt-get install --no-install-recommends -y" # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes. # See also https://github.com/sphinx-doc/sphinx/issues/7551. # We should use the latest Sphinx version once this is fixed. -ARG PIP_PKGS="sphinx==3.0.4 mkdocs==1.0.4 numpy==1.18.1 pydata_sphinx_theme==0.3.1" +ARG PIP_PKGS="sphinx==3.0.4 mkdocs==1.0.4 numpy==1.18.1 pydata_sphinx_theme==0.3.1 ipython==7.16.1 nbsphinx==0.7.1 numpydoc==1.1.0" ARG GEM_PKGS="jekyll:4.0.0 jekyll-redirect-from:0.16.0 rouge:3.15.0" # Install extra needed repos and refresh. @@ -66,6 +66,8 @@ RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \ $APT_INSTALL nodejs && \ # Install needed python packages. Use pip for installing packages (for consistency). $APT_INSTALL libpython3-dev python3-pip python3-setuptools && \ + # qpdf is required for CRAN checks to pass. + $APT_INSTALL qpdf jq && \ # Change default python version to python3. update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && \ update-alternatives --install /usr/bin/python python /usr/bin/python3.6 2 && \ @@ -73,6 +75,7 @@ RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \ pip3 install $PIP_PKGS && \ # Install R packages and dependencies used when building. # R depends on pandoc*, libssl (which are installed above). + # Note that PySpark doc generation also needs pandoc due to nbsphinx $APT_INSTALL r-base r-base-dev && \ $APT_INSTALL texlive-latex-base texlive texlive-fonts-extra texinfo qpdf && \ Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')" && \ diff --git a/dev/create-release/translate-contributors.py b/dev/create-release/translate-contributors.py index 554acc8b8a379..8340266527fc6 100755 --- a/dev/create-release/translate-contributors.py +++ b/dev/create-release/translate-contributors.py @@ -31,7 +31,15 @@ import os import sys -from releaseutils import * +from releaseutils import JIRA, JIRAError, get_jira_name, Github, get_github_name, \ + contributors_file_name, is_valid_author, raw_input, capitalize_author, yesOrNoPrompt + +try: + import unidecode +except ImportError: + print("This tool requires the unidecode library to decode obscure github usernames") + print("Install using 'sudo pip install unidecode'") + sys.exit(-1) # You must set the following before use! JIRA_API_BASE = os.environ.get("JIRA_API_BASE", "https://issues.apache.org/jira") @@ -135,7 +143,7 @@ def generate_candidates(author, issues): # Note that the candidate name may already be in unicode (JIRA returns this) for i, (candidate, source) in enumerate(candidates): try: - candidate = unicode(candidate, "UTF-8") + candidate = unicode(candidate, "UTF-8") # noqa: F821 except TypeError: # already in unicode pass diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2 deleted file mode 100644 index 4936d808db85a..0000000000000 --- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 +++ /dev/null @@ -1,211 +0,0 @@ -JLargeArrays/1.5//JLargeArrays-1.5.jar -JTransforms/3.1//JTransforms-3.1.jar -JavaEWAH/0.3.2//JavaEWAH-0.3.2.jar -RoaringBitmap/0.9.0//RoaringBitmap-0.9.0.jar -ST4/4.0.4//ST4-4.0.4.jar -activation/1.1.1//activation-1.1.1.jar -aircompressor/0.10//aircompressor-0.10.jar -algebra_2.12/2.0.0-M2//algebra_2.12-2.0.0-M2.jar -antlr-runtime/3.4//antlr-runtime-3.4.jar -antlr/2.7.7//antlr-2.7.7.jar -antlr4-runtime/4.7.1//antlr4-runtime-4.7.1.jar -aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar -aopalliance/1.0//aopalliance-1.0.jar -apache-log4j-extras/1.2.17//apache-log4j-extras-1.2.17.jar -apacheds-i18n/2.0.0-M15//apacheds-i18n-2.0.0-M15.jar -apacheds-kerberos-codec/2.0.0-M15//apacheds-kerberos-codec-2.0.0-M15.jar -api-asn1-api/1.0.0-M20//api-asn1-api-1.0.0-M20.jar -api-util/1.0.0-M20//api-util-1.0.0-M20.jar -arpack_combined_all/0.1//arpack_combined_all-0.1.jar -arrow-format/0.15.1//arrow-format-0.15.1.jar -arrow-memory/0.15.1//arrow-memory-0.15.1.jar -arrow-vector/0.15.1//arrow-vector-0.15.1.jar -audience-annotations/0.5.0//audience-annotations-0.5.0.jar -automaton/1.11-8//automaton-1.11-8.jar -avro-ipc/1.8.2//avro-ipc-1.8.2.jar -avro-mapred/1.8.2/hadoop2/avro-mapred-1.8.2-hadoop2.jar -avro/1.8.2//avro-1.8.2.jar -bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar -breeze-macros_2.12/1.0//breeze-macros_2.12-1.0.jar -breeze_2.12/1.0//breeze_2.12-1.0.jar -cats-kernel_2.12/2.0.0-M4//cats-kernel_2.12-2.0.0-M4.jar -chill-java/0.9.5//chill-java-0.9.5.jar -chill_2.12/0.9.5//chill_2.12-0.9.5.jar -commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar -commons-cli/1.2//commons-cli-1.2.jar -commons-codec/1.10//commons-codec-1.10.jar -commons-collections/3.2.2//commons-collections-3.2.2.jar -commons-compiler/3.1.2//commons-compiler-3.1.2.jar -commons-compress/1.8.1//commons-compress-1.8.1.jar -commons-configuration/1.6//commons-configuration-1.6.jar -commons-crypto/1.0.0//commons-crypto-1.0.0.jar -commons-dbcp/1.4//commons-dbcp-1.4.jar -commons-digester/1.8//commons-digester-1.8.jar -commons-httpclient/3.1//commons-httpclient-3.1.jar -commons-io/2.4//commons-io-2.4.jar -commons-lang/2.6//commons-lang-2.6.jar -commons-lang3/3.10//commons-lang3-3.10.jar -commons-logging/1.1.3//commons-logging-1.1.3.jar -commons-math3/3.4.1//commons-math3-3.4.1.jar -commons-net/3.1//commons-net-3.1.jar -commons-pool/1.5.4//commons-pool-1.5.4.jar -commons-text/1.6//commons-text-1.6.jar -compress-lzf/1.0.3//compress-lzf-1.0.3.jar -core/1.1.2//core-1.1.2.jar -curator-client/2.7.1//curator-client-2.7.1.jar -curator-framework/2.7.1//curator-framework-2.7.1.jar -curator-recipes/2.7.1//curator-recipes-2.7.1.jar -datanucleus-api-jdo/3.2.6//datanucleus-api-jdo-3.2.6.jar -datanucleus-core/3.2.10//datanucleus-core-3.2.10.jar -datanucleus-rdbms/3.2.9//datanucleus-rdbms-3.2.9.jar -derby/10.12.1.1//derby-10.12.1.1.jar -flatbuffers-java/1.9.0//flatbuffers-java-1.9.0.jar -generex/1.0.2//generex-1.0.2.jar -gson/2.2.4//gson-2.2.4.jar -guava/14.0.1//guava-14.0.1.jar -guice-servlet/3.0//guice-servlet-3.0.jar -guice/3.0//guice-3.0.jar -hadoop-annotations/2.7.4//hadoop-annotations-2.7.4.jar -hadoop-auth/2.7.4//hadoop-auth-2.7.4.jar -hadoop-client/2.7.4//hadoop-client-2.7.4.jar -hadoop-common/2.7.4//hadoop-common-2.7.4.jar -hadoop-hdfs/2.7.4//hadoop-hdfs-2.7.4.jar -hadoop-mapreduce-client-app/2.7.4//hadoop-mapreduce-client-app-2.7.4.jar -hadoop-mapreduce-client-common/2.7.4//hadoop-mapreduce-client-common-2.7.4.jar -hadoop-mapreduce-client-core/2.7.4//hadoop-mapreduce-client-core-2.7.4.jar -hadoop-mapreduce-client-jobclient/2.7.4//hadoop-mapreduce-client-jobclient-2.7.4.jar -hadoop-mapreduce-client-shuffle/2.7.4//hadoop-mapreduce-client-shuffle-2.7.4.jar -hadoop-yarn-api/2.7.4//hadoop-yarn-api-2.7.4.jar -hadoop-yarn-client/2.7.4//hadoop-yarn-client-2.7.4.jar -hadoop-yarn-common/2.7.4//hadoop-yarn-common-2.7.4.jar -hadoop-yarn-server-common/2.7.4//hadoop-yarn-server-common-2.7.4.jar -hadoop-yarn-server-web-proxy/2.7.4//hadoop-yarn-server-web-proxy-2.7.4.jar -hk2-api/2.6.1//hk2-api-2.6.1.jar -hk2-locator/2.6.1//hk2-locator-2.6.1.jar -hk2-utils/2.6.1//hk2-utils-2.6.1.jar -htrace-core/3.1.0-incubating//htrace-core-3.1.0-incubating.jar -httpclient/4.5.6//httpclient-4.5.6.jar -httpcore/4.4.12//httpcore-4.4.12.jar -istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar -ivy/2.4.0//ivy-2.4.0.jar -jackson-annotations/2.10.0//jackson-annotations-2.10.0.jar -jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar -jackson-core/2.10.0//jackson-core-2.10.0.jar -jackson-databind/2.10.0//jackson-databind-2.10.0.jar -jackson-dataformat-yaml/2.10.0//jackson-dataformat-yaml-2.10.0.jar -jackson-datatype-jsr310/2.10.3//jackson-datatype-jsr310-2.10.3.jar -jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar -jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar -jackson-module-jaxb-annotations/2.10.0//jackson-module-jaxb-annotations-2.10.0.jar -jackson-module-paranamer/2.10.0//jackson-module-paranamer-2.10.0.jar -jackson-module-scala_2.12/2.10.0//jackson-module-scala_2.12-2.10.0.jar -jackson-xc/1.9.13//jackson-xc-1.9.13.jar -jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar -jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar -jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar -jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar -jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar -jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar -janino/3.1.2//janino-3.1.2.jar -javassist/3.25.0-GA//javassist-3.25.0-GA.jar -javax.inject/1//javax.inject-1.jar -javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar -javolution/5.5.1//javolution-5.5.1.jar -jaxb-api/2.2.2//jaxb-api-2.2.2.jar -jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar -jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar -jdo-api/3.0.1//jdo-api-3.0.1.jar -jersey-client/2.30//jersey-client-2.30.jar -jersey-common/2.30//jersey-common-2.30.jar -jersey-container-servlet-core/2.30//jersey-container-servlet-core-2.30.jar -jersey-container-servlet/2.30//jersey-container-servlet-2.30.jar -jersey-hk2/2.30//jersey-hk2-2.30.jar -jersey-media-jaxb/2.30//jersey-media-jaxb-2.30.jar -jersey-server/2.30//jersey-server-2.30.jar -jetty-sslengine/6.1.26//jetty-sslengine-6.1.26.jar -jetty-util/6.1.26//jetty-util-6.1.26.jar -jetty/6.1.26//jetty-6.1.26.jar -jline/2.14.6//jline-2.14.6.jar -joda-time/2.10.5//joda-time-2.10.5.jar -jodd-core/3.5.2//jodd-core-3.5.2.jar -jpam/1.1//jpam-1.1.jar -json4s-ast_2.12/3.7.0-M5//json4s-ast_2.12-3.7.0-M5.jar -json4s-core_2.12/3.7.0-M5//json4s-core_2.12-3.7.0-M5.jar -json4s-jackson_2.12/3.7.0-M5//json4s-jackson_2.12-3.7.0-M5.jar -json4s-scalap_2.12/3.7.0-M5//json4s-scalap_2.12-3.7.0-M5.jar -jsp-api/2.1//jsp-api-2.1.jar -jsr305/3.0.0//jsr305-3.0.0.jar -jta/1.1//jta-1.1.jar -jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar -kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar -kubernetes-client/4.9.2//kubernetes-client-4.9.2.jar -kubernetes-model-common/4.9.2//kubernetes-model-common-4.9.2.jar -kubernetes-model/4.9.2//kubernetes-model-4.9.2.jar -leveldbjni-all/1.8//leveldbjni-all-1.8.jar -libfb303/0.9.3//libfb303-0.9.3.jar -libthrift/0.12.0//libthrift-0.12.0.jar -log4j/1.2.17//log4j-1.2.17.jar -logging-interceptor/3.12.6//logging-interceptor-3.12.6.jar -lz4-java/1.7.1//lz4-java-1.7.1.jar -machinist_2.12/0.6.8//machinist_2.12-0.6.8.jar -macro-compat_2.12/1.1.1//macro-compat_2.12-1.1.1.jar -mesos/1.4.0/shaded-protobuf/mesos-1.4.0-shaded-protobuf.jar -metrics-core/4.1.1//metrics-core-4.1.1.jar -metrics-graphite/4.1.1//metrics-graphite-4.1.1.jar -metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar -metrics-json/4.1.1//metrics-json-4.1.1.jar -metrics-jvm/4.1.1//metrics-jvm-4.1.1.jar -minlog/1.3.0//minlog-1.3.0.jar -netty-all/4.1.51.Final//netty-all-4.1.51.Final.jar -objenesis/2.6//objenesis-2.6.jar -okhttp/3.12.6//okhttp-3.12.6.jar -okio/1.14.0//okio-1.14.0.jar -opencsv/2.3//opencsv-2.3.jar -orc-core/1.5.10/nohive/orc-core-1.5.10-nohive.jar -orc-mapreduce/1.5.10/nohive/orc-mapreduce-1.5.10-nohive.jar -orc-shims/1.5.10//orc-shims-1.5.10.jar -oro/2.0.8//oro-2.0.8.jar -osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar -paranamer/2.8//paranamer-2.8.jar -parquet-column/1.10.1//parquet-column-1.10.1.jar -parquet-common/1.10.1//parquet-common-1.10.1.jar -parquet-encoding/1.10.1//parquet-encoding-1.10.1.jar -parquet-format/2.4.0//parquet-format-2.4.0.jar -parquet-hadoop-bundle/1.6.0//parquet-hadoop-bundle-1.6.0.jar -parquet-hadoop/1.10.1//parquet-hadoop-1.10.1.jar -parquet-jackson/1.10.1//parquet-jackson-1.10.1.jar -protobuf-java/2.5.0//protobuf-java-2.5.0.jar -py4j/0.10.9//py4j-0.10.9.jar -pyrolite/4.30//pyrolite-4.30.jar -scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar -scala-compiler/2.12.10//scala-compiler-2.12.10.jar -scala-library/2.12.10//scala-library-2.12.10.jar -scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar -scala-reflect/2.12.10//scala-reflect-2.12.10.jar -scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar -shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar -shims/0.9.0//shims-0.9.0.jar -slf4j-api/1.7.30//slf4j-api-1.7.30.jar -slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar -snakeyaml/1.24//snakeyaml-1.24.jar -snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar -snappy/0.2//snappy-0.2.jar -spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar -spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar -spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar -spire_2.12/0.17.0-M1//spire_2.12-0.17.0-M1.jar -stax-api/1.0-2//stax-api-1.0-2.jar -stax-api/1.0.1//stax-api-1.0.1.jar -stream/2.9.6//stream-2.9.6.jar -stringtemplate/3.2.1//stringtemplate-3.2.1.jar -super-csv/2.2.0//super-csv-2.2.0.jar -threeten-extra/1.5.0//threeten-extra-1.5.0.jar -univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar -xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar -xercesImpl/2.12.0//xercesImpl-2.12.0.jar -xml-apis/1.4.01//xml-apis-1.4.01.jar -xmlenc/0.52//xmlenc-0.52.jar -xz/1.5//xz-1.5.jar -zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar -zookeeper/3.4.14//zookeeper-3.4.14.jar -zstd-jni/1.4.5-4//zstd-jni-1.4.5-4.jar diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 index 82cd20bf77191..b0b215a316df2 100644 --- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3 @@ -15,9 +15,10 @@ apacheds-kerberos-codec/2.0.0-M15//apacheds-kerberos-codec-2.0.0-M15.jar api-asn1-api/1.0.0-M20//api-asn1-api-1.0.0-M20.jar api-util/1.0.0-M20//api-util-1.0.0-M20.jar arpack_combined_all/0.1//arpack_combined_all-0.1.jar -arrow-format/0.15.1//arrow-format-0.15.1.jar -arrow-memory/0.15.1//arrow-memory-0.15.1.jar -arrow-vector/0.15.1//arrow-vector-0.15.1.jar +arrow-format/1.0.1//arrow-format-1.0.1.jar +arrow-memory-core/1.0.1//arrow-memory-core-1.0.1.jar +arrow-memory-netty/1.0.1//arrow-memory-netty-1.0.1.jar +arrow-vector/1.0.1//arrow-vector-1.0.1.jar audience-annotations/0.5.0//audience-annotations-0.5.0.jar automaton/1.11-8//automaton-1.11-8.jar avro-ipc/1.8.2//avro-ipc-1.8.2.jar @@ -33,7 +34,7 @@ commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar commons-cli/1.2//commons-cli-1.2.jar commons-codec/1.10//commons-codec-1.10.jar commons-collections/3.2.2//commons-collections-3.2.2.jar -commons-compiler/3.1.2//commons-compiler-3.1.2.jar +commons-compiler/3.0.16//commons-compiler-3.0.16.jar commons-compress/1.8.1//commons-compress-1.8.1.jar commons-configuration/1.6//commons-configuration-1.6.jar commons-crypto/1.0.0//commons-crypto-1.0.0.jar @@ -87,11 +88,12 @@ hive-jdbc/2.3.7//hive-jdbc-2.3.7.jar hive-llap-common/2.3.7//hive-llap-common-2.3.7.jar hive-metastore/2.3.7//hive-metastore-2.3.7.jar hive-serde/2.3.7//hive-serde-2.3.7.jar +hive-service-rpc/2.3.7//hive-service-rpc-2.3.7.jar hive-shims-0.23/2.3.7//hive-shims-0.23-2.3.7.jar hive-shims-common/2.3.7//hive-shims-common-2.3.7.jar hive-shims-scheduler/2.3.7//hive-shims-scheduler-2.3.7.jar hive-shims/2.3.7//hive-shims-2.3.7.jar -hive-storage-api/2.7.1//hive-storage-api-2.7.1.jar +hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar hive-vector-code-gen/2.3.7//hive-vector-code-gen-2.3.7.jar hk2-api/2.6.1//hk2-api-2.6.1.jar hk2-locator/2.6.1//hk2-locator-2.6.1.jar @@ -119,13 +121,13 @@ jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar -janino/3.1.2//janino-3.1.2.jar +janino/3.0.16//janino-3.0.16.jar javassist/3.25.0-GA//javassist-3.25.0-GA.jar javax.inject/1//javax.inject-1.jar javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar javolution/5.5.1//javolution-5.5.1.jar -jaxb-api/2.2.2//jaxb-api-2.2.2.jar +jaxb-api/2.2.11//jaxb-api-2.2.11.jar jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar jdo-api/3.0.1//jdo-api-3.0.1.jar @@ -153,14 +155,31 @@ jsr305/3.0.0//jsr305-3.0.0.jar jta/1.1//jta-1.1.jar jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar -kubernetes-client/4.9.2//kubernetes-client-4.9.2.jar -kubernetes-model-common/4.9.2//kubernetes-model-common-4.9.2.jar -kubernetes-model/4.9.2//kubernetes-model-4.9.2.jar +kubernetes-client/4.10.3//kubernetes-client-4.10.3.jar +kubernetes-model-admissionregistration/4.10.3//kubernetes-model-admissionregistration-4.10.3.jar +kubernetes-model-apiextensions/4.10.3//kubernetes-model-apiextensions-4.10.3.jar +kubernetes-model-apps/4.10.3//kubernetes-model-apps-4.10.3.jar +kubernetes-model-autoscaling/4.10.3//kubernetes-model-autoscaling-4.10.3.jar +kubernetes-model-batch/4.10.3//kubernetes-model-batch-4.10.3.jar +kubernetes-model-certificates/4.10.3//kubernetes-model-certificates-4.10.3.jar +kubernetes-model-common/4.10.3//kubernetes-model-common-4.10.3.jar +kubernetes-model-coordination/4.10.3//kubernetes-model-coordination-4.10.3.jar +kubernetes-model-core/4.10.3//kubernetes-model-core-4.10.3.jar +kubernetes-model-discovery/4.10.3//kubernetes-model-discovery-4.10.3.jar +kubernetes-model-events/4.10.3//kubernetes-model-events-4.10.3.jar +kubernetes-model-extensions/4.10.3//kubernetes-model-extensions-4.10.3.jar +kubernetes-model-metrics/4.10.3//kubernetes-model-metrics-4.10.3.jar +kubernetes-model-networking/4.10.3//kubernetes-model-networking-4.10.3.jar +kubernetes-model-policy/4.10.3//kubernetes-model-policy-4.10.3.jar +kubernetes-model-rbac/4.10.3//kubernetes-model-rbac-4.10.3.jar +kubernetes-model-scheduling/4.10.3//kubernetes-model-scheduling-4.10.3.jar +kubernetes-model-settings/4.10.3//kubernetes-model-settings-4.10.3.jar +kubernetes-model-storageclass/4.10.3//kubernetes-model-storageclass-4.10.3.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar libfb303/0.9.3//libfb303-0.9.3.jar libthrift/0.12.0//libthrift-0.12.0.jar log4j/1.2.17//log4j-1.2.17.jar -logging-interceptor/3.12.6//logging-interceptor-3.12.6.jar +logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar lz4-java/1.7.1//lz4-java-1.7.1.jar machinist_2.12/0.6.8//machinist_2.12-0.6.8.jar macro-compat_2.12/1.1.1//macro-compat_2.12-1.1.1.jar @@ -173,12 +192,13 @@ metrics-jvm/4.1.1//metrics-jvm-4.1.1.jar minlog/1.3.0//minlog-1.3.0.jar netty-all/4.1.51.Final//netty-all-4.1.51.Final.jar objenesis/2.6//objenesis-2.6.jar -okhttp/3.12.6//okhttp-3.12.6.jar +okhttp/3.12.12//okhttp-3.12.12.jar okio/1.14.0//okio-1.14.0.jar opencsv/2.3//opencsv-2.3.jar -orc-core/1.5.10//orc-core-1.5.10.jar -orc-mapreduce/1.5.10//orc-mapreduce-1.5.10.jar -orc-shims/1.5.10//orc-shims-1.5.10.jar +openshift-model/4.10.3//openshift-model-4.10.3.jar +orc-core/1.5.12//orc-core-1.5.12.jar +orc-mapreduce/1.5.12//orc-mapreduce-1.5.12.jar +orc-shims/1.5.12//orc-shims-1.5.12.jar oro/2.0.8//oro-2.0.8.jar osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar paranamer/2.8//paranamer-2.8.jar @@ -202,18 +222,17 @@ shims/0.9.0//shims-0.9.0.jar slf4j-api/1.7.30//slf4j-api-1.7.30.jar slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar snakeyaml/1.24//snakeyaml-1.24.jar -snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar +snappy-java/1.1.8//snappy-java-1.1.8.jar spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar spire_2.12/0.17.0-M1//spire_2.12-0.17.0-M1.jar -stax-api/1.0-2//stax-api-1.0-2.jar stax-api/1.0.1//stax-api-1.0.1.jar stream/2.9.6//stream-2.9.6.jar super-csv/2.2.0//super-csv-2.2.0.jar threeten-extra/1.5.0//threeten-extra-1.5.0.jar transaction-api/1.1//transaction-api-1.1.jar -univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar +univocity-parsers/2.9.0//univocity-parsers-2.9.0.jar velocity/1.5//velocity-1.5.jar xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar xercesImpl/2.12.0//xercesImpl-2.12.0.jar @@ -222,4 +241,4 @@ xmlenc/0.52//xmlenc-0.52.jar xz/1.5//xz-1.5.jar zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar zookeeper/3.4.14//zookeeper-3.4.14.jar -zstd-jni/1.4.5-4//zstd-jni-1.4.5-4.jar +zstd-jni/1.4.5-6//zstd-jni-1.4.5-6.jar diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index 7257fb8722422..b64c7989a4e02 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -3,18 +3,17 @@ JLargeArrays/1.5//JLargeArrays-1.5.jar JTransforms/3.1//JTransforms-3.1.jar RoaringBitmap/0.9.0//RoaringBitmap-0.9.0.jar ST4/4.0.4//ST4-4.0.4.jar -accessors-smart/1.2//accessors-smart-1.2.jar activation/1.1.1//activation-1.1.1.jar aircompressor/0.10//aircompressor-0.10.jar algebra_2.12/2.0.0-M2//algebra_2.12-2.0.0-M2.jar antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar antlr4-runtime/4.7.1//antlr4-runtime-4.7.1.jar aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar -aopalliance/1.0//aopalliance-1.0.jar arpack_combined_all/0.1//arpack_combined_all-0.1.jar -arrow-format/0.15.1//arrow-format-0.15.1.jar -arrow-memory/0.15.1//arrow-memory-0.15.1.jar -arrow-vector/0.15.1//arrow-vector-0.15.1.jar +arrow-format/1.0.1//arrow-format-1.0.1.jar +arrow-memory-core/1.0.1//arrow-memory-core-1.0.1.jar +arrow-memory-netty/1.0.1//arrow-memory-netty-1.0.1.jar +arrow-vector/1.0.1//arrow-vector-1.0.1.jar audience-annotations/0.5.0//audience-annotations-0.5.0.jar automaton/1.11-8//automaton-1.11-8.jar avro-ipc/1.8.2//avro-ipc-1.8.2.jar @@ -26,15 +25,12 @@ breeze_2.12/1.0//breeze_2.12-1.0.jar cats-kernel_2.12/2.0.0-M4//cats-kernel_2.12-2.0.0-M4.jar chill-java/0.9.5//chill-java-0.9.5.jar chill_2.12/0.9.5//chill_2.12-0.9.5.jar -commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar commons-cli/1.2//commons-cli-1.2.jar commons-codec/1.10//commons-codec-1.10.jar commons-collections/3.2.2//commons-collections-3.2.2.jar -commons-compiler/3.1.2//commons-compiler-3.1.2.jar +commons-compiler/3.0.16//commons-compiler-3.0.16.jar commons-compress/1.8.1//commons-compress-1.8.1.jar -commons-configuration2/2.1.1//commons-configuration2-2.1.1.jar commons-crypto/1.0.0//commons-crypto-1.0.0.jar -commons-daemon/1.0.13//commons-daemon-1.0.13.jar commons-dbcp/1.4//commons-dbcp-1.4.jar commons-httpclient/3.1//commons-httpclient-3.1.jar commons-io/2.5//commons-io-2.5.jar @@ -54,30 +50,13 @@ datanucleus-api-jdo/4.2.4//datanucleus-api-jdo-4.2.4.jar datanucleus-core/4.1.17//datanucleus-core-4.1.17.jar datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar derby/10.12.1.1//derby-10.12.1.1.jar -dnsjava/2.1.7//dnsjava-2.1.7.jar dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar -ehcache/3.3.1//ehcache-3.3.1.jar flatbuffers-java/1.9.0//flatbuffers-java-1.9.0.jar generex/1.0.2//generex-1.0.2.jar -geronimo-jcache_1.0_spec/1.0-alpha-1//geronimo-jcache_1.0_spec-1.0-alpha-1.jar gson/2.2.4//gson-2.2.4.jar guava/14.0.1//guava-14.0.1.jar -guice-servlet/4.0//guice-servlet-4.0.jar -guice/4.0//guice-4.0.jar -hadoop-annotations/3.2.0//hadoop-annotations-3.2.0.jar -hadoop-auth/3.2.0//hadoop-auth-3.2.0.jar -hadoop-client/3.2.0//hadoop-client-3.2.0.jar -hadoop-common/3.2.0//hadoop-common-3.2.0.jar -hadoop-hdfs-client/3.2.0//hadoop-hdfs-client-3.2.0.jar -hadoop-mapreduce-client-common/3.2.0//hadoop-mapreduce-client-common-3.2.0.jar -hadoop-mapreduce-client-core/3.2.0//hadoop-mapreduce-client-core-3.2.0.jar -hadoop-mapreduce-client-jobclient/3.2.0//hadoop-mapreduce-client-jobclient-3.2.0.jar -hadoop-yarn-api/3.2.0//hadoop-yarn-api-3.2.0.jar -hadoop-yarn-client/3.2.0//hadoop-yarn-client-3.2.0.jar -hadoop-yarn-common/3.2.0//hadoop-yarn-common-3.2.0.jar -hadoop-yarn-registry/3.2.0//hadoop-yarn-registry-3.2.0.jar -hadoop-yarn-server-common/3.2.0//hadoop-yarn-server-common-3.2.0.jar -hadoop-yarn-server-web-proxy/3.2.0//hadoop-yarn-server-web-proxy-3.2.0.jar +hadoop-client-api/3.2.0//hadoop-client-api-3.2.0.jar +hadoop-client-runtime/3.2.0//hadoop-client-runtime-3.2.0.jar hive-beeline/2.3.7//hive-beeline-2.3.7.jar hive-cli/2.3.7//hive-cli-2.3.7.jar hive-common/2.3.7//hive-common-2.3.7.jar @@ -86,11 +65,12 @@ hive-jdbc/2.3.7//hive-jdbc-2.3.7.jar hive-llap-common/2.3.7//hive-llap-common-2.3.7.jar hive-metastore/2.3.7//hive-metastore-2.3.7.jar hive-serde/2.3.7//hive-serde-2.3.7.jar +hive-service-rpc/2.3.7//hive-service-rpc-2.3.7.jar hive-shims-0.23/2.3.7//hive-shims-0.23-2.3.7.jar hive-shims-common/2.3.7//hive-shims-common-2.3.7.jar hive-shims-scheduler/2.3.7//hive-shims-scheduler-2.3.7.jar hive-shims/2.3.7//hive-shims-2.3.7.jar -hive-storage-api/2.7.1//hive-storage-api-2.7.1.jar +hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar hive-vector-code-gen/2.3.7//hive-vector-code-gen-2.3.7.jar hk2-api/2.6.1//hk2-api-2.6.1.jar hk2-locator/2.6.1//hk2-locator-2.6.1.jar @@ -106,8 +86,6 @@ jackson-core/2.10.0//jackson-core-2.10.0.jar jackson-databind/2.10.0//jackson-databind-2.10.0.jar jackson-dataformat-yaml/2.10.0//jackson-dataformat-yaml-2.10.0.jar jackson-datatype-jsr310/2.10.3//jackson-datatype-jsr310-2.10.3.jar -jackson-jaxrs-base/2.9.5//jackson-jaxrs-base-2.9.5.jar -jackson-jaxrs-json-provider/2.9.5//jackson-jaxrs-json-provider-2.9.5.jar jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar jackson-module-jaxb-annotations/2.10.0//jackson-module-jaxb-annotations-2.10.0.jar jackson-module-paranamer/2.10.0//jackson-module-paranamer-2.10.0.jar @@ -118,15 +96,13 @@ jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar -janino/3.1.2//janino-3.1.2.jar +janino/3.0.16//janino-3.0.16.jar javassist/3.25.0-GA//javassist-3.25.0-GA.jar -javax.inject/1//javax.inject-1.jar javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar javolution/5.5.1//javolution-5.5.1.jar jaxb-api/2.2.11//jaxb-api-2.2.11.jar jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar -jcip-annotations/1.0-1//jcip-annotations-1.0-1.jar jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar jdo-api/3.0.1//jdo-api-3.0.1.jar jersey-client/2.30//jersey-client-2.30.jar @@ -140,39 +116,40 @@ jline/2.14.6//jline-2.14.6.jar joda-time/2.10.5//joda-time-2.10.5.jar jodd-core/3.5.2//jodd-core-3.5.2.jar jpam/1.1//jpam-1.1.jar -json-smart/2.3//json-smart-2.3.jar json/1.8//json-1.8.jar json4s-ast_2.12/3.7.0-M5//json4s-ast_2.12-3.7.0-M5.jar json4s-core_2.12/3.7.0-M5//json4s-core_2.12-3.7.0-M5.jar json4s-jackson_2.12/3.7.0-M5//json4s-jackson_2.12-3.7.0-M5.jar json4s-scalap_2.12/3.7.0-M5//json4s-scalap_2.12-3.7.0-M5.jar -jsp-api/2.1//jsp-api-2.1.jar jsr305/3.0.0//jsr305-3.0.0.jar jta/1.1//jta-1.1.jar jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar -kerb-admin/1.0.1//kerb-admin-1.0.1.jar -kerb-client/1.0.1//kerb-client-1.0.1.jar -kerb-common/1.0.1//kerb-common-1.0.1.jar -kerb-core/1.0.1//kerb-core-1.0.1.jar -kerb-crypto/1.0.1//kerb-crypto-1.0.1.jar -kerb-identity/1.0.1//kerb-identity-1.0.1.jar -kerb-server/1.0.1//kerb-server-1.0.1.jar -kerb-simplekdc/1.0.1//kerb-simplekdc-1.0.1.jar -kerb-util/1.0.1//kerb-util-1.0.1.jar -kerby-asn1/1.0.1//kerby-asn1-1.0.1.jar -kerby-config/1.0.1//kerby-config-1.0.1.jar -kerby-pkix/1.0.1//kerby-pkix-1.0.1.jar -kerby-util/1.0.1//kerby-util-1.0.1.jar -kerby-xdr/1.0.1//kerby-xdr-1.0.1.jar kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar -kubernetes-client/4.9.2//kubernetes-client-4.9.2.jar -kubernetes-model-common/4.9.2//kubernetes-model-common-4.9.2.jar -kubernetes-model/4.9.2//kubernetes-model-4.9.2.jar +kubernetes-client/4.10.3//kubernetes-client-4.10.3.jar +kubernetes-model-admissionregistration/4.10.3//kubernetes-model-admissionregistration-4.10.3.jar +kubernetes-model-apiextensions/4.10.3//kubernetes-model-apiextensions-4.10.3.jar +kubernetes-model-apps/4.10.3//kubernetes-model-apps-4.10.3.jar +kubernetes-model-autoscaling/4.10.3//kubernetes-model-autoscaling-4.10.3.jar +kubernetes-model-batch/4.10.3//kubernetes-model-batch-4.10.3.jar +kubernetes-model-certificates/4.10.3//kubernetes-model-certificates-4.10.3.jar +kubernetes-model-common/4.10.3//kubernetes-model-common-4.10.3.jar +kubernetes-model-coordination/4.10.3//kubernetes-model-coordination-4.10.3.jar +kubernetes-model-core/4.10.3//kubernetes-model-core-4.10.3.jar +kubernetes-model-discovery/4.10.3//kubernetes-model-discovery-4.10.3.jar +kubernetes-model-events/4.10.3//kubernetes-model-events-4.10.3.jar +kubernetes-model-extensions/4.10.3//kubernetes-model-extensions-4.10.3.jar +kubernetes-model-metrics/4.10.3//kubernetes-model-metrics-4.10.3.jar +kubernetes-model-networking/4.10.3//kubernetes-model-networking-4.10.3.jar +kubernetes-model-policy/4.10.3//kubernetes-model-policy-4.10.3.jar +kubernetes-model-rbac/4.10.3//kubernetes-model-rbac-4.10.3.jar +kubernetes-model-scheduling/4.10.3//kubernetes-model-scheduling-4.10.3.jar +kubernetes-model-settings/4.10.3//kubernetes-model-settings-4.10.3.jar +kubernetes-model-storageclass/4.10.3//kubernetes-model-storageclass-4.10.3.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar libfb303/0.9.3//libfb303-0.9.3.jar libthrift/0.12.0//libthrift-0.12.0.jar log4j/1.2.17//log4j-1.2.17.jar -logging-interceptor/3.12.6//logging-interceptor-3.12.6.jar +logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar lz4-java/1.7.1//lz4-java-1.7.1.jar machinist_2.12/0.6.8//machinist_2.12-0.6.8.jar macro-compat_2.12/1.1.1//macro-compat_2.12-1.1.1.jar @@ -184,15 +161,14 @@ metrics-json/4.1.1//metrics-json-4.1.1.jar metrics-jvm/4.1.1//metrics-jvm-4.1.1.jar minlog/1.3.0//minlog-1.3.0.jar netty-all/4.1.51.Final//netty-all-4.1.51.Final.jar -nimbus-jose-jwt/4.41.1//nimbus-jose-jwt-4.41.1.jar objenesis/2.6//objenesis-2.6.jar -okhttp/2.7.5//okhttp-2.7.5.jar -okhttp/3.12.6//okhttp-3.12.6.jar +okhttp/3.12.12//okhttp-3.12.12.jar okio/1.14.0//okio-1.14.0.jar opencsv/2.3//opencsv-2.3.jar -orc-core/1.5.10//orc-core-1.5.10.jar -orc-mapreduce/1.5.10//orc-mapreduce-1.5.10.jar -orc-shims/1.5.10//orc-shims-1.5.10.jar +openshift-model/4.10.3//openshift-model-4.10.3.jar +orc-core/1.5.12//orc-core-1.5.12.jar +orc-mapreduce/1.5.12//orc-mapreduce-1.5.12.jar +orc-shims/1.5.12//orc-shims-1.5.12.jar oro/2.0.8//oro-2.0.8.jar osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar paranamer/2.8//paranamer-2.8.jar @@ -205,7 +181,6 @@ parquet-jackson/1.10.1//parquet-jackson-1.10.1.jar protobuf-java/2.5.0//protobuf-java-2.5.0.jar py4j/0.10.9//py4j-0.10.9.jar pyrolite/4.30//pyrolite-4.30.jar -re2j/1.1//re2j-1.1.jar scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar scala-compiler/2.12.10//scala-compiler-2.12.10.jar scala-library/2.12.10//scala-library-2.12.10.jar @@ -217,23 +192,20 @@ shims/0.9.0//shims-0.9.0.jar slf4j-api/1.7.30//slf4j-api-1.7.30.jar slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar snakeyaml/1.24//snakeyaml-1.24.jar -snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar +snappy-java/1.1.8//snappy-java-1.1.8.jar spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar spire_2.12/0.17.0-M1//spire_2.12-0.17.0-M1.jar stax-api/1.0.1//stax-api-1.0.1.jar -stax2-api/3.1.4//stax2-api-3.1.4.jar stream/2.9.6//stream-2.9.6.jar super-csv/2.2.0//super-csv-2.2.0.jar threeten-extra/1.5.0//threeten-extra-1.5.0.jar -token-provider/1.0.1//token-provider-1.0.1.jar transaction-api/1.1//transaction-api-1.1.jar -univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar +univocity-parsers/2.9.0//univocity-parsers-2.9.0.jar velocity/1.5//velocity-1.5.jar -woodstox-core/5.0.3//woodstox-core-5.0.3.jar xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar xz/1.5//xz-1.5.jar zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar zookeeper/3.4.14//zookeeper-3.4.14.jar -zstd-jni/1.4.5-4//zstd-jni-1.4.5-4.jar +zstd-jni/1.4.5-6//zstd-jni-1.4.5-6.jar diff --git a/dev/github_jira_sync.py b/dev/github_jira_sync.py index d916e2dbd8223..9bcebaa22ab86 100755 --- a/dev/github_jira_sync.py +++ b/dev/github_jira_sync.py @@ -30,7 +30,7 @@ import jira.client except ImportError: print("This tool requires the jira-python library") - print("Install using 'sudo pip install jira'") + print("Install using 'sudo pip3 install jira'") sys.exit(-1) # User facing configs diff --git a/dev/lint-python b/dev/lint-python index 41da41bfda5dd..2c244e0c0b297 100755 --- a/dev/lint-python +++ b/dev/lint-python @@ -18,7 +18,7 @@ # define test binaries + versions FLAKE8_BUILD="flake8" MINIMUM_FLAKE8="3.5.0" - +MYPY_BUILD="mypy" PYCODESTYLE_BUILD="pycodestyle" MINIMUM_PYCODESTYLE="2.6.0" @@ -122,6 +122,31 @@ function pycodestyle_test { fi } +function mypy_test { + local MYPY_REPORT= + local MYPY_STATUS= + + # TODO(SPARK-32797): Install mypy on the Jenkins CI workers + if ! hash "$MYPY_BUILD" 2> /dev/null; then + echo "The $MYPY_BUILD command was not found. Skipping for now." + return + fi + + echo "starting $MYPY_BUILD test..." + MYPY_REPORT=$( ($MYPY_BUILD --config-file python/mypy.ini python/pyspark) 2>&1) + MYPY_STATUS=$? + + if [ "$MYPY_STATUS" -ne 0 ]; then + echo "mypy checks failed:" + echo "$MYPY_REPORT" + echo "$MYPY_STATUS" + exit "$MYPY_STATUS" + else + echo "mypy checks passed." + echo + fi +} + function flake8_test { local FLAKE8_VERSION= local EXPECTED_FLAKE8= @@ -147,8 +172,7 @@ flake8 checks failed." fi echo "starting $FLAKE8_BUILD test..." - FLAKE8_REPORT=$( ($FLAKE8_BUILD . --count --select=E901,E999,F821,F822,F823 \ - --max-line-length=100 --show-source --statistics) 2>&1) + FLAKE8_REPORT=$( ($FLAKE8_BUILD --append-config dev/tox.ini --count --show-source --statistics .) 2>&1) FLAKE8_STATUS=$? if [ "$FLAKE8_STATUS" -ne 0 ]; then @@ -197,6 +221,30 @@ function sphinx_test { return fi + # TODO(SPARK-32666): Install nbsphinx in Jenkins machines + PYTHON_HAS_NBSPHINX=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("nbsphinx") is not None)') + if [[ "$PYTHON_HAS_NBSPHINX" == "False" ]]; then + echo "$PYTHON_EXECUTABLE does not have nbsphinx installed. Skipping Sphinx build for now." + echo + return + fi + + # TODO(SPARK-32666): Install ipython in Jenkins machines + PYTHON_HAS_IPYTHON=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("IPython") is not None)') + if [[ "$PYTHON_HAS_IPYTHON" == "False" ]]; then + echo "$PYTHON_EXECUTABLE does not have ipython installed. Skipping Sphinx build for now." + echo + return + fi + + # TODO(SPARK-33242): Install numpydoc in Jenkins machines + PYTHON_HAS_NUMPYDOC=$("$PYTHON_EXECUTABLE" -c 'import importlib.util; print(importlib.util.find_spec("numpydoc") is not None)') + if [[ "$PYTHON_HAS_NUMPYDOC" == "False" ]]; then + echo "$PYTHON_EXECUTABLE does not have numpydoc installed. Skipping Sphinx build for now." + echo + return + fi + echo "starting $SPHINX_BUILD tests..." pushd python/docs &> /dev/null make clean &> /dev/null @@ -231,6 +279,7 @@ PYTHON_SOURCE="$(find . -name "*.py")" compile_python_test "$PYTHON_SOURCE" pycodestyle_test "$PYTHON_SOURCE" flake8_test +mypy_test sphinx_test echo diff --git a/dev/merge_spark_pr.py b/dev/merge_spark_pr.py index 2e376fb343bbe..2a40618228c5f 100755 --- a/dev/merge_spark_pr.py +++ b/dev/merge_spark_pr.py @@ -559,7 +559,7 @@ def main(): print("JIRA_USERNAME and JIRA_PASSWORD not set") print("Exiting without trying to close the associated JIRA.") else: - print("Could not find jira-python library. Run 'sudo pip install jira' to install.") + print("Could not find jira-python library. Run 'sudo pip3 install jira' to install.") print("Exiting without trying to close the associated JIRA.") if __name__ == "__main__": diff --git a/dev/pip-sanity-check.py b/dev/pip-sanity-check.py index e9f10233b12b7..469e27b78b40d 100644 --- a/dev/pip-sanity-check.py +++ b/dev/pip-sanity-check.py @@ -16,7 +16,6 @@ # from pyspark.sql import SparkSession -from pyspark.mllib.linalg import * import sys if __name__ == "__main__": diff --git a/dev/requirements.txt b/dev/requirements.txt index a862a6e986791..c1546c8b8d4d3 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -4,3 +4,6 @@ PyGithub==1.26.0 Unidecode==0.04.19 sphinx pydata_sphinx_theme +ipython +nbsphinx +numpydoc diff --git a/dev/run-tests-jenkins.py b/dev/run-tests-jenkins.py index 4ff5b327e3325..610fb1fd27027 100755 --- a/dev/run-tests-jenkins.py +++ b/dev/run-tests-jenkins.py @@ -175,8 +175,6 @@ def main(): if "test-hadoop3.2" in ghprb_pull_title: os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop3.2" # Switch the Hive profile based on the PR title: - if "test-hive1.2" in ghprb_pull_title: - os.environ["AMPLAB_JENKINS_BUILD_HIVE_PROFILE"] = "hive1.2" if "test-hive2.3" in ghprb_pull_title: os.environ["AMPLAB_JENKINS_BUILD_HIVE_PROFILE"] = "hive2.3" diff --git a/dev/run-tests.py b/dev/run-tests.py index 6aae3bdaefaff..5bdbc0ffb850c 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -26,7 +26,6 @@ import subprocess import glob import shutil -from collections import namedtuple from sparktestsupport import SPARK_HOME, USER_HOME, ERROR_CODES from sparktestsupport.shellutils import exit_from_command_with_retcode, run_cmd, rm_r, which @@ -43,7 +42,8 @@ def determine_modules_for_files(filenames): """ Given a list of filenames, return the set of modules that contain those files. If a file is not associated with a more specific submodule, then this method will consider that - file to belong to the 'root' module. GitHub Action and Appveyor files are ignored. + file to belong to the 'root' module. `.github` directory is counted only in GitHub Actions, + and `appveyor.yml` is always ignored because this file is dedicated only to AppVeyor builds. >>> sorted(x.name for x in determine_modules_for_files(["python/pyspark/a.py", "sql/core/foo"])) ['pyspark-core', 'sql'] @@ -56,6 +56,8 @@ def determine_modules_for_files(filenames): for filename in filenames: if filename in ("appveyor.yml",): continue + if ("GITHUB_ACTIONS" not in os.environ) and filename.startswith(".github"): + continue matched_at_least_one_module = False for module in modules.all_modules: if module.contains_file(filename): @@ -326,7 +328,6 @@ def get_hive_profiles(hive_version): """ sbt_maven_hive_profiles = { - "hive1.2": ["-Phive-1.2"], "hive2.3": ["-Phive-2.3"], } @@ -637,7 +638,7 @@ def main(): else: # else we're running locally or Github Actions. build_tool = "sbt" - hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop2.7") + hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop3.2") hive_version = os.environ.get("HIVE_PROFILE", "hive2.3") if "GITHUB_ACTIONS" in os.environ: test_env = "github_actions" @@ -656,7 +657,13 @@ def main(): # If we're running the tests in Github Actions, attempt to detect and test # only the affected modules. if test_env == "github_actions": - if os.environ["GITHUB_BASE_REF"] != "": + if os.environ["GITHUB_INPUT_BRANCH"] != "": + # Dispatched request + # Note that it assumes Github Actions has already merged + # the given `GITHUB_INPUT_BRANCH` branch. + changed_files = identify_changed_files_from_git_commits( + "HEAD", target_branch=os.environ["GITHUB_SHA"]) + elif os.environ["GITHUB_BASE_REF"] != "": # Pull requests changed_files = identify_changed_files_from_git_commits( os.environ["GITHUB_SHA"], target_branch=os.environ["GITHUB_BASE_REF"]) diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index 3c438e309c22d..868e4a5d23ed7 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -386,6 +386,7 @@ def __hash__(self): "pyspark.tests.test_conf", "pyspark.tests.test_context", "pyspark.tests.test_daemon", + "pyspark.tests.test_install_spark", "pyspark.tests.test_join", "pyspark.tests.test_profiler", "pyspark.tests.test_rdd", diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh index 129b073d75254..e9e9227d239e1 100755 --- a/dev/test-dependencies.sh +++ b/dev/test-dependencies.sh @@ -32,7 +32,6 @@ export LC_ALL=C HADOOP_MODULE_PROFILES="-Phive-thriftserver -Pmesos -Pkubernetes -Pyarn -Phive" MVN="build/mvn" HADOOP_HIVE_PROFILES=( - hadoop-2.7-hive-1.2 hadoop-2.7-hive-2.3 hadoop-3.2-hive-2.3 ) @@ -71,12 +70,9 @@ for HADOOP_HIVE_PROFILE in "${HADOOP_HIVE_PROFILES[@]}"; do if [[ $HADOOP_HIVE_PROFILE == **hadoop-3.2-hive-2.3** ]]; then HADOOP_PROFILE=hadoop-3.2 HIVE_PROFILE=hive-2.3 - elif [[ $HADOOP_HIVE_PROFILE == **hadoop-2.7-hive-2.3** ]]; then - HADOOP_PROFILE=hadoop-2.7 - HIVE_PROFILE=hive-2.3 else HADOOP_PROFILE=hadoop-2.7 - HIVE_PROFILE=hive-1.2 + HIVE_PROFILE=hive-2.3 fi echo "Performing Maven install for $HADOOP_HIVE_PROFILE" $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE jar:jar jar:test-jar install:install clean -q diff --git a/dev/tox.ini b/dev/tox.ini index 5bf27d1abde9b..7edf7d597fb58 100644 --- a/dev/tox.ini +++ b/dev/tox.ini @@ -17,3 +17,8 @@ ignore=E226,E241,E305,E402,E722,E731,E741,W503,W504 max-line-length=100 exclude=python/pyspark/cloudpickle/*.py,shared.py,python/docs/source/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/* + +[flake8] +select = E901,E999,F821,F822,F823,F401,F405 +exclude = python/pyspark/cloudpickle/*.py,shared.py*,python/docs/source/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*,python/out,python/pyspark/sql/pandas/functions.pyi,python/pyspark/sql/column.pyi,python/pyspark/worker.pyi,python/pyspark/java_gateway.pyi +max-line-length = 100 diff --git a/docs/README.md b/docs/README.md index e2002a66b0433..af51dca6180a9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -63,7 +63,7 @@ See also https://github.com/sphinx-doc/sphinx/issues/7551. --> ```sh -$ sudo pip install 'sphinx<3.1.0' mkdocs numpy pydata_sphinx_theme +$ sudo pip install 'sphinx<3.1.0' mkdocs numpy pydata_sphinx_theme ipython nbsphinx numpydoc ``` ## Generating the Documentation HTML diff --git a/docs/_config.yml b/docs/_config.yml index 93e1566a8e638..3be9807f81082 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -14,6 +14,8 @@ plugins: include: - _static - _modules + - _images + - _sources # These allow the documentation to be updated with newer releases # of Spark, Scala, and Mesos. diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml index 63f6b4a0a204b..2207bd6a17656 100644 --- a/docs/_data/menu-sql.yaml +++ b/docs/_data/menu-sql.yaml @@ -175,6 +175,8 @@ url: sql-ref-syntax-qry-select-hints.html - text: Inline Table url: sql-ref-syntax-qry-select-inline-table.html + - text: File + url: sql-ref-syntax-qry-select-file.html - text: JOIN url: sql-ref-syntax-qry-select-join.html - text: LIKE Predicate diff --git a/docs/_layouts/global.html b/docs/_layouts/global.html index d05ac6bbe129d..5f6cd7c6b7f20 100755 --- a/docs/_layouts/global.html +++ b/docs/_layouts/global.html @@ -25,7 +25,6 @@ } - @@ -55,75 +54,82 @@ -