Skip to content

Commit 4e2b315

Browse files
ganeshas-dbcloud-fan
authored andcommitted
[SPARK-54501][SQL] Improve error handling for Hive metastore partition filter failures
### What changes were proposed in this pull request? This PR refactors the error handling for Hive metastore partition filter failures by migrating from the legacy error code _LEGACY_ERROR_TEMP_2193 to a properly defined error condition INTERNAL_ERROR_HIVE_METASTORE_PARTITION_FILTER with SQL state 58030. The error message is restructured to include the underlying exception details. ### Why are the changes needed? The previous error message was verbose and lacked important diagnostic information. The legacy error code needed to be migrated to a proper error condition with an appropriate SQL state for better error categorization. ### Does this PR introduce _any_ user-facing change? Yes. Users will see an improved error message that includes the actual exception details and clearer guidance. ### How was this patch tested? Updated existing unit tests in HivePartitionFilteringSuite and ExternalCatalogSuite to verify the new error condition. ### Was this patch authored or co-authored using generative AI tooling? Generated-by: Claude Sonnet 4.5 Closes #53212 from ganeshashree/SPARK-54501. Authored-by: Ganesha S <[email protected]> Signed-off-by: Wenchen Fan <[email protected]>
1 parent 54cde81 commit 4e2b315

File tree

4 files changed

+19
-12
lines changed

4 files changed

+19
-12
lines changed

common/utils/src/main/resources/error/error-conditions.json

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2436,6 +2436,14 @@
24362436
],
24372437
"sqlState" : "XX000"
24382438
},
2439+
"INTERNAL_ERROR_HIVE_METASTORE_PARTITION_FILTER" : {
2440+
"message" : [
2441+
"Failed to get partition metadata by filter from Hive metastore.",
2442+
"To work around this issue, set '<hiveMetastorePartitionPruningFallbackOnException>' to true. Note that this may result in degraded performance as Spark will fetch all partition metadata instead of filtering at the metastore level.",
2443+
"To report this issue, visit: https://issues.apache.org/jira/browse/SPARK"
2444+
],
2445+
"sqlState" : "XX000"
2446+
},
24392447
"INTERNAL_ERROR_MEMORY" : {
24402448
"message" : [
24412449
"<message>"
@@ -9028,11 +9036,6 @@
90289036
"Partition filter cannot have both `\"` and `'` characters."
90299037
]
90309038
},
9031-
"_LEGACY_ERROR_TEMP_2193" : {
9032-
"message" : [
9033-
"Caught Hive MetaException attempting to get partition metadata by filter from Hive. You can set the Spark configuration setting <hiveMetastorePartitionPruningFallbackOnException> to true to work around this problem, however this will result in degraded performance. Please report a bug: https://issues.apache.org/jira/browse/SPARK."
9034-
]
9035-
},
90369039
"_LEGACY_ERROR_TEMP_2194" : {
90379040
"message" : [
90389041
"Unsupported Hive Metastore version <version>. Please set <key> with a valid version."

sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1667,7 +1667,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE
16671667

16681668
def getPartitionMetadataByFilterError(e: Exception): SparkRuntimeException = {
16691669
new SparkRuntimeException(
1670-
errorClass = "_LEGACY_ERROR_TEMP_2193",
1670+
errorClass = "INTERNAL_ERROR_HIVE_METASTORE_PARTITION_FILTER",
16711671
messageParameters = Map(
16721672
"hiveMetastorePartitionPruningFallbackOnException" ->
16731673
SQLConf.HIVE_METASTORE_PARTITION_PRUNING_FALLBACK_ON_EXCEPTION.key),

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -573,7 +573,7 @@ abstract class ExternalCatalogSuite extends SparkFunSuite {
573573
// then be caught and converted to a RuntimeException with a descriptive message.
574574
case ex: RuntimeException if ex.getMessage.contains("MetaException") =>
575575
throw new AnalysisException(
576-
errorClass = "_LEGACY_ERROR_TEMP_2193",
576+
errorClass = "INTERNAL_ERROR_HIVE_METASTORE_PARTITION_FILTER",
577577
messageParameters = Map(
578578
"hiveMetastorePartitionPruningFallbackOnException" ->
579579
SQLConf.HIVE_METASTORE_PARTITION_PRUNING_FALLBACK_ON_EXCEPTION.key))

sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
2626
import org.apache.hadoop.mapred.TextInputFormat
2727
import org.scalatest.BeforeAndAfterAll
2828

29+
import org.apache.spark.SparkRuntimeException
2930
import org.apache.spark.sql.catalyst.TableIdentifier
3031
import org.apache.spark.sql.catalyst.catalog._
3132
import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.DEFAULT_PARTITION_NAME
@@ -142,11 +143,14 @@ class HivePartitionFilteringSuite(version: String)
142143

143144
test(s"getPartitionsByFilter should fail when $fallbackKey=false") {
144145
withSQLConf(fallbackKey -> "false") {
145-
val e = intercept[RuntimeException](
146-
clientWithoutDirectSql.getPartitionsByFilter(
147-
clientWithoutDirectSql.getRawHiveTable("default", "test"),
148-
Seq(attr("ds") === 20170101)))
149-
assert(e.getMessage.contains("Caught Hive MetaException"))
146+
checkError(
147+
exception = intercept[SparkRuntimeException](
148+
clientWithoutDirectSql.getPartitionsByFilter(
149+
clientWithoutDirectSql.getRawHiveTable("default", "test"),
150+
Seq(attr("ds") === 20170101))),
151+
condition = "INTERNAL_ERROR_HIVE_METASTORE_PARTITION_FILTER",
152+
parameters = Map("hiveMetastorePartitionPruningFallbackOnException" ->
153+
SQLConf.HIVE_METASTORE_PARTITION_PRUNING_FALLBACK_ON_EXCEPTION.key))
150154
}
151155
}
152156

0 commit comments

Comments
 (0)