[SPARK-51034][SQL] Reformat Describe As JSON statistics dict for parse-ability

asl3 · cloud-fan · commit 9d6fb581726f · 2025-02-04T12:38:58.000+08:00
### What changes were proposed in this pull request? Reformat `Describe As JSON` statistics table metadata into a dict rather than string for improved parse-ability ### Why are the changes needed? Existing `Describe` formats table statistics into a string `xxx bytes, xxx rows` which is not conducive to parsing. This PR stores the table statistics metadata in raw format and delegates the formatting to the caller of toJsonLinkedHashmap, for improved parsing of the JSON statistics. ### Does this PR introduce _any_ user-facing change? Yes, it affects the output of `Describe As JSON`. ### How was this patch tested? Added to test suite `DescribeTableSuite` ### Was this patch authored or co-authored using generative AI tooling? No Closes #49728 from asl3/asl3/describetable-statistics. Authored-by: Amanda Liu <amanda.liu@databricks.com> Signed-off-by: Wenchen Fan <wenchen@databricks.com>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -28,7 +28,7 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include
 import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
 import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule}
 import org.apache.commons.lang3.StringUtils
-import org.json4s.JsonAST.{JArray, JBool, JDouble, JInt, JLong, JNull, JObject, JString, JValue}
+import org.json4s.JsonAST.{JArray, JBool, JDecimal, JDouble, JInt, JLong, JNull, JObject, JString, JValue}
 import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.SparkException
@@ -61,36 +61,60 @@ trait MetadataMapSupport {
     jsonToString(toJsonLinkedHashMap)
   }
 
+  /**
+   * Some fields from JsonLinkedHashMap are reformatted for human readability in `describe table`.
+   * If a field does not require special re-formatting, it is simply handled by `jsonToString`.
+   */
+  private def jsonToStringReformat(key: String, jValue: JValue): Option[(String, String)] = {
+    val reformattedValue: Option[String] = key match {
+      case "Statistics" =>
+        jValue match {
+          case JObject(fields) =>
+            Some(fields.flatMap {
+              case ("size_in_bytes", JDecimal(bytes)) => Some(s"$bytes bytes")
+              case ("num_rows", JDecimal(rows)) => Some(s"$rows rows")
+              case _ => None
+            }.mkString(", "))
+          case _ => Some(jValue.values.toString)
+        }
+      case "Created Time" | "Last Access" =>
+        jValue match {
+          case JLong(value) => Some(new Date(value).toString)
+          case _ => Some(jValue.values.toString)
+        }
+      case _ => None
+    }
+    reformattedValue.map(value => key -> value)
+  }
+
   protected def jsonToString(
       jsonMap: mutable.LinkedHashMap[String, JValue]): mutable.LinkedHashMap[String, String] = {
     val map = new mutable.LinkedHashMap[String, String]()
-    val timestampKeys = Set("Created Time", "Last Access")
     jsonMap.foreach { case (key, jValue) =>
-      val stringValue = jValue match {
-        case JString(value) => value
-        case JArray(values) =>
-          values.map(_.values)
-            .map {
-              case str: String => quoteIdentifier(str)
-              case other => other.toString
-            }
-            .mkString("[", ", ", "]")
-        case JObject(fields) =>
-          fields.map { case (k, v) =>
-            s"$k=${v.values.toString}"
-          }
-            .mkString("[", ", ", "]")
-        case JInt(value) => value.toString
-        case JDouble(value) => value.toString
-        case JLong(value) =>
-          if (timestampKeys.contains(key)) {
-            new Date(value).toString
-          } else {
-            value.toString
+      jsonToStringReformat(key, jValue) match {
+        case Some((formattedKey, formattedValue)) =>
+          map.put(formattedKey, formattedValue)
+        case None =>
+          val stringValue = jValue match {
+            case JString(value) => value
+            case JArray(values) =>
+              values.map(_.values)
+                .map {
+                  case str: String => quoteIdentifier(str)
+                  case other => other.toString
+                }
+                .mkString("[", ", ", "]")
+            case JObject(fields) =>
+              fields.map { case (k, v) =>
+                s"$k=${v.values.toString}"
+              }.mkString("[", ", ", "]")
+            case JInt(value) => value.toString
+            case JDouble(value) => value.toString
+            case JLong(value) => value.toString
+            case _ => jValue.values.toString
           }
-        case _ => jValue.values.toString
+          map.put(key, stringValue)
       }
-      map.put(key, stringValue)
     }
     map
   }
@@ -642,7 +666,9 @@ case class CatalogTable(
       map += "View Query Output Columns" -> viewQueryOutputColumns
     }
     if (tableProperties != JNull) map += "Table Properties" -> tableProperties
-    if (stats.isDefined) map += "Statistics" -> JString(stats.get.simpleString)
+    stats.foreach { s =>
+      map += "Statistics" -> JObject(s.jsonString.toList)
+    }
     map ++= storage.toJsonLinkedHashMap.map { case (k, v) => k -> v }
     if (tracksPartitionsInCatalog) map += "Partition Provider" -> JString("Catalog")
     if (partitionColumns != JNull) map += "Partition Columns" -> partitionColumns
@@ -811,6 +837,14 @@ case class CatalogStatistics(
     val rowCountString = if (rowCount.isDefined) s", ${rowCount.get} rows" else ""
     s"$sizeInBytes bytes$rowCountString"
   }
+
+  def jsonString: Map[String, JValue] = {
+    val rowCountInt: BigInt = rowCount.getOrElse(0L)
+    Map(
+      "size_in_bytes" -> JDecimal(BigDecimal(sizeInBytes)),
+      "num_rows" -> JDecimal(BigDecimal(rowCountInt))
+    )
+  }
 }
 
 /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala
@@ -298,7 +298,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
     }
   }
 
-  test("DESCRIBE AS JSON partition spec") {
+  test("DESCRIBE AS JSON partition spec and statistics") {
     withNamespaceAndTable("ns", "table") { t =>
       val tableCreationStr =
         s"""
@@ -314,6 +314,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
            |""".stripMargin
       spark.sql(tableCreationStr)
       spark.sql(s"ALTER TABLE $t ADD PARTITION (region='USA', category='tech')")
+      spark.sql(s"ANALYZE TABLE $t COMPUTE STATISTICS FOR ALL COLUMNS")
 
       val descriptionDf =
         spark.sql(s"DESCRIBE FORMATTED $t PARTITION (region='USA', category='tech') AS JSON")
@@ -349,7 +350,11 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
         },
         partition_provider = Some("Catalog"),
         partition_columns = Some(List("region", "category")),
-        partition_values = Some(Map("region" -> "USA", "category" -> "tech"))
+        partition_values = Some(Map("region" -> "USA", "category" -> "tech")),
+        statistics = Some(Map(
+          "size_in_bytes" -> 0,
+          "num_rows" -> 0
+        ))
       )
 
       assert(parsedOutput.location.isDefined)
@@ -751,6 +756,7 @@ case class DescribeTableJson(
     partition_provider: Option[String] = None,
     partition_columns: Option[List[String]] = Some(Nil),
     partition_values: Option[Map[String, String]] = None,
+    statistics: Option[Map[String, Any]] = None,
     view_text: Option[String] = None,
     view_original_text: Option[String] = None,
     view_schema_mode: Option[String] = None,
