HIVE-28197: Add deserializer to convert JSON plans to RelNodes

Author: soumyakanti3578

iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out

@@ -14,4 +14,4 @@ POSTHOOK: query: explain formatted select * from test
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@test
 POSTHOOK: Output: hdfs://### HDFS PATH ###
-{"CBOPlan":"{\n  \"rels\": [\n    {\n      \"id\": \"0\",\n      \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n      \"table\": [\n        \"default\",\n        \"test\"\n      ],\n      \"table:alias\": \"test\",\n      \"inputs\": [],\n      \"rowCount\": 1.0,\n      \"avgRowSize\": 8.0,\n      \"rowType\": {\n        \"fields\": [\n          {\n            \"type\": \"INTEGER\",\n            \"nullable\": true,\n            \"name\": \"a\"\n          },\n          {\n            \"type\": \"INTEGER\",\n            \"nullable\": true,\n            \"name\": \"b\"\n          },\n          {\n            \"type\": \"INTEGER\",\n            \"nullable\": true,\n            \"name\": \"PARTITION__SPEC__ID\"\n          },\n          {\n            \"type\": \"BIGINT\",\n            \"nullable\": true,\n            \"name\": \"PARTITION__HASH\"\n          },\n          {\n            \"type\": \"VARCHAR\",\n            \"nullable\": true,\n            \"precision\": 2147483647,\n            \"name\": \"FILE__PATH\"\n          },\n          {\n            \"type\": \"BIGINT\",\n            \"nullable\": true,\n            \"name\": \"ROW__POSITION\"\n          },\n          {\n            \"type\": \"VARCHAR\",\n            \"nullable\": true,\n            \"precision\": 2147483647,\n            \"name\": \"PARTITION__PROJECTION\"\n          },\n          {\n            \"type\": \"BIGINT\",\n            \"nullable\": true,\n            \"name\": \"SNAPSHOT__ID\"\n          }\n        ],\n        \"nullable\": false\n      },\n      \"colStats\": [\n        {\n          \"name\": \"a\",\n          \"ndv\": 1,\n          \"minValue\": -2147483648,\n          \"maxValue\": 2147483647\n        },\n        {\n          \"name\": \"b\",\n          \"ndv\": 1,\n          \"minValue\": -2147483648,\n          \"maxValue\": 2147483647\n        }\n      ]\n    },\n    {\n      \"id\": \"1\",\n      \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n      \"fields\": [\n        \"a\",\n        \"b\"\n      ],\n      \"exprs\": [\n        {\n          \"input\": 0,\n          \"name\": \"$0\"\n        },\n        {\n          \"input\": 1,\n          \"name\": \"$1\"\n        }\n      ],\n      \"rowCount\": 1.0\n    }\n  ]\n}","optimizedSQL":"SELECT `a`, `b`\nFROM `default`.`test`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"test","columns:":["a","b"],"database:":"default","table:":"test","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"a (type: int), b (type: int)","columnExprMap:":{"_col0":"a","_col1":"b"},"outputColumnNames:":["_col0","_col1"],"OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}}
+{"CBOPlan":"{\n  \"rels\": [\n    {\n      \"id\": \"0\",\n      \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n      \"table\": [\n        \"default\",\n        \"test\"\n      ],\n      \"table:alias\": \"test\",\n      \"inputs\": [],\n      \"rowType\": {\n        \"fields\": [\n          {\n            \"type\": \"INTEGER\",\n            \"nullable\": true,\n            \"name\": \"a\"\n          },\n          {\n            \"type\": \"INTEGER\",\n            \"nullable\": true,\n            \"name\": \"b\"\n          },\n          {\n            \"type\": \"INTEGER\",\n            \"nullable\": true,\n            \"name\": \"PARTITION__SPEC__ID\"\n          },\n          {\n            \"type\": \"BIGINT\",\n            \"nullable\": true,\n            \"name\": \"PARTITION__HASH\"\n          },\n          {\n            \"type\": \"VARCHAR\",\n            \"nullable\": true,\n            \"precision\": 2147483647,\n            \"name\": \"FILE__PATH\"\n          },\n          {\n            \"type\": \"BIGINT\",\n            \"nullable\": true,\n            \"name\": \"ROW__POSITION\"\n          },\n          {\n            \"type\": \"VARCHAR\",\n            \"nullable\": true,\n            \"precision\": 2147483647,\n            \"name\": \"PARTITION__PROJECTION\"\n          },\n          {\n            \"type\": \"BIGINT\",\n            \"nullable\": true,\n            \"name\": \"SNAPSHOT__ID\"\n          }\n        ],\n        \"nullable\": false\n      },\n      \"virtualColumns\": [\n        \"PARTITION__SPEC__ID\",\n        \"PARTITION__HASH\",\n        \"FILE__PATH\",\n        \"ROW__POSITION\",\n        \"PARTITION__PROJECTION\",\n        \"SNAPSHOT__ID\"\n      ],\n      \"rowCount\": 1.0,\n      \"avgRowSize\": 8.0,\n      \"colStats\": [\n        {\n          \"name\": \"a\",\n          \"ndv\": 1,\n          \"minValue\": -2147483648,\n          \"maxValue\": 2147483647\n        },\n        {\n          \"name\": \"b\",\n          \"ndv\": 1,\n          \"minValue\": -2147483648,\n          \"maxValue\": 2147483647\n        }\n      ]\n    },\n    {\n      \"id\": \"1\",\n      \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n      \"fields\": [\n        \"a\",\n        \"b\"\n      ],\n      \"exprs\": [\n        {\n          \"input\": 0,\n          \"name\": \"$0\"\n        },\n        {\n          \"input\": 1,\n          \"name\": \"$1\"\n        }\n      ],\n      \"rowCount\": 1.0\n    }\n  ]\n}","optimizedSQL":"SELECT `a`, `b`\nFROM `default`.`test`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"test","columns:":["a","b"],"database:":"default","table:":"test","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"a (type: int), b (type: int)","columnExprMap:":{"_col0":"a","_col1":"b"},"outputColumnNames:":["_col0","_col1"],"OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}}

ql/src/java/org/apache/hadoop/hive/ql/metadata/NotNullConstraint.java

@@ -19,12 +19,12 @@
 package org.apache.hadoop.hive.ql.metadata;
 
 import java.io.Serializable;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
-import com.google.common.collect.ImmutableList;
 import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
 
 /**
@@ -56,7 +56,7 @@ public NotNullConstraint(List<SQLNotNullConstraint> nns, String tableName, Strin
         String enable = pk.isEnable_cstr()? "ENABLE": "DISABLE";
         String validate = pk.isValidate_cstr()? "VALIDATE": "NOVALIDATE";
         String rely = pk.isRely_cstr()? "RELY": "NORELY";
-        enableValidateRely.put(pk.getNn_name(), ImmutableList.of(enable, validate, rely));
+        enableValidateRely.put(pk.getNn_name(), Arrays.asList(enable, validate, rely));
         notNullConstraints.put(pk.getNn_name(), pk.getColumn_name());
       }
     }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelDistribution.java

@@ -28,6 +28,7 @@
 import org.apache.calcite.plan.RelTraitDef;
 import org.apache.calcite.rel.RelDistribution;
 import org.apache.calcite.rel.RelDistributionTraitDef;
+import org.apache.calcite.rel.RelDistributions;
 import org.apache.calcite.rel.RelFieldCollation;
 import org.apache.calcite.util.mapping.IntPair;
 import org.apache.calcite.util.mapping.Mappings.TargetMapping;
@@ -38,6 +39,9 @@ public class HiveRelDistribution implements RelDistribution {
 
   private static final Ordering<Iterable<Integer>> ORDERING = Ordering.<Integer>natural().lexicographical();
 
+  public static final HiveRelDistribution ANY =
+      new HiveRelDistribution(RelDistribution.Type.ANY, RelDistributions.ANY.getKeys());
+
   public static HiveRelDistribution from(
           List<RelFieldCollation> fieldCollations, RelDistribution.Type distributionType) {
     List<Integer> newDistributionKeys = new ArrayList<>(fieldCollations.size());

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelEnumTypes.java

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.calcite.avatica.util.TimeUnitRange;
+import org.apache.calcite.rel.core.TableModify;
+import org.apache.calcite.sql.JoinConditionType;
+import org.apache.calcite.sql.JoinType;
+import org.apache.calcite.sql.SqlExplain;
+import org.apache.calcite.sql.SqlExplainFormat;
+import org.apache.calcite.sql.SqlExplainLevel;
+import org.apache.calcite.sql.SqlInsertKeyword;
+import org.apache.calcite.sql.SqlJsonConstructorNullClause;
+import org.apache.calcite.sql.SqlJsonQueryWrapperBehavior;
+import org.apache.calcite.sql.SqlJsonValueEmptyOrErrorBehavior;
+import org.apache.calcite.sql.SqlMatchRecognize;
+import org.apache.calcite.sql.SqlSelectKeyword;
+import org.apache.calcite.sql.fun.SqlTrimFunction;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Copy of org.apache.calcite.rel.externalize.RelEnumTypes as it has
+ * a private constructor - so cannot extend.
+ * <p>
+ * This class is not needed if toEnum method's access is changed to
+ * public
+ */
+public class HiveRelEnumTypes {
+  private HiveRelEnumTypes(){}
+  private static final ImmutableMap<String, Enum<?>> ENUM_BY_NAME;
+
+  static {
+    // Build a mapping from enum constants (e.g. LEADING) to the enum
+    // that contains them (e.g. SqlTrimFunction.Flag). If there two
+    // enum constants have the same name, the builder will throw.
+    final ImmutableMap.Builder<String, Enum<?>> enumByName =
+        ImmutableMap.builder();
+    register(enumByName, JoinConditionType.class);
+    register(enumByName, JoinType.class);
+    register(enumByName, SqlExplain.Depth.class);
+    register(enumByName, SqlExplainFormat.class);
+    register(enumByName, SqlExplainLevel.class);
+    register(enumByName, SqlInsertKeyword.class);
+    register(enumByName, SqlJsonConstructorNullClause.class);
+    register(enumByName, SqlJsonQueryWrapperBehavior.class);
+    register(enumByName, SqlJsonValueEmptyOrErrorBehavior.class);
+    register(enumByName, SqlMatchRecognize.AfterOption.class);
+    register(enumByName, SqlSelectKeyword.class);
+    register(enumByName, SqlTrimFunction.Flag.class);
+    register(enumByName, TimeUnitRange.class);
+    register(enumByName, TableModify.Operation.class);
+    register(enumByName, HiveTableScan.HiveTableScanTrait.class);
+    ENUM_BY_NAME = enumByName.build();
+  }
+
+  private static void register(ImmutableMap.Builder<String, Enum<?>> builder,
+                               Class<? extends Enum> aClass) {
+    for (Enum enumConstant : aClass.getEnumConstants()) {
+      builder.put(enumConstant.name(), enumConstant);
+    }
+  }
+  
+  public static <E extends Enum<E>> E toEnum(String name) {
+    return (E) requireNonNull(ENUM_BY_NAME.get(name));
+  }
+}