[DRAFT] Update Calcite to 1.38

common/src/main/java/org/apache/drill/common/types/Types.java

@@ -17,22 +17,23 @@
  */
 package org.apache.drill.common.types;
 
-import static org.apache.drill.common.types.TypeProtos.DataMode.REPEATED;
+import com.google.protobuf.TextFormat;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-
-import com.google.protobuf.TextFormat;
+import static org.apache.drill.common.types.TypeProtos.DataMode.REPEATED;
 
 public class Types {
-  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Types.class);
+  private static final Logger logger = LoggerFactory.getLogger(Types.class);
 
   public static final int MAX_VARCHAR_LENGTH = 65535;
   public static final int DEFAULT_TIMESTAMP_PRECISION = 3;
@@ -936,11 +937,10 @@ public static int maxPrecision(MinorType type) {
       return 28;
     case DECIMAL38DENSE:
     case DECIMAL38SPARSE:
+    case VARDECIMAL:
       return 38;
     case DECIMAL9:
       return 9;
-    case VARDECIMAL:
-      return 38;
     default:
       return 0;
     }

exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java

@@ -17,16 +17,11 @@
  */
 package org.apache.drill.exec.expr.fn;
 
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Field;
-import java.util.Arrays;
-import java.util.List;
-
-import javax.inject.Inject;
-
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
 import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.scanner.persistence.FieldDescriptor;
 import org.apache.drill.common.scanner.persistence.AnnotatedClassDescriptor;
+import org.apache.drill.common.scanner.persistence.FieldDescriptor;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.exec.expr.DrillFunc;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -38,12 +33,15 @@
 import org.apache.drill.exec.ops.UdfUtilities;
 import org.apache.drill.exec.vector.complex.reader.FieldReader;
 import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
-
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.inject.Inject;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.List;
+
 /**
  * Converts FunctionCalls to Java Expressions.
  */
@@ -100,11 +98,11 @@ public DrillFuncHolder getHolder(AnnotatedClassDescriptor func, ClassLoader clas
           fieldClass = fieldClass.getComponentType();
           varArgsCount++;
         } else if (varArgsCount > 0 && param != null) {
-          return failure("Vararg should be the last argument in the function.", func, field);
+          return failure("Vararg should be the last argument in the function {}.  Field: {}", func, field);
         }
 
         if (varArgsCount > 1) {
-          return failure("Function should contain single vararg argument", func, field);
+          return failure("Function {} should contain single vararg argument.  Only contains {}", func, field);
         }
 
         // Special processing for @Param FieldReader

exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java

@@ -17,19 +17,32 @@
  */
 package org.apache.drill.exec.planner.logical;
 
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.GregorianCalendar;
-import java.util.LinkedList;
-import java.util.List;
-
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
 import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexCorrelVariable;
+import org.apache.calcite.rex.RexDynamicParam;
+import org.apache.calcite.rex.RexFieldAccess;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexLocalRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexOver;
+import org.apache.calcite.rex.RexRangeRef;
 import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.rex.RexVisitorImpl;
 import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlSyntax;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.sql.type.BasicSqlType;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.util.DateString;
+import org.apache.calcite.util.NlsString;
 import org.apache.calcite.util.TimeString;
 import org.apache.calcite.util.TimestampString;
 import org.apache.commons.lang3.StringUtils;
@@ -53,30 +66,16 @@
 import org.apache.drill.exec.alias.AliasRegistry;
 import org.apache.drill.exec.alias.AliasRegistryProvider;
 import org.apache.drill.exec.planner.StarColumnHelper;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.type.RelDataTypeField;
-import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexCorrelVariable;
-import org.apache.calcite.rex.RexDynamicParam;
-import org.apache.calcite.rex.RexFieldAccess;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexLiteral;
-import org.apache.calcite.rex.RexLocalRef;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexOver;
-import org.apache.calcite.rex.RexRangeRef;
-import org.apache.calcite.rex.RexVisitorImpl;
-import org.apache.calcite.sql.SqlSyntax;
-import org.apache.calcite.sql.fun.SqlStdOperatorTable;
-import org.apache.calcite.util.NlsString;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.drill.exec.planner.physical.PlannerSettings;
 import org.apache.drill.exec.work.ExecErrorConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.GregorianCalendar;
+import java.util.LinkedList;
+import java.util.List;
 
 import static org.apache.drill.exec.planner.physical.PlannerSettings.ENABLE_DECIMAL_DATA_TYPE;
 
@@ -183,7 +182,7 @@ public LogicalExpression visitInputRef(RexInputRef inputRef) {
 
     @Override
     public LogicalExpression visitCall(RexCall call) {
-//      logger.debug("RexCall {}, {}", call);
+      logger.debug("RexCall {}, {}", call);
       final SqlSyntax syntax = call.getOperator().getSyntax();
       switch (syntax) {
       case BINARY:

exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillReduceAggregatesRule.java

@@ -419,6 +419,7 @@ private static AggregateCall getAggCall(AggregateCall oldCall,
         oldCall.isDistinct(),
         oldCall.isApproximate(),
         oldCall.ignoreNulls(),
+        oldCall.rexList,
         oldCall.getArgList(),
         oldCall.filterArg,
         oldCall.distinctKeys,
@@ -541,6 +542,7 @@ private RexNode reduceStddev(
             oldCall.isDistinct(),
             oldCall.isApproximate(),
             oldCall.ignoreNulls(),
+            oldCall.rexList,
             ImmutableIntList.of(argSquaredOrdinal),
             oldCall.filterArg,
             oldCall.distinctKeys,
@@ -562,6 +564,7 @@ private RexNode reduceStddev(
             oldCall.isDistinct(),
             oldCall.isApproximate(),
             oldCall.ignoreNulls(),
+            oldCall.rexList,
             ImmutableIntList.of(argOrdinal),
             oldCall.filterArg,
             oldCall.distinctKeys,
@@ -739,6 +742,7 @@ public void onMatch(RelOptRuleCall call) {
                   oldAggregateCall.isDistinct(),
                   oldAggregateCall.isApproximate(),
                   oldAggregateCall.ignoreNulls(),
+                  oldAggregateCall.rexList,
                   oldAggregateCall.getArgList(),
                   oldAggregateCall.filterArg,
                   oldAggregateCall.distinctKeys,

exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/AggPrelBase.java

@@ -17,6 +17,7 @@
  */
 package org.apache.drill.exec.planner.physical;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.drill.common.expression.IfExpression;
 import org.apache.drill.common.expression.NullExpression;
 import com.google.common.collect.Lists;
@@ -179,6 +180,7 @@ protected void createKeysAndExprs() {
                   aggCall.e.isDistinct(),
                   aggCall.e.isApproximate(),
                   false,
+                  ImmutableList.of(),
                   Collections.singletonList(aggExprOrdinal),
                   aggCall.e.filterArg,
                   null,
@@ -194,6 +196,7 @@ protected void createKeysAndExprs() {
                   aggCall.e.isDistinct(),
                   aggCall.e.isApproximate(),
                   false,
+                  ImmutableList.of(),
                   Collections.singletonList(aggExprOrdinal),
                   aggCall.e.filterArg,
                   null,
@@ -270,6 +273,7 @@ public Prel prepareForLateralUnnestPipeline(List<RelNode> children) {
           aggCall.isDistinct(),
           aggCall.isApproximate(),
           false,
+          ImmutableList.of(),
           arglist,
           aggCall.filterArg,
           null,

exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/TypeInferenceUtils.java

@@ -17,12 +17,9 @@
  */
 package org.apache.drill.exec.planner.sql;
 
-import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.calcite.avatica.util.TimeUnit;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
@@ -35,17 +32,18 @@
 import org.apache.calcite.sql.SqlNode;
 import org.apache.calcite.sql.SqlNumericLiteral;
 import org.apache.calcite.sql.SqlOperatorBinding;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.sql.parser.SqlParserPos;
 import org.apache.calcite.sql.type.SqlReturnTypeInference;
 import org.apache.calcite.sql.type.SqlTypeFamily;
 import org.apache.calcite.sql.type.SqlTypeName;
-
+import org.apache.calcite.util.NlsString;
+import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.common.expression.ExpressionPosition;
 import org.apache.drill.common.expression.FunctionCall;
 import org.apache.drill.common.expression.FunctionCallFactory;
 import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.expression.MajorTypeInLogicalExpression;
-import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -54,6 +52,8 @@
 import org.apache.drill.exec.resolver.FunctionResolver;
 import org.apache.drill.exec.resolver.FunctionResolverFactory;
 import org.apache.drill.exec.resolver.TypeCastRules;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.List;
 import java.util.Optional;
@@ -744,7 +744,7 @@ public RelDataType inferReturnType(SqlOperatorBinding opBinding) {
       }
 
       final String part = ((SqlCharStringLiteral) ((SqlCallBinding) opBinding).operand(0))
-          .getNlsString()
+          .getValueAs(NlsString.class)
           .getValue()
           .toUpperCase();
 
@@ -1023,14 +1023,19 @@ public static RelDataType createCalciteTypeWithNullability(RelDataTypeFactory ty
   public static RelDataType convertToCalciteType(RelDataTypeFactory typeFactory,
                                                  TypeProtos.MajorType drillType, boolean isNullable) {
     SqlTypeName sqlTypeName = getCalciteTypeFromDrillType(drillType.getMinorType());
+    int precision;
+    int scale;
+
     switch (sqlTypeName) {
       case DECIMAL:
+        precision = drillType.getPrecision();
+        scale = drillType.getScale();
         return typeFactory.createTypeWithNullability(
-          typeFactory.createSqlType(sqlTypeName, drillType.getPrecision(),
-              drillType.getScale()), isNullable);
+          typeFactory.createSqlType(sqlTypeName, precision,
+              scale), isNullable);
       case TIME:
       case TIMESTAMP:
-        int precision = drillType.hasPrecision()
+        precision = drillType.hasPrecision()
           ? drillType.getPrecision()
           : typeFactory.getTypeSystem().getDefaultPrecision(sqlTypeName);
         return typeFactory.createTypeWithNullability(

exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/DrillRexBuilder.java

@@ -17,8 +17,6 @@
  */
 package org.apache.drill.exec.planner.sql.conversion;
 
-import java.math.BigDecimal;
-
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rex.RexBuilder;
@@ -30,6 +28,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.math.BigDecimal;
+
 class DrillRexBuilder extends RexBuilder {
 
   private static final Logger logger = LoggerFactory.getLogger(DrillRexBuilder.class);
@@ -39,7 +39,7 @@ class DrillRexBuilder extends RexBuilder {
   }
 
   /**
-   * Since Drill has different mechanism and rules for implicit casting,
+   * Since Drill has a different mechanism and rules for implicit casting,
    * ensureType() is overridden to avoid conflicting cast functions being added to the expressions.
    */
   @Override
@@ -59,19 +59,11 @@ public RexNode ensureType(
    * {@link RexLiteral} if {@code matchNullability} is false.
    *
    * @param type             Type to cast to
-   * @param exp              Expression being cast
-   * @param matchNullability Whether to ensure the result has the same
-   *                         nullability as {@code type}
+   * @param exp              Expression being castnullability as {@code type}
    * @return Call to CAST operator
    */
   @Override
-  public RexNode makeCast(RelDataType type, RexNode exp, boolean matchNullability) {
-    if (matchNullability) {
-      return makeAbstractCast(type, exp);
-    }
-    // for the case when BigDecimal literal has a scale or precision
-    // that differs from the value from specified RelDataType, cast cannot be removed
-    // TODO: remove this code when CALCITE-1468 is fixed
+  public RexNode makeCast(RelDataType type, RexNode exp) {
     if (type.getSqlTypeName() == SqlTypeName.DECIMAL && exp instanceof RexLiteral) {
       int precision = type.getPrecision();
       int scale = type.getScale();
@@ -81,11 +73,11 @@ public RexNode makeCast(RelDataType type, RexNode exp, boolean matchNullability)
         BigDecimal bigDecimal = (BigDecimal) value;
         DecimalUtility.checkValueOverflow(bigDecimal, precision, scale);
         if (bigDecimal.precision() != precision || bigDecimal.scale() != scale) {
-          return makeAbstractCast(type, exp);
+          return makeAbstractCast(type, exp, false);
         }
       }
     }
-    return super.makeCast(type, exp, false);
+    return super.makeCast(type, exp);
   }
 
   private void validatePrecisionAndScale(int precision, int scale) {

exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/conversion/SqlConverter.java

@@ -120,6 +120,7 @@ public SqlConverter(QueryContext context) {
       .withConformance(DRILL_CONFORMANCE)
       .withUnquotedCasing(Casing.UNCHANGED)
       .withQuotedCasing(Casing.UNCHANGED);
+    // TODO Should we add more of these optimizations?
     this.sqlToRelConverterConfig = SqlToRelConverter.config()
         .withInSubQueryThreshold((int) settings.getInSubqueryThreshold())
         .withRemoveSortInSubQuery(false)