DRILL 1221: Update for changes in Parquet-12 for new converted types.…

… Also converted existing binary files in the test-data directory.
arina-ielchiieva · Jul 29, 2014 · c2e32ac · c2e32ac
1 parent 5bae9c8
commit c2e32ac
Show file tree

Hide file tree

Showing 14 changed files with 16 additions and 15 deletions.
diff --git a/exec/java-exec/src/main/codegen/templates/ParquetTypeHelper.java b/exec/java-exec/src/main/codegen/templates/ParquetTypeHelper.java
@@ -110,9 +110,9 @@ public class ParquetTypeHelper {
     </#list>
             originalTypeMap.put(MinorType.VARCHAR, OriginalType.UTF8);
             originalTypeMap.put(MinorType.DATE, OriginalType.DATE);
-            originalTypeMap.put(MinorType.TIME, OriginalType.TIME);
-            originalTypeMap.put(MinorType.TIMESTAMP, OriginalType.TIMESTAMP);
-            originalTypeMap.put(MinorType.TIMESTAMPTZ, OriginalType.TIMESTAMPTZ);
+            originalTypeMap.put(MinorType.TIME, OriginalType.TIME_MILLIS);
+            originalTypeMap.put(MinorType.TIMESTAMP, OriginalType.TIMESTAMP_MILLIS);
+//            originalTypeMap.put(MinorType.TIMESTAMPTZ, OriginalType.TIMESTAMPTZ);
   }
 
   public static PrimitiveTypeName getPrimitiveTypeNameForMinorType(MinorType minorType) {

diff --git a/...n/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java b/...n/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
@@ -59,9 +59,10 @@ public static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName p
             switch(convertedType) {
               case DECIMAL:
                 return Types.withScaleAndPrecision(TypeProtos.MinorType.DECIMAL18, DataMode.OPTIONAL, schemaElement.getScale(), schemaElement.getPrecision());
-              case FINETIME:
-                throw new UnsupportedOperationException();
-              case TIMESTAMP:
+              // TODO - add this back if it is decided to be added upstream, was removed form our pull request July 2014
+//              case TIME_MICROS:
+//                throw new UnsupportedOperationException();
+              case TIMESTAMP_MILLIS:
                 return Types.optional(MinorType.TIMESTAMP);
               default:
                 throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
@@ -75,7 +76,7 @@ public static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName p
                 return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.OPTIONAL, schemaElement.getScale(), schemaElement.getPrecision());
               case DATE:
                 return Types.optional(MinorType.DATE);
-              case TIME:
+              case TIME_MILLIS:
                 return Types.optional(MinorType.TIME);
               default:
                 throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
@@ -123,9 +124,9 @@ public static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName p
             switch(convertedType) {
               case DECIMAL:
                 return Types.withScaleAndPrecision(MinorType.DECIMAL18, DataMode.REQUIRED, schemaElement.getScale(), schemaElement.getPrecision());
-              case FINETIME:
-                throw new UnsupportedOperationException();
-              case TIMESTAMP:
+//              case FINETIME:
+//                throw new UnsupportedOperationException();
+              case TIMESTAMP_MILLIS:
                 return Types.required(MinorType.TIMESTAMP);
               default:
                 throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
@@ -139,7 +140,7 @@ public static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName p
                 return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.REQUIRED, schemaElement.getScale(), schemaElement.getPrecision());
               case DATE:
                 return Types.required(MinorType.DATE);
-              case TIME:
+              case TIME_MILLIS:
                 return Types.required(MinorType.TIME);
               default:
                 throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
@@ -187,9 +188,9 @@ public static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName p
             switch(convertedType) {
               case DECIMAL:
                 return Types.withScaleAndPrecision(MinorType.DECIMAL18, DataMode.REPEATED, schemaElement.getScale(), schemaElement.getPrecision());
-              case FINETIME:
-                throw new UnsupportedOperationException();
-              case TIMESTAMP:
+//              case FINETIME:
+//                throw new UnsupportedOperationException();
+              case TIMESTAMP_MILLIS:
                 return Types.repeated(MinorType.TIMESTAMP);
               default:
                 throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
@@ -203,7 +204,7 @@ public static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName p
                 return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.REPEATED, schemaElement.getScale(), schemaElement.getPrecision());
               case DATE:
                 return Types.repeated(MinorType.DATE);
-              case TIME:
+              case TIME_MILLIS:
                 return Types.repeated(MinorType.TIME);
               default:
                 throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));

diff --git a/sample-data/nation.parquet b/sample-data/nation.parquet
diff --git a/sample-data/nationsMF/nation.parquet b/sample-data/nationsMF/nation.parquet
diff --git a/sample-data/nationsMF/nation2.parquet b/sample-data/nationsMF/nation2.parquet
diff --git a/sample-data/nationsMF/nationsMF.parquet b/sample-data/nationsMF/nationsMF.parquet
diff --git a/sample-data/nationsSF/nation.parquet b/sample-data/nationsSF/nation.parquet
diff --git a/sample-data/nationsSF/nationsSF.parquet b/sample-data/nationsSF/nationsSF.parquet
diff --git a/sample-data/region.parquet b/sample-data/region.parquet
diff --git a/sample-data/regionsMF/region.parquet b/sample-data/regionsMF/region.parquet
diff --git a/sample-data/regionsMF/region2.parquet b/sample-data/regionsMF/region2.parquet
diff --git a/sample-data/regionsMF/regionsMF_Typed.parquet b/sample-data/regionsMF/regionsMF_Typed.parquet
diff --git a/sample-data/regionsSF/region.parquet b/sample-data/regionsSF/region.parquet
diff --git a/sample-data/regionsSF/regionsSF.parquet b/sample-data/regionsSF/regionsSF.parquet