From 599ea39dc4fd79cf2bc6a63f70aef6a5f5179dba Mon Sep 17 00:00:00 2001 From: James Turton Date: Sat, 31 Aug 2024 07:45:19 +0200 Subject: [PATCH 1/3] Remove remnants of MapR-DB support. The 1.21 branch is the last to include MapR-related code. --- contrib/format-maprdb/README.md | 8 - contrib/format-maprdb/pom.xml | 382 ---- .../index/MapRDBFunctionalIndexInfo.java | 168 -- .../planner/index/MapRDBIndexDescriptor.java | 220 -- .../planner/index/MapRDBIndexDiscover.java | 362 --- .../exec/planner/index/MapRDBStatistics.java | 997 -------- .../index/MapRDBStatisticsPayload.java | 56 - .../exec/store/mapr/PluginConstants.java | 95 - .../exec/store/mapr/PluginErrorHandler.java | 49 - .../exec/store/mapr/TableFormatMatcher.java | 77 - .../exec/store/mapr/TableFormatPlugin.java | 131 -- .../store/mapr/TableFormatPluginConfig.java | 35 - .../drill/exec/store/mapr/db/MapRDBCost.java | 104 - .../store/mapr/db/MapRDBFormatMatcher.java | 118 - .../store/mapr/db/MapRDBFormatPlugin.java | 190 -- .../mapr/db/MapRDBFormatPluginConfig.java | 136 -- .../exec/store/mapr/db/MapRDBGroupScan.java | 372 --- .../mapr/db/MapRDBPushFilterIntoScan.java | 206 -- .../mapr/db/MapRDBPushLimitIntoScan.java | 203 -- .../mapr/db/MapRDBPushProjectIntoScan.java | 140 -- .../db/MapRDBRestrictedScanBatchCreator.java | 50 - .../store/mapr/db/MapRDBScanBatchCreator.java | 65 - .../exec/store/mapr/db/MapRDBSubScan.java | 156 -- .../exec/store/mapr/db/MapRDBSubScanSpec.java | 162 -- .../exec/store/mapr/db/MapRDBTableCache.java | 235 -- .../exec/store/mapr/db/MapRDBTableStats.java | 46 - .../mapr/db/RestrictedMapRDBSubScan.java | 82 - .../mapr/db/RestrictedMapRDBSubScanSpec.java | 192 -- .../store/mapr/db/TabletFragmentInfo.java | 115 - .../mapr/db/binary/BinaryTableGroupScan.java | 249 -- .../mapr/db/binary/MapRDBBinaryTable.java | 34 - .../mapr/db/binary/MapRDBFilterBuilder.java | 354 --- .../MaprDBCompareFunctionsProcessor.java | 64 - .../mapr/db/json/AllTextValueWriter.java | 80 - .../db/json/CompareFunctionsProcessor.java | 279 --- .../db/json/DocumentReaderVectorWriter.java | 42 - .../store/mapr/db/json/FieldPathHelper.java | 75 - .../db/json/FieldTransferVectorWriter.java | 49 - .../mapr/db/json/IdOnlyVectorWriter.java | 65 - .../mapr/db/json/JsonConditionBuilder.java | 269 --- .../exec/store/mapr/db/json/JsonScanSpec.java | 152 -- .../store/mapr/db/json/JsonSubScanSpec.java | 75 - .../mapr/db/json/JsonTableGroupScan.java | 773 ------- .../json/JsonTableRangePartitionFunction.java | 244 -- .../mapr/db/json/MaprDBJsonRecordReader.java | 557 ----- .../db/json/NumbersAsDoubleValueWriter.java | 51 - .../mapr/db/json/OjaiFunctionsProcessor.java | 211 -- .../store/mapr/db/json/OjaiValueWriter.java | 194 -- .../ProjectionPassthroughVectorWriter.java | 82 - .../db/json/RestrictedJsonRecordReader.java | 234 -- .../db/json/RestrictedJsonTableGroupScan.java | 188 -- .../mapr/db/json/RowCountVectorWriter.java | 40 - .../exec/store/mapr/db/util/CommonFns.java | 26 - .../store/mapr/db/util/FieldPathHelper.java | 54 - .../mapr/streams/StreamsFormatMatcher.java | 42 - .../mapr/streams/StreamsFormatPlugin.java | 96 - .../streams/StreamsFormatPluginConfig.java | 49 - .../udf/mapr/db/ConditionPlaceholder.java | 54 - .../exec/udf/mapr/db/DecodeFieldPath.java | 65 - .../exec/udf/mapr/db/MatchesPlaceholder.java | 53 - .../udf/mapr/db/NotMatchesPlaceholder.java | 53 - .../udf/mapr/db/NotTypeOfPlaceholder.java | 53 - .../exec/udf/mapr/db/SizeOfPlaceholder.java | 54 - .../exec/udf/mapr/db/TypeOfPlaceholder.java | 53 - .../resources/bootstrap-format-plugins.json | 20 - .../src/main/resources/drill-module.conf | 41 - .../drill/maprdb/tests/MaprDBTestsSuite.java | 160 -- .../binary/TestMapRDBCFAsJSONString.java | 46 - .../binary/TestMapRDBFilterPushDown.java | 47 - .../binary/TestMapRDBProjectPushDown.java | 46 - .../tests/binary/TestMapRDBQueries.java | 46 - .../maprdb/tests/binary/TestMapRDBSimple.java | 53 - .../maprdb/tests/index/IndexHintPlanTest.java | 165 -- .../maprdb/tests/index/IndexPlanTest.java | 1995 ----------------- .../maprdb/tests/index/LargeTableGen.java | 172 -- .../maprdb/tests/index/LargeTableGenBase.java | 185 -- .../maprdb/tests/index/StatisticsTest.java | 100 - .../maprdb/tests/index/TableIndexCmd.java | 123 - .../drill/maprdb/tests/json/BaseJsonTest.java | 79 - .../tests/json/TestEncodedFieldPaths.java | 128 -- .../tests/json/TestFieldPathHelper.java | 53 - .../maprdb/tests/json/TestScanRanges.java | 157 -- .../maprdb/tests/json/TestSimpleJson.java | 485 ---- .../com/mapr/drill/json/business.json | 10 - .../drill/json/encoded_fields_userdata.json | 5 - .../src/test/resources/core-site.xml | 33 - ...veMapRDBJsonScanToDrillMapRDBJsonScan.java | 202 -- 87 files changed, 14516 deletions(-) delete mode 100644 contrib/format-maprdb/README.md delete mode 100644 contrib/format-maprdb/pom.xml delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDescriptor.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBStatistics.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBStatisticsPayload.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/PluginConstants.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/PluginErrorHandler.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatMatcher.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPlugin.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPluginConfig.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBCost.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatMatcher.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPlugin.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBGroupScan.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushFilterIntoScan.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushLimitIntoScan.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushProjectIntoScan.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBRestrictedScanBatchCreator.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScanSpec.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableCache.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableStats.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/RestrictedMapRDBSubScan.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/RestrictedMapRDBSubScanSpec.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/TabletFragmentInfo.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBBinaryTable.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MaprDBCompareFunctionsProcessor.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/AllTextValueWriter.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/DocumentReaderVectorWriter.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/FieldPathHelper.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/FieldTransferVectorWriter.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/IdOnlyVectorWriter.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonScanSpec.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonSubScanSpec.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableRangePartitionFunction.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/NumbersAsDoubleValueWriter.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/OjaiFunctionsProcessor.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/OjaiValueWriter.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/ProjectionPassthroughVectorWriter.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/RestrictedJsonRecordReader.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/RestrictedJsonTableGroupScan.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/RowCountVectorWriter.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/util/CommonFns.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/util/FieldPathHelper.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatMatcher.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPlugin.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPluginConfig.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/ConditionPlaceholder.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/DecodeFieldPath.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/MatchesPlaceholder.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/NotMatchesPlaceholder.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/NotTypeOfPlaceholder.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/SizeOfPlaceholder.java delete mode 100644 contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/TypeOfPlaceholder.java delete mode 100644 contrib/format-maprdb/src/main/resources/bootstrap-format-plugins.json delete mode 100644 contrib/format-maprdb/src/main/resources/drill-module.conf delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBCFAsJSONString.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBProjectPushDown.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBQueries.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/IndexHintPlanTest.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/IndexPlanTest.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/LargeTableGen.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/LargeTableGenBase.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/StatisticsTest.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/TableIndexCmd.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestEncodedFieldPaths.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestFieldPathHelper.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestScanRanges.java delete mode 100644 contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java delete mode 100644 contrib/format-maprdb/src/test/resources/com/mapr/drill/json/business.json delete mode 100644 contrib/format-maprdb/src/test/resources/com/mapr/drill/json/encoded_fields_userdata.json delete mode 100644 contrib/format-maprdb/src/test/resources/core-site.xml delete mode 100644 contrib/storage-hive/core/scrMapr/main/java/org/apache/drill/exec/planner/sql/logical/ConvertHiveMapRDBJsonScanToDrillMapRDBJsonScan.java diff --git a/contrib/format-maprdb/README.md b/contrib/format-maprdb/README.md deleted file mode 100644 index a94a7cb012b..00000000000 --- a/contrib/format-maprdb/README.md +++ /dev/null @@ -1,8 +0,0 @@ -drill-mapr-plugin -================= -By default all the tests in contrib/format-maprdb are disabled. -To enable and run these tests please use -Pmapr profile to -compile and execute the tests. - -Here is an example of the mvn command to use to run these tests. -mvn install -Dtests=cluster -Pmapr diff --git a/contrib/format-maprdb/pom.xml b/contrib/format-maprdb/pom.xml deleted file mode 100644 index 7a77911c7af..00000000000 --- a/contrib/format-maprdb/pom.xml +++ /dev/null @@ -1,382 +0,0 @@ - - - - 4.0.0 - - - drill-contrib-parent - org.apache.drill.contrib - 1.22.0-SNAPSHOT - - - drill-format-mapr - Drill : Contrib : Format : MaprDB - - - 1.1.1-mapr-1602-m7-5.2.0 - **/MaprDBTestsSuite.class - true - - - - - - org.apache.hbase - hbase-client - ${mapr-format-plugin.hbase.version} - - - log4j - log4j - - - commons-logging - commons-logging - - - org.codehaus.jackson - jackson-core-asl - - - io.netty - netty-all - - - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${mapr.skip.tests} - - ${mapr.TestSuite} - - - - logback.log.dir - ${project.build.directory}/surefire-reports - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - **/core-site.xml - **/logback.xml - - - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-sources-as-resources - process-sources - - add-resource - - - - - src/main/java - - - - - - - - - - - - - com.mapr.hadoop - maprfs - - - commons-codec - commons-codec - - - - - com.mapr.fs - mapr-hbase - - - commons-codec - commons-codec - - - commons-httpclient - commons-httpclient - - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-mapper-asl - - - org.codehaus.jackson - jackson-jaxrs - - - io.netty - netty - - - - - com.mapr.db - maprdb - - - org.apache.drill.exec - drill-java-exec - ${project.version} - - - log4j-over-slf4j - org.slf4j - - - - - org.apache.drill.contrib - drill-storage-hbase - ${project.version} - - - log4j-over-slf4j - org.slf4j - - - - - com.tdunning - json - - - - com.mapr.db - maprdb - ${mapr.release.version} - tests - test - - - com.mapr.hadoop - maprfs - ${mapr.release.version} - tests - test - - - com.jcraft - jsch - 0.1.54 - test - - - com.mapr - mapr-java-utils - ${mapr.release.version} - test - tests - - - com.mapr - mapr-test-annotations - ${mapr.release.version} - - - org.apache.drill.exec - drill-java-exec - ${project.version} - tests - test - - - - - org.apache.drill - drill-common - ${project.version} - tests - test - - - log4j-over-slf4j - org.slf4j - - - - - org.apache.drill.contrib - drill-storage-hbase - ${project.version} - tests - test - - - - - - - - - mapr - - false - - - - - - - simple-tests-default - - !tests - - - com.mapr.tests.annotations.ClusterTest - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - - - - simple-tests - - testssimple - - - com.mapr.tests.annotations.ClusterTest - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - - - - cluster-tests - - testscluster - - - com.mapr.tests.annotations.StressTest - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - - - - stress-tests - - testsstress - - - com.mapr.tests.annotations.IntegrationTest - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - - - - all-tests - - testsall - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - - - - selected-tests - - - test - - - com.mapr.tests.annotations.AlwaysExclude - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - - - - - - diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java deleted file mode 100644 index 67938f3e8db..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBFunctionalIndexInfo.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.planner.index; - -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.apache.drill.common.expression.CastExpression; -import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.SchemaPath; - -import java.util.Map; -import java.util.Set; - -public class MapRDBFunctionalIndexInfo implements FunctionalIndexInfo { - - final private IndexDescriptor indexDesc; - - private boolean hasFunctionalField = false; - - // When we scan schemaPath in groupscan's columns, we check if this column(schemaPath) should be rewritten to '$N', - // When there are more than two functions on the same column in index, CAST(a.b as INT), CAST(a.b as VARCHAR), - // then we should map SchemaPath a.b to a set of SchemaPath, e.g. $1, $2 - private Map> columnToConvert; - - // map of functional index expression to destination SchemaPath e.g. $N - private Map exprToConvert; - - // map of SchemaPath involved in a functional field - private Map> pathsInExpr; - - private Set newPathsForIndexedFunction; - - private Set allPathsInFunction; - - public MapRDBFunctionalIndexInfo(IndexDescriptor indexDesc) { - this.indexDesc = indexDesc; - columnToConvert = Maps.newHashMap(); - exprToConvert = Maps.newHashMap(); - pathsInExpr = Maps.newHashMap(); - // keep the order of new paths, it may be related to the naming policy - newPathsForIndexedFunction = Sets.newLinkedHashSet(); - allPathsInFunction = Sets.newHashSet(); - init(); - } - - private void init() { - int count = 0; - for (LogicalExpression indexedExpr : indexDesc.getIndexColumns()) { - if (!(indexedExpr instanceof SchemaPath)) { - hasFunctionalField = true; - SchemaPath functionalFieldPath = SchemaPath.getSimplePath("$"+count); - newPathsForIndexedFunction.add(functionalFieldPath); - - // now we handle only cast expression - if (indexedExpr instanceof CastExpression) { - // We handle only CAST directly on SchemaPath for now. - SchemaPath pathBeingCasted = (SchemaPath)((CastExpression) indexedExpr).getInput(); - addTargetPathForOriginalPath(pathBeingCasted, functionalFieldPath); - addPathInExpr(indexedExpr, pathBeingCasted); - exprToConvert.put(indexedExpr, functionalFieldPath); - allPathsInFunction.add(pathBeingCasted); - } - - count++; - } - } - } - - private void addPathInExpr(LogicalExpression expr, SchemaPath path) { - if (!pathsInExpr.containsKey(expr)) { - Set newSet = Sets.newHashSet(); - newSet.add(path); - pathsInExpr.put(expr, newSet); - } - else { - pathsInExpr.get(expr).add(path); - } - } - - private void addTargetPathForOriginalPath(SchemaPath origPath, SchemaPath newPath) { - if (!columnToConvert.containsKey(origPath)) { - Set newSet = Sets.newHashSet(); - newSet.add(newPath); - columnToConvert.put(origPath, newSet); - } - else { - columnToConvert.get(origPath).add(newPath); - } - } - - - public boolean hasFunctional() { - return hasFunctionalField; - } - - public IndexDescriptor getIndexDesc() { - return indexDesc; - } - - /** - * getNewPath: for an original path, return new rename '$N' path, notice there could be multiple renamed paths - * if the there are multiple functional indexes refer original path. - * @param path - * @return - */ - public SchemaPath getNewPath(SchemaPath path) { - if (columnToConvert.containsKey(path)) { - return columnToConvert.get(path).iterator().next(); - } - return null; - } - - /** - * return a plain field path if the incoming index expression 'expr' is replaced to be a plain field - * @param expr suppose to be an indexed expression - * @return the renamed schemapath in index table for the indexed expression - */ - public SchemaPath getNewPathFromExpr(LogicalExpression expr) { - if (exprToConvert.containsKey(expr)) { - return (SchemaPath)exprToConvert.get(expr); - } - return null; - } - - /** - * Suppose the index key has functions (rather than plain columns): CAST(a as int), CAST(b as varchar(10)), - * then we want to maintain a mapping of the logical expression of that function to the schema path of the - * base column involved in the function. In this example map has 2 entries: - * CAST(a as int) --> 'a' - * CAST(b as varchar(10)) --> 'b' - * @return the map of indexed expression --> the involved schema paths in a indexed expression - */ - public Map> getPathsInFunctionExpr() { - return pathsInExpr; - } - - - public Map getExprMap() { - return exprToConvert; - } - - public Set allNewSchemaPaths() { - return newPathsForIndexedFunction; - } - - public Set allPathsInFunction() { - return allPathsInFunction; - } - - public boolean supportEqualCharConvertToLike() { - return true; - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDescriptor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDescriptor.java deleted file mode 100644 index e16f138881b..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDescriptor.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.planner.index; - - -import java.util.Collection; -import java.util.List; -import java.util.Set; - -import org.apache.calcite.plan.RelOptCost; -import org.apache.calcite.plan.RelOptPlanner; -import org.apache.calcite.rel.RelFieldCollation.NullDirection; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.exec.expr.CloneVisitor; -import org.apache.drill.exec.physical.base.DbGroupScan; -import org.apache.drill.exec.physical.base.GroupScan; -import org.apache.drill.exec.planner.cost.DrillCostBase; -import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory; -import org.apache.drill.exec.planner.cost.PluginCost; -import org.apache.drill.exec.store.mapr.PluginConstants; -import org.apache.drill.exec.util.EncodedSchemaPathSet; -import org.apache.drill.common.expression.LogicalExpression; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - -public class MapRDBIndexDescriptor extends DrillIndexDescriptor { - - protected final Object desc; - protected final Set allFields; - protected final Set indexedFields; - protected MapRDBFunctionalIndexInfo functionalInfo; - protected PluginCost pluginCost; - - public MapRDBIndexDescriptor(List indexCols, - CollationContext indexCollationContext, - List nonIndexCols, - List rowKeyColumns, - String indexName, - String tableName, - IndexType type, - Object desc, - DbGroupScan scan, - NullDirection nullsDirection) { - super(indexCols, indexCollationContext, nonIndexCols, rowKeyColumns, indexName, tableName, type, nullsDirection); - this.desc = desc; - this.indexedFields = ImmutableSet.copyOf(indexColumns); - this.allFields = new ImmutableSet.Builder() - .add(PluginConstants.DOCUMENT_SCHEMA_PATH) - .addAll(indexColumns) - .addAll(nonIndexColumns) - .build(); - this.pluginCost = scan.getPluginCostModel(); - } - - public Object getOriginalDesc(){ - return desc; - } - - @Override - public boolean isCoveringIndex(List expressions) { - List decodedCols = new DecodePathinExpr().parseExpressions(expressions); - return columnsInIndexFields(decodedCols, allFields); - } - - @Override - public boolean allColumnsIndexed(Collection expressions) { - List decodedCols = new DecodePathinExpr().parseExpressions(expressions); - return columnsInIndexFields(decodedCols, indexedFields); - } - - @Override - public boolean someColumnsIndexed(Collection columns) { - return columnsIndexed(columns, false); - } - - private boolean columnsIndexed(Collection expressions, boolean allColsIndexed) { - List decodedCols = new DecodePathinExpr().parseExpressions(expressions); - if (allColsIndexed) { - return columnsInIndexFields(decodedCols, indexedFields); - } else { - return someColumnsInIndexFields(decodedCols, indexedFields); - } - } - - public FunctionalIndexInfo getFunctionalInfo() { - if (this.functionalInfo == null) { - this.functionalInfo = new MapRDBFunctionalIndexInfo(this); - } - return this.functionalInfo; - } - - /** - * Search through a LogicalExpression, finding all referenced schema paths - * and replace them with decoded paths. - * If one encoded path could be decoded to multiple paths, add these decoded paths to - * the end of returned list of expressions from parseExpressions. - */ - private class DecodePathinExpr extends CloneVisitor { - Set schemaPathSet = Sets.newHashSet(); - - public List parseExpressions(Collection expressions) { - List allCols = Lists.newArrayList(); - Collection decoded; - - for (LogicalExpression expr : expressions) { - LogicalExpression nonDecoded = expr.accept(this, null); - if (nonDecoded != null) { - allCols.add(nonDecoded); - } - } - - if (schemaPathSet.size() > 0) { - decoded = EncodedSchemaPathSet.decode(schemaPathSet); - allCols.addAll(decoded); - } - return allCols; - } - - @Override - public LogicalExpression visitSchemaPath(SchemaPath path, Void value) { - if (EncodedSchemaPathSet.isEncodedSchemaPath(path)) { - // if decoded size is not one, incoming path is encoded path thus there is no cast or other function applied on it, - // since users won't pass in encoded fields, so it is safe to return null, - schemaPathSet.add(path); - return null; - } else { - return path; - } - } - - } - - @Override - public RelOptCost getCost(IndexProperties indexProps, RelOptPlanner planner, - int numProjectedFields, GroupScan primaryTableGroupScan) { - Preconditions.checkArgument(primaryTableGroupScan instanceof DbGroupScan); - DbGroupScan dbGroupScan = (DbGroupScan) primaryTableGroupScan; - DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory(); - double totalRows = indexProps.getTotalRows(); - double leadRowCount = indexProps.getLeadingSelectivity() * totalRows; - double avgRowSize = indexProps.getAvgRowSize(); - if (indexProps.isCovering()) { // covering index - // int numIndexCols = allFields.size(); - // for disk i/o, all index columns are going to be read into memory - double numBlocks = Math.ceil((leadRowCount * avgRowSize)/pluginCost.getBlockSize(primaryTableGroupScan)); - double diskCost = numBlocks * pluginCost.getSequentialBlockReadCost(primaryTableGroupScan); - // cpu cost is cost of filter evaluation for the remainder condition - double cpuCost = 0.0; - if (indexProps.getTotalRemainderFilter() != null) { - cpuCost = leadRowCount * DrillCostBase.COMPARE_CPU_COST; - } - double networkCost = 0.0; // TODO: add network cost once full table scan also considers network cost - return costFactory.makeCost(leadRowCount, cpuCost, diskCost, networkCost); - - } else { // non-covering index - // int numIndexCols = allFields.size(); - double numBlocksIndex = Math.ceil((leadRowCount * avgRowSize)/pluginCost.getBlockSize(primaryTableGroupScan)); - double diskCostIndex = numBlocksIndex * pluginCost.getSequentialBlockReadCost(primaryTableGroupScan); - // for the primary table join-back each row may belong to a different block, so in general num_blocks = num_rows; - // however, num_blocks cannot exceed the total number of blocks of the table - double totalBlocksPrimary = Math.ceil((dbGroupScan.getColumns().size() * - pluginCost.getAverageColumnSize(primaryTableGroupScan) * totalRows)/ - pluginCost.getBlockSize(primaryTableGroupScan)); - double diskBlocksPrimary = Math.min(totalBlocksPrimary, leadRowCount); - double diskCostPrimary = diskBlocksPrimary * pluginCost.getRandomBlockReadCost(primaryTableGroupScan); - double diskCostTotal = diskCostIndex + diskCostPrimary; - - // cpu cost of remainder condition evaluation over the selected rows - double cpuCost = 0.0; - if (indexProps.getTotalRemainderFilter() != null) { - cpuCost = leadRowCount * DrillCostBase.COMPARE_CPU_COST; - } - double networkCost = 0.0; // TODO: add network cost once full table scan also considers network cost - return costFactory.makeCost(leadRowCount, cpuCost, diskCostTotal, networkCost); - } - } - - // Future use once full table scan also includes network cost - private double getNetworkCost(double leadRowCount, int numProjectedFields, boolean isCovering, - GroupScan primaryTableGroupScan) { - if (isCovering) { - // db server will send only the projected columns to the db client for the selected - // number of rows, so network cost is based on the number of actual projected columns - double networkCost = leadRowCount * numProjectedFields * pluginCost.getAverageColumnSize(primaryTableGroupScan); - return networkCost; - } else { - // only the rowkey column is projected from the index and sent over the network - double networkCostIndex = leadRowCount * 1 * pluginCost.getAverageColumnSize(primaryTableGroupScan); - - // after join-back to primary table, all projected columns are sent over the network - double networkCostPrimary = leadRowCount * numProjectedFields * pluginCost.getAverageColumnSize(primaryTableGroupScan); - - return networkCostIndex + networkCostPrimary; - } - - } - - @Override - public PluginCost getPluginCostModel() { - return pluginCost; - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java deleted file mode 100644 index 255afe9e3e6..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBIndexDiscover.java +++ /dev/null @@ -1,362 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.drill.exec.planner.index; - -import org.apache.drill.common.parser.LogicalExpressionParser; -import com.mapr.db.Admin; -import com.mapr.db.MapRDB; -import com.mapr.db.exceptions.DBException; -import com.mapr.db.index.IndexDesc; -import com.mapr.db.index.IndexDesc.MissingAndNullOrdering; -import com.mapr.db.index.IndexFieldDesc; -import org.apache.calcite.rel.RelFieldCollation; -import org.apache.calcite.rel.RelFieldCollation.NullDirection; -import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.drill.common.exceptions.DrillRuntimeException; -import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.types.TypeProtos; -import org.apache.drill.common.types.Types; -import org.apache.drill.common.util.DrillFileUtils; -import org.apache.drill.exec.physical.base.AbstractDbGroupScan; -import org.apache.drill.exec.physical.base.GroupScan; -import org.apache.drill.exec.planner.common.DrillScanRelBase; -import org.apache.drill.exec.planner.logical.DrillTable; -import org.apache.drill.exec.planner.physical.ScanPrel; -import org.apache.drill.exec.store.dfs.DrillFileSystem; -import org.apache.drill.exec.store.dfs.FileSelection; -import org.apache.drill.exec.store.dfs.FileSystemPlugin; -import org.apache.drill.exec.store.mapr.db.MapRDBFormatMatcher; -import org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin; -import org.apache.drill.exec.store.mapr.db.MapRDBGroupScan; -import org.apache.drill.exec.store.mapr.db.json.FieldPathHelper; -import org.apache.drill.exec.util.ImpersonationUtil; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; -import org.ojai.FieldPath; - -import java.io.IOException; -import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import com.google.common.collect.Maps; - -public class MapRDBIndexDiscover extends IndexDiscoverBase implements IndexDiscover { - - public static final String DEFAULT_STRING_CAST_LEN_STR = "256"; - - public MapRDBIndexDiscover(GroupScan inScan, DrillScanRelBase scanRel) { - super((AbstractDbGroupScan) inScan, scanRel); - } - - public MapRDBIndexDiscover(GroupScan inScan, ScanPrel scanRel) { - super((AbstractDbGroupScan) inScan, scanRel); - } - - @Override - public IndexCollection getTableIndex(String tableName) { - return getTableIndexFromMFS(tableName); - } - - /** - * For a given table name get the list of indexes defined on the table according to the visibility of - * the indexes based on permissions. - * @param tableName table name - * @return an IndexCollection representing the list of indexes for that table - */ - private IndexCollection getTableIndexFromMFS(String tableName) { - try { - Set idxSet = new HashSet<>(); - Collection indexes = admin().getTableIndexes(new Path(tableName)); - if (indexes.size() == 0 ) { - logger.error("No index returned from Admin.getTableIndexes for table {}", tableName); - return null; - } - for (IndexDesc idx : indexes) { - DrillIndexDescriptor hbaseIdx = buildIndexDescriptor(tableName, idx); - if (hbaseIdx == null) { - // not able to build a valid index based on the index info from MFS - logger.error("Not able to build index for {}", idx.toString()); - continue; - } - idxSet.add(hbaseIdx); - } - if (idxSet.size() == 0) { - logger.error("No index found for table {}.", tableName); - return null; - } - return new DrillIndexCollection(getOriginalScanRel(), idxSet); - } catch (DBException ex) { - logger.error("Could not get table index from File system.", ex); - } - catch(InvalidIndexDefinitionException ex) { - logger.error("Invalid index definition detected.", ex); - } - return null; - } - - FileSelection deriveFSSelection(DrillFileSystem fs, IndexDescriptor idxDesc) throws IOException { - String tableName = idxDesc.getTableName(); - String[] tablePath = tableName.split(DrillFileUtils.SEPARATOR); - String tableParent = tableName.substring(0, tableName.lastIndexOf(DrillFileUtils.SEPARATOR)); - - return FileSelection.create(fs, tableParent, tablePath[tablePath.length - 1], false); - } - - @Override - public DrillTable getNativeDrillTable(IndexDescriptor idxDescriptor) { - - try { - final AbstractDbGroupScan origScan = getOriginalScan(); - if (!(origScan instanceof MapRDBGroupScan)) { - return null; - } - MapRDBFormatPlugin maprFormatPlugin = ((MapRDBGroupScan) origScan).getFormatPlugin(); - FileSystemPlugin fsPlugin = (FileSystemPlugin) (origScan.getStoragePlugin()); - - DrillFileSystem fs = ImpersonationUtil.createFileSystem(origScan.getUserName(), fsPlugin.getFsConf()); - MapRDBFormatMatcher matcher = (MapRDBFormatMatcher) (maprFormatPlugin.getMatcher()); - FileSelection fsSelection = deriveFSSelection(fs, idxDescriptor); - return matcher.isReadableIndex(fs, fsSelection, fsPlugin, fsPlugin.getName(), - origScan.getUserName(), idxDescriptor); - - } catch (Exception e) { - logger.error("Failed to get native DrillTable.", e); - } - return null; - } - - private SchemaPath fieldName2SchemaPath(String fieldName) { - if (fieldName.contains(":")) { - fieldName = fieldName.split(":")[1]; - } - if (fieldName.contains(".")) { - return FieldPathHelper.fieldPath2SchemaPath(FieldPath.parseFrom(fieldName)); - } - return SchemaPath.getSimplePath(fieldName); - } - - String getDrillTypeStr(String maprdbTypeStr) { - String typeStr = maprdbTypeStr.toUpperCase(); - String[] typeTokens = typeStr.split("[)(]"); - String typeData = DEFAULT_STRING_CAST_LEN_STR; - if(typeTokens.length > 1) { - typeStr = typeTokens[0]; - typeData = typeTokens[1]; - } - switch(typeStr){ - case "STRING": - // set default width since it is not specified - return "VARCHAR("+typeData+")"; - case "LONG": - return "BIGINT"; - case "INT": - case "INTEGER": - return "INT"; - case "FLOAT": - return "FLOAT4"; - case "DOUBLE": - return "FLOAT8"; - case "INTERVAL_YEAR_MONTH": - return "INTERVALYEAR"; - case "INTERVAL_DAY_TIME": - return "INTERVALDAY"; - case "BOOLEAN": - return "BIT"; - case "BINARY": - return "VARBINARY"; - case "ANY": - case "DECIMAL": - return null; - default: return typeStr; - } - - } - - TypeProtos.MajorType getDrillType(String typeStr) { - switch(typeStr){ - case "VARCHAR": - case "CHAR": - case "STRING": - // set default width since it is not specified - return - Types.required(TypeProtos.MinorType.VARCHAR).toBuilder().setWidth( - getOriginalScanRel().getCluster().getTypeFactory().createSqlType(SqlTypeName.VARCHAR).getPrecision()).build(); - case "LONG": - case "BIGINT": - return Types.required(TypeProtos.MinorType.BIGINT); - case "INT": - case "INTEGER": - return Types.required(TypeProtos.MinorType.INT); - case "FLOAT": - return Types.required(TypeProtos.MinorType.FLOAT4); - case "DOUBLE": - return Types.required(TypeProtos.MinorType.FLOAT8); - case "INTERVAL_YEAR_MONTH": - return Types.required(TypeProtos.MinorType.INTERVALYEAR); - case "INTERVAL_DAY_TIME": - return Types.required(TypeProtos.MinorType.INTERVALDAY); - case "BOOLEAN": - return Types.required(TypeProtos.MinorType.BIT); - case "BINARY": - return Types.required(TypeProtos.MinorType.VARBINARY).toBuilder().build(); - case "ANY": - case "DECIMAL": - return null; - default: return Types.required(TypeProtos.MinorType.valueOf(typeStr)); - } - } - - private LogicalExpression castFunctionSQLSyntax(String field, String type) throws InvalidIndexDefinitionException { - // get castTypeStr so we can construct SQL syntax string before MapRDB could provide such syntax - String castTypeStr = getDrillTypeStr(type); - if(castTypeStr == null) { // no cast - throw new InvalidIndexDefinitionException("cast function type not recognized: " + type + "for field " + field); - } - try { - String castFunc = String.format("cast( %s as %s)", field, castTypeStr); - return LogicalExpressionParser.parse(castFunc); - } catch (Exception ex) { - logger.error("parse failed: {}", ex); - } - return null; - } - - private LogicalExpression getIndexExpression(IndexFieldDesc desc) throws InvalidIndexDefinitionException { - final String fieldName = desc.getFieldPath().asPathString(); - final String functionDef = desc.getFunctionName(); - if ((functionDef != null)) { // this is a function - String[] tokens = functionDef.split("\\s+"); - if (tokens[0].equalsIgnoreCase("cast")) { - if (tokens.length != 3) { - throw new InvalidIndexDefinitionException("cast function definition not recognized: " + functionDef); - } - LogicalExpression idxExpr = castFunctionSQLSyntax(fieldName, tokens[2]); - if (idxExpr == null) { - throw new InvalidIndexDefinitionException("got null expression for function definition: " + functionDef); - } - return idxExpr; - } else { - throw new InvalidIndexDefinitionException("function definition is not supported for indexing: " + functionDef); - } - } - // else it is a schemaPath - return fieldName2SchemaPath(fieldName); - } - - private List field2SchemaPath(Collection descCollection) - throws InvalidIndexDefinitionException { - List listSchema = new ArrayList<>(); - for (IndexFieldDesc field : descCollection) { - listSchema.add(getIndexExpression(field)); - } - return listSchema; - } - - private List getFieldCollations(IndexDesc desc, Collection descCollection) { - List fieldCollations = new ArrayList<>(); - int i = 0; - for (IndexFieldDesc field : descCollection) { - RelFieldCollation.Direction direction = (field.getSortOrder() == IndexFieldDesc.Order.Asc) ? - RelFieldCollation.Direction.ASCENDING : (field.getSortOrder() == IndexFieldDesc.Order.Desc ? - RelFieldCollation.Direction.DESCENDING : null); - if (direction != null) { - // assume null direction of NULLS UNSPECIFIED for now until MapR-DB adds that to the APIs - RelFieldCollation.NullDirection nulldir = - direction == RelFieldCollation.Direction.ASCENDING ? NullDirection.LAST : - (direction == RelFieldCollation.Direction.DESCENDING ? - NullDirection.FIRST : NullDirection.UNSPECIFIED); - RelFieldCollation c = new RelFieldCollation(i++, direction, nulldir); - fieldCollations.add(c); - } else { - // if the direction is not present for a field, no need to examine remaining fields - break; - } - } - return fieldCollations; - } - - private CollationContext buildCollationContext(List indexFields, - List indexFieldCollations) { - assert indexFieldCollations.size() <= indexFields.size(); - Map collationMap = Maps.newHashMap(); - for (int i = 0; i < indexFieldCollations.size(); i++) { - collationMap.put(indexFields.get(i), indexFieldCollations.get(i)); - } - return new CollationContext(collationMap, indexFieldCollations); - } - - private DrillIndexDescriptor buildIndexDescriptor(String tableName, IndexDesc desc) - throws InvalidIndexDefinitionException { - if (desc.isExternal()) { - // External index is not currently supported - return null; - } - - IndexDescriptor.IndexType idxType = IndexDescriptor.IndexType.NATIVE_SECONDARY_INDEX; - List indexFields = field2SchemaPath(desc.getIndexedFields()); - List coveringFields = field2SchemaPath(desc.getIncludedFields()); - coveringFields.add(SchemaPath.getSimplePath("_id")); - CollationContext collationContext = null; - if (!desc.isHashed()) { // hash index has no collation property - List indexFieldCollations = getFieldCollations(desc, desc.getIndexedFields()); - collationContext = buildCollationContext(indexFields, indexFieldCollations); - } - - DrillIndexDescriptor idx = new MapRDBIndexDescriptor ( - indexFields, - collationContext, - coveringFields, - null, - desc.getIndexName(), - tableName, - idxType, - desc, - this.getOriginalScan(), - desc.getMissingAndNullOrdering() == MissingAndNullOrdering.MissingAndNullFirst ? NullDirection.FIRST : - (desc.getMissingAndNullOrdering() == MissingAndNullOrdering.MissingAndNullLast ? - NullDirection.LAST : NullDirection.UNSPECIFIED)); - - String storageName = this.getOriginalScan().getStoragePlugin().getName(); - materializeIndex(storageName, idx); - return idx; - } - - @SuppressWarnings("deprecation") - private Admin admin() { - assert getOriginalScan() instanceof MapRDBGroupScan; - - final MapRDBGroupScan dbGroupScan = (MapRDBGroupScan) getOriginalScan(); - final UserGroupInformation currentUser = ImpersonationUtil.createProxyUgi(dbGroupScan.getUserName()); - final Configuration conf = dbGroupScan.getFormatPlugin().getFsConf(); - - final Admin admin; - try { - admin = currentUser.doAs((PrivilegedExceptionAction) () -> MapRDB.getAdmin(conf)); - } catch (Exception e) { - throw new DrillRuntimeException("Failed to get Admin instance for user: " + currentUser.getUserName(), e); - } - return admin; - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBStatistics.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBStatistics.java deleted file mode 100644 index 7edfe8026d4..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBStatistics.java +++ /dev/null @@ -1,997 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.planner.index; - -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.metadata.RelMdUtil; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexLiteral; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.util.Pair; -import org.apache.drill.common.expression.ExpressionStringBuilder; -import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.exec.physical.base.DbGroupScan; -import org.apache.drill.exec.physical.base.GroupScan; -import org.apache.drill.exec.planner.common.DrillScanRelBase; -import org.apache.drill.exec.planner.logical.DrillOptiq; -import org.apache.drill.exec.planner.logical.DrillParseContext; -import org.apache.drill.exec.planner.logical.DrillScanRel; -import org.apache.drill.exec.planner.physical.PlannerSettings; -import org.apache.drill.exec.planner.physical.PrelUtil; -import org.apache.drill.exec.planner.physical.ScanPrel; -import org.apache.drill.exec.store.hbase.HBaseRegexParser; -import org.apache.drill.exec.store.mapr.db.json.JsonTableGroupScan; -import org.apache.hadoop.hbase.HConstants; -import org.ojai.store.QueryCondition; - -import java.util.Arrays; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -public class MapRDBStatistics implements Statistics { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBStatistics.class); - static final String nullConditionAsString = ""; - private double rowKeyJoinBackIOFactor = 1.0; - private boolean statsAvailable = false; - private StatisticsPayload fullTableScanPayload = null; - /* - * The computed statistics are cached in so that any subsequent calls are returned - * from the cache. The is a map of >. The - * does not have a comparator so it is converted to a String for serving as a Map key. This may result - * in logically equivalent conditions considered differently e.g. sal<10 OR sal>100, sal>100 OR sal<10 - * the second map maintains statistics per index as not all statistics are independent of the index - * e.g. average row size. - */ - private Map> statsCache; - /* - * The filter independent computed statistics are cached in so that any subsequent - * calls are returned from the cache. The is a map of . This - * cache maintains statistics per index as not all statistics are independent of the index - * e.g. average row size. - */ - private Map fIStatsCache; - /* - /* - * The mapping between and is kept in . This mapping - * is useful to obtain rowCount for condition specified as required during physical - * planning. Again, both the and are converted to Strings for the lack - * of a comparator. - */ - private Map conditionRexNodeMap; - - public MapRDBStatistics() { - statsCache = new HashMap<>(); - fIStatsCache = new HashMap<>(); - conditionRexNodeMap = new HashMap<>(); - } - - public double getRowKeyJoinBackIOFactor() { - return rowKeyJoinBackIOFactor; - } - - @Override - public boolean isStatsAvailable() { - return statsAvailable; - } - - @Override - public String buildUniqueIndexIdentifier(IndexDescriptor idx) { - if (idx == null) { - return null; - } else { - return idx.getTableName() + "_" + idx.getIndexName(); - } - } - - public String buildUniqueIndexIdentifier(String tableName, String idxName) { - if (tableName == null || idxName == null) { - return null; - } else { - return tableName + "_" + idxName; - } - } - - @Override - /** Returns the number of rows satisfying the given FILTER condition - * @param condition - FILTER specified as a {@link RexNode} - * @param tabIdxName - The table/index identifier - * @return approximate rows satisfying the filter - */ - public double getRowCount(RexNode condition, String tabIdxName, RelNode scanRel) { - String conditionAsStr = nullConditionAsString; - Map payloadMap; - if ((scanRel instanceof DrillScanRel && ((DrillScanRel)scanRel).getGroupScan() instanceof DbGroupScan) - || (scanRel instanceof ScanPrel && ((ScanPrel)scanRel).getGroupScan() instanceof DbGroupScan)) { - if (condition == null && fullTableScanPayload != null) { - return fullTableScanPayload.getRowCount(); - } else if (condition != null) { - conditionAsStr = convertRexToString(condition, scanRel.getRowType()); - payloadMap = statsCache.get(conditionAsStr); - if (payloadMap != null) { - if (payloadMap.get(tabIdxName) != null) { - return payloadMap.get(tabIdxName).getRowCount(); - } else { - // We might not have computed rowcount for the given condition from the tab/index in question. - // For rowcount it does not matter which index was used to get the rowcount for the given condition. - // Hence, just use the first one! - for (String payloadKey : payloadMap.keySet()) { - if (payloadKey != null && payloadMap.get(payloadKey) != null) { - return payloadMap.get(payloadKey).getRowCount(); - } - } - StatisticsPayload anyPayload = payloadMap.entrySet().iterator().next().getValue(); - return anyPayload.getRowCount(); - } - } - } - } - if (statsAvailable) { - logger.debug("Statistics: Filter row count is UNKNOWN for filter: {}", conditionAsStr); - } - return ROWCOUNT_UNKNOWN; - } - - /** Returns the number of rows satisfying the given FILTER condition - * @param condition - FILTER specified as a {@link QueryCondition} - * @param tabIdxName - The table/index identifier - * @return approximate rows satisfying the filter - */ - public double getRowCount(QueryCondition condition, String tabIdxName) { - String conditionAsStr = nullConditionAsString; - Map payloadMap; - if (condition != null - && conditionRexNodeMap.get(condition.toString()) != null) { - String rexConditionAsString = conditionRexNodeMap.get(condition.toString()); - payloadMap = statsCache.get(rexConditionAsString); - if (payloadMap != null) { - if (payloadMap.get(tabIdxName) != null) { - return payloadMap.get(tabIdxName).getRowCount(); - } else { - // We might not have computed rowcount for the given condition from the tab/index in question. - // For rowcount it does not matter which index was used to get the rowcount for the given condition. - // if tabIdxName is null, most likely we have found one from payloadMap and won't come to here. - // If we come to here, we are looking for payload for an index, so let us use any index's payload first! - for (String payloadKey : payloadMap.keySet()) { - if (payloadKey != null && payloadMap.get(payloadKey) != null) { - return payloadMap.get(payloadKey).getRowCount(); - } - } - StatisticsPayload anyPayload = payloadMap.entrySet().iterator().next().getValue(); - return anyPayload.getRowCount(); - } - } - } else if (condition == null - && fullTableScanPayload != null) { - return fullTableScanPayload.getRowCount(); - } - if (condition != null) { - conditionAsStr = condition.toString(); - } - if (statsAvailable) { - logger.debug("Statistics: Filter row count is UNKNOWN for filter: {}", conditionAsStr); - } - return ROWCOUNT_UNKNOWN; - } - - /** Returns the number of leading rows satisfying the given FILTER condition - * @param condition - FILTER specified as a {@link RexNode} - * @param tabIdxName - The table/index identifier - * @param scanRel - The current scanRel - * @return approximate rows satisfying the leading filter - */ - @Override - public double getLeadingRowCount(RexNode condition, String tabIdxName, DrillScanRelBase scanRel) { - String conditionAsStr = nullConditionAsString; - Map payloadMap; - if ((scanRel instanceof DrillScanRel && ((DrillScanRel)scanRel).getGroupScan() instanceof DbGroupScan) - || (scanRel instanceof ScanPrel && ((ScanPrel)scanRel).getGroupScan() instanceof DbGroupScan)) { - if (condition == null && fullTableScanPayload != null) { - return fullTableScanPayload.getLeadingRowCount(); - } else if (condition != null) { - conditionAsStr = convertRexToString(condition, scanRel.getRowType()); - payloadMap = statsCache.get(conditionAsStr); - if (payloadMap != null) { - if (payloadMap.get(tabIdxName) != null) { - return payloadMap.get(tabIdxName).getLeadingRowCount(); - } - // Unlike rowcount, leading rowcount is dependent on the index. So, if tab/idx is - // not found, we are out of luck! - } - } - } - if (statsAvailable) { - logger.debug("Statistics: Leading filter row count is UNKNOWN for filter: {}", conditionAsStr); - } - return ROWCOUNT_UNKNOWN; - } - - /** Returns the number of leading rows satisfying the given FILTER condition - * @param condition - FILTER specified as a {@link QueryCondition} - * @param tabIdxName - The table/index identifier - * @return approximate rows satisfying the leading filter - */ - public double getLeadingRowCount(QueryCondition condition, String tabIdxName) { - String conditionAsStr = nullConditionAsString; - Map payloadMap; - if (condition != null - && conditionRexNodeMap.get(condition.toString()) != null) { - String rexConditionAsString = conditionRexNodeMap.get(condition.toString()); - payloadMap = statsCache.get(rexConditionAsString); - if (payloadMap != null) { - if (payloadMap.get(tabIdxName) != null) { - return payloadMap.get(tabIdxName).getLeadingRowCount(); - } - // Unlike rowcount, leading rowcount is dependent on the index. So, if tab/idx is - // not found, we are out of luck! - } - } else if (condition == null - && fullTableScanPayload != null) { - return fullTableScanPayload.getLeadingRowCount(); - } - if (condition != null) { - conditionAsStr = condition.toString(); - } - if (statsAvailable) { - logger.debug("Statistics: Leading filter row count is UNKNOWN for filter: {}", conditionAsStr); - } - return ROWCOUNT_UNKNOWN; - } - - @Override - public double getAvgRowSize(String tabIdxName, boolean isTableScan) { - StatisticsPayload payloadMap; - if (isTableScan && fullTableScanPayload != null) { - return fullTableScanPayload.getAvgRowSize(); - } else if (!isTableScan) { - payloadMap = fIStatsCache.get(tabIdxName); - if (payloadMap != null) { - return payloadMap.getAvgRowSize(); - } - } - if (statsAvailable) { - logger.debug("Statistics: Average row size is UNKNOWN for table: {}", tabIdxName); - } - return AVG_ROWSIZE_UNKNOWN; - } - - public boolean initialize(RexNode condition, DrillScanRelBase scanRel, IndexCallContext context) { - GroupScan scan = IndexPlanUtils.getGroupScan(scanRel); - - PlannerSettings settings = PrelUtil.getPlannerSettings(scanRel.getCluster().getPlanner()); - rowKeyJoinBackIOFactor = settings.getIndexRowKeyJoinCostFactor(); - if (scan instanceof DbGroupScan) { - String conditionAsStr = convertRexToString(condition, scanRel.getRowType()); - if (statsCache.get(conditionAsStr) == null) { - IndexCollection indexes = ((DbGroupScan)scan).getSecondaryIndexCollection(scanRel); - populateStats(condition, indexes, scanRel, context); - logger.info("index_plan_info: initialize: scanRel #{} and groupScan {} got fulltable {}, statsCache: {}, fiStatsCache: {}", - scanRel.getId(), System.identityHashCode(scan), fullTableScanPayload, statsCache, fIStatsCache); - return true; - } - } - return false; - } - - /** - * This function computes statistics when there is no query condition - * @param jTabGrpScan - The current group scan - * @param indexes - The collection of indexes to use for getting statistics - * @param scanRel - The current scanRel - * @param context - The index plan call context - */ - private void populateStatsForNoFilter(JsonTableGroupScan jTabGrpScan, IndexCollection indexes, RelNode scanRel, - IndexCallContext context) { - // Get the stats payload for full table (has total rows in the table) - StatisticsPayload ftsPayload = jTabGrpScan.getFirstKeyEstimatedStats(null, null, scanRel); - addToCache(null, null, context, ftsPayload, jTabGrpScan, scanRel, scanRel.getRowType()); - addToCache(null, jTabGrpScan.getAverageRowSizeStats(null), ftsPayload); - // Get the stats for all indexes - for (IndexDescriptor idx: indexes) { - StatisticsPayload idxPayload = jTabGrpScan.getFirstKeyEstimatedStats(null, idx, scanRel); - StatisticsPayload idxRowSizePayload = jTabGrpScan.getAverageRowSizeStats(idx); - RelDataType newRowType; - FunctionalIndexInfo functionInfo = idx.getFunctionalInfo(); - if (functionInfo.hasFunctional()) { - newRowType = FunctionalIndexHelper.rewriteFunctionalRowType(scanRel, context, functionInfo); - } else { - newRowType = scanRel.getRowType(); - } - addToCache(null, idx, context, idxPayload, jTabGrpScan, scanRel, newRowType); - addToCache(idx, idxRowSizePayload, ftsPayload); - } - } - - /** - * This is the core statistics function for populating the statistics. The statistics populated correspond to the query - * condition. Based on different types of plans, we would need statistics for different combinations of predicates. Currently, - * we do not have a tree-walker for {@link QueryCondition}. Hence, instead of using the individual predicates stats, to construct - * the stats for the overall predicates, we rely on using the final predicates. Hence, this has a limitation(susceptible) to - * predicate modification post stats generation. Statistics computed/stored are rowcounts, leading rowcounts, average rowsize. - * Rowcounts and leading rowcounts (i.e. corresponding to predicates on the leading index columns) are stored in the statsCache. - * Average rowsizes are stored in the fiStatsCache (FI stands for Filter Independent). - * - * @param condition - The condition for which to obtain statistics - * @param indexes - The collection of indexes to use for getting statistics - * @param scanRel - The current scanRel - * @param context - The index plan call context - */ - private void populateStats(RexNode condition, IndexCollection indexes, DrillScanRelBase scanRel, - IndexCallContext context) { - JsonTableGroupScan jTabGrpScan; - Map firstKeyIdxConditionMap; - Map idxConditionMap; - /* Map containing the individual base conditions of an ANDed/ORed condition and their selectivities. - * This is used to compute the overall selectivity of a complex ANDed/ORed condition using its base - * conditions. Helps prevent over/under estimates and guessed selectivity for ORed predicates. - */ - Map baseConditionMap; - GroupScan grpScan = IndexPlanUtils.getGroupScan(scanRel); - - if ((scanRel instanceof DrillScanRel || scanRel instanceof ScanPrel) && - grpScan instanceof JsonTableGroupScan) { - jTabGrpScan = (JsonTableGroupScan) grpScan; - } else { - logger.debug("Statistics: populateStats exit early - not an instance of JsonTableGroupScan!"); - return; - } - if (condition == null) { - populateStatsForNoFilter(jTabGrpScan, indexes, scanRel, context); - statsAvailable = true; - return; - } - - RexBuilder builder = scanRel.getCluster().getRexBuilder(); - PlannerSettings settings = PrelUtil.getSettings(scanRel.getCluster()); - // Get the stats payload for full table (has total rows in the table) - StatisticsPayload ftsPayload = jTabGrpScan.getFirstKeyEstimatedStats(null, null, scanRel); - - // Get the average row size for table and all indexes - addToCache(null, jTabGrpScan.getAverageRowSizeStats(null), ftsPayload); - if (ftsPayload == null || ftsPayload.getRowCount() == 0) { - return; - } - for (IndexDescriptor idx : indexes) { - StatisticsPayload idxRowSizePayload = jTabGrpScan.getAverageRowSizeStats(idx); - addToCache(idx, idxRowSizePayload, ftsPayload); - } - - /* Only use indexes with distinct first key */ - IndexCollection distFKeyIndexes = distinctFKeyIndexes(indexes, scanRel); - IndexConditionInfo.Builder infoBuilder = IndexConditionInfo.newBuilder(condition, - distFKeyIndexes, builder, scanRel); - idxConditionMap = infoBuilder.getIndexConditionMap(); - firstKeyIdxConditionMap = infoBuilder.getFirstKeyIndexConditionMap(); - baseConditionMap = new HashMap<>(); - for (IndexDescriptor idx : firstKeyIdxConditionMap.keySet()) { - if(IndexPlanUtils.conditionIndexed(context.getOrigMarker(), idx) == IndexPlanUtils.ConditionIndexed.NONE) { - continue; - } - RexNode idxCondition = firstKeyIdxConditionMap.get(idx).indexCondition; - /* Use the pre-processed condition only for getting actual statistic from MapR-DB APIs. Use the - * original condition everywhere else (cache store/lookups) since the RexNode condition and its - * corresponding QueryCondition will be used to get statistics. e.g. we convert LIKE into RANGE - * condition to get statistics. However, statistics are always asked for LIKE and NOT the RANGE - */ - RexNode preProcIdxCondition = convertToStatsCondition(idxCondition, idx, context, scanRel, - Arrays.asList(SqlKind.CAST, SqlKind.LIKE)); - RelDataType newRowType; - FunctionalIndexInfo functionInfo = idx.getFunctionalInfo(); - if (functionInfo.hasFunctional()) { - newRowType = FunctionalIndexHelper.rewriteFunctionalRowType(scanRel, context, functionInfo); - } else { - newRowType = scanRel.getRowType(); - } - - QueryCondition queryCondition = jTabGrpScan.convertToQueryCondition( - convertToLogicalExpression(preProcIdxCondition, newRowType, settings, builder)); - // Cap rows/size at total rows in case of issues with DB APIs - StatisticsPayload idxPayload = jTabGrpScan.getFirstKeyEstimatedStats(queryCondition, idx, scanRel); - double rowCount = Math.min(idxPayload.getRowCount(), ftsPayload.getRowCount()); - double leadingRowCount = Math.min(idxPayload.getLeadingRowCount(), rowCount); - double avgRowSize = Math.min(idxPayload.getAvgRowSize(), ftsPayload.getAvgRowSize()); - StatisticsPayload payload = new MapRDBStatisticsPayload(rowCount, leadingRowCount, avgRowSize); - addToCache(idxCondition, idx, context, payload, jTabGrpScan, scanRel, newRowType); - addBaseConditions(idxCondition, payload, false, baseConditionMap, scanRel.getRowType()); - } - /* Add the row count for index conditions on all indexes. Stats are only computed for leading - * keys but index conditions can be pushed and would be required for access path costing - */ - for (IndexDescriptor idx : idxConditionMap.keySet()) { - if(IndexPlanUtils.conditionIndexed(context.getOrigMarker(), idx) == IndexPlanUtils.ConditionIndexed.NONE) { - continue; - } - Map leadingPrefixMap = Maps.newHashMap(); - double rowCount, leadingRowCount, avgRowSize; - RexNode idxCondition = idxConditionMap.get(idx).indexCondition; - // Ignore conditions which always evaluate to true - if (idxCondition.isAlwaysTrue()) { - continue; - } - RexNode idxIncColCondition = idxConditionMap.get(idx).remainderCondition; - RexNode idxRemColCondition = IndexPlanUtils.getLeadingPrefixMap(leadingPrefixMap, idx.getIndexColumns(), infoBuilder, idxCondition); - RexNode idxLeadColCondition = IndexPlanUtils.getLeadingColumnsFilter( - IndexPlanUtils.getLeadingFilters(leadingPrefixMap, idx.getIndexColumns()), builder); - RexNode idxTotRemColCondition = IndexPlanUtils.getTotalRemainderFilter(idxRemColCondition, idxIncColCondition, builder); - RexNode idxTotColCondition = IndexPlanUtils.getTotalFilter(idxLeadColCondition, idxTotRemColCondition, builder); - FunctionalIndexInfo functionInfo = idx.getFunctionalInfo(); - RelDataType newRowType = scanRel.getRowType(); - if (functionInfo.hasFunctional()) { - newRowType = FunctionalIndexHelper.rewriteFunctionalRowType(scanRel, context, functionInfo); - } - /* For non-covering plans we would need the index leading condition */ - rowCount = ftsPayload.getRowCount() * computeSelectivity(idxLeadColCondition, idx, - ftsPayload.getRowCount(), scanRel, baseConditionMap).left; - leadingRowCount = rowCount; - avgRowSize = fIStatsCache.get(buildUniqueIndexIdentifier(idx)).getAvgRowSize(); - addToCache(idxLeadColCondition, idx, context, new MapRDBStatisticsPayload(rowCount, leadingRowCount, avgRowSize), - jTabGrpScan, scanRel, newRowType); - /* For covering plans we would need the full condition */ - rowCount = ftsPayload.getRowCount() * computeSelectivity(idxTotColCondition, idx, - ftsPayload.getRowCount(), scanRel, baseConditionMap).left; - addToCache(idxTotColCondition, idx, context, new MapRDBStatisticsPayload(rowCount, leadingRowCount, avgRowSize), - jTabGrpScan, scanRel, newRowType); - /* For intersect plans we would need the index condition */ - rowCount = ftsPayload.getRowCount() * computeSelectivity(idxCondition, idx, - ftsPayload.getRowCount(), scanRel, baseConditionMap).left; - addToCache(idxCondition, idx, context, new MapRDBStatisticsPayload(rowCount, leadingRowCount, avgRowSize), - jTabGrpScan, scanRel, newRowType); - /* Add the rowCount for condition on only included columns - no leading columns here! */ - if (idxIncColCondition != null) { - rowCount = ftsPayload.getRowCount() * computeSelectivity(idxIncColCondition, null, - ftsPayload.getRowCount(), scanRel, baseConditionMap).left; - addToCache(idxIncColCondition, idx, context, new MapRDBStatisticsPayload(rowCount, rowCount, avgRowSize), - jTabGrpScan, scanRel, newRowType); - } - } - - // Add the rowCount for the complete condition - based on table - double rowCount = ftsPayload.getRowCount() * computeSelectivity(condition, null, - ftsPayload.getRowCount(), scanRel, baseConditionMap).left; - // Here, ftsLeadingKey rowcount is based on _id predicates - StatisticsPayload ftsLeadingKeyPayload = jTabGrpScan.getFirstKeyEstimatedStats(jTabGrpScan.convertToQueryCondition( - convertToLogicalExpression(condition, scanRel.getRowType(), settings, builder)), null, scanRel); - addToCache(condition, null, null, new MapRDBStatisticsPayload(rowCount, ftsLeadingKeyPayload.getRowCount(), - ftsPayload.getAvgRowSize()), jTabGrpScan, scanRel, scanRel.getRowType()); - // Add the full table rows while we are at it - represented by RexNode, QueryCondition. - // No ftsLeadingKey so leadingKeyRowcount = totalRowCount - addToCache(null, null, null, new MapRDBStatisticsPayload(ftsPayload.getRowCount(), ftsPayload.getRowCount(), - ftsPayload.getAvgRowSize()), jTabGrpScan, scanRel, scanRel.getRowType()); - // mark stats has been statsAvailable - statsAvailable = true; - } - - private boolean addBaseConditions(RexNode condition, StatisticsPayload payload, boolean redundant, - Map baseConditionMap, RelDataType rowType) { - boolean res = redundant; - if (condition.getKind() == SqlKind.AND) { - for(RexNode pred : RelOptUtil.conjunctions(condition)) { - res = addBaseConditions(pred, payload, res, baseConditionMap, rowType); - } - } else if (condition.getKind() == SqlKind.OR) { - for(RexNode pred : RelOptUtil.disjunctions(condition)) { - res = addBaseConditions(pred, payload, res, baseConditionMap, rowType); - } - } else { - // base condition - String conditionAsStr = convertRexToString(condition, rowType); - if (!redundant) { - baseConditionMap.put(conditionAsStr, payload.getRowCount()); - return true; - } else { - baseConditionMap.put(conditionAsStr, -1.0); - return false; - } - } - return res; - } - /* - * Adds the statistic(row count) to the cache. Also adds the corresponding QueryCondition->RexNode - * condition mapping. - */ - private void addToCache(RexNode condition, IndexDescriptor idx, IndexCallContext context, - StatisticsPayload payload, JsonTableGroupScan jTabGrpScan, RelNode scanRel, RelDataType rowType) { - if (condition != null - && !condition.isAlwaysTrue()) { - RexBuilder builder = scanRel.getCluster().getRexBuilder(); - PlannerSettings settings = PrelUtil.getSettings(scanRel.getCluster()); - String conditionAsStr = convertRexToString(condition, scanRel.getRowType()); - if (statsCache.get(conditionAsStr) == null - && payload.getRowCount() != Statistics.ROWCOUNT_UNKNOWN) { - Map payloadMap = new HashMap<>(); - payloadMap.put(buildUniqueIndexIdentifier(idx), payload); - statsCache.put(conditionAsStr, payloadMap); - logger.debug("Statistics: StatsCache:<{}, {}>",conditionAsStr, payload); - // Always pre-process CAST conditions - Otherwise queryCondition will not be generated correctly - RexNode preProcIdxCondition = convertToStatsCondition(condition, idx, context, scanRel, - Arrays.asList(SqlKind.CAST)); - QueryCondition queryCondition = - jTabGrpScan.convertToQueryCondition(convertToLogicalExpression(preProcIdxCondition, - rowType, settings, builder)); - if (queryCondition != null) { - String queryConditionAsStr = queryCondition.toString(); - if (conditionRexNodeMap.get(queryConditionAsStr) == null) { - conditionRexNodeMap.put(queryConditionAsStr, conditionAsStr); - logger.debug("Statistics: QCRNCache:<{}, {}>",queryConditionAsStr, conditionAsStr); - } - } else { - logger.debug("Statistics: QCRNCache: Unable to generate QueryCondition for {}", conditionAsStr); - logger.debug("Statistics: QCRNCache: Unable to generate QueryCondition for {}", conditionAsStr); - } - } else { - Map payloadMap = statsCache.get(conditionAsStr); - if (payloadMap != null) { - if (payloadMap.get(buildUniqueIndexIdentifier(idx)) == null) { - payloadMap.put(buildUniqueIndexIdentifier(idx), payload); - - // rowCount for the same condition should be the same on primary table or index, - // let us sync them to the smallest since currently both are over-estimated. - // DO NOT sync the leading rowCount since it is based on the leading condition and not the - // condition (key for this cache). Hence, for the same condition the leading condition and - // consequently the leading rowCount will vary with the index. Syncing them may lead to - // unintended side-effects e.g. given a covering index and full table scan and a condition - // on a non-id field which happens to be the leading key in the index, the leading rowcount - // for the full table scan should be the full table rowcount. Syncing them would incorrectly - // make the full table scan cheaper! If required, syncing should be only done based on - // leading condition and NOT the condition - double minimalRowCount = payload.getRowCount(); - for (StatisticsPayload existing : payloadMap.values()) { - if (existing.getRowCount() < minimalRowCount) { - minimalRowCount = existing.getRowCount(); - } - } - for (StatisticsPayload existing : payloadMap.values()) { - if (existing instanceof MapRDBStatisticsPayload) { - ((MapRDBStatisticsPayload)existing).rowCount = minimalRowCount; - } - } - } else { - logger.debug("Statistics: Filter row count already exists for filter: {}. Skip!", conditionAsStr); - } - } else { - logger.debug("Statistics: Filter row count is UNKNOWN for filter: {}", conditionAsStr); - } - } - } else if (condition == null && idx == null) { - fullTableScanPayload = new MapRDBStatisticsPayload(payload.getRowCount(), - payload.getLeadingRowCount(), payload.getAvgRowSize()); - logger.debug("Statistics: StatsCache:<{}, {}>","NULL", fullTableScanPayload); - } - } - - private void addToCache(IndexDescriptor idx, StatisticsPayload payload, StatisticsPayload ftsPayload) { - String tabIdxIdentifier = buildUniqueIndexIdentifier(idx); - if (fIStatsCache.get(tabIdxIdentifier) == null) { - if (ftsPayload.getAvgRowSize() >= payload.getAvgRowSize()) { - fIStatsCache.put(tabIdxIdentifier, payload); - logger.debug("Statistics: fIStatsCache:<{}, {}>",tabIdxIdentifier, payload); - } else { - StatisticsPayload cappedPayload = - new MapRDBStatisticsPayload(ROWCOUNT_UNKNOWN, ROWCOUNT_UNKNOWN, ftsPayload.getAvgRowSize()); - fIStatsCache.put(tabIdxIdentifier,cappedPayload); - logger.debug("Statistics: fIStatsCache:<{}, {}> (Capped)",tabIdxIdentifier, cappedPayload); - } - } else { - logger.debug("Statistics: Average row size already exists for :<{}, {}>. Skip!",tabIdxIdentifier, payload); - } - } - - /* - * Convert the given RexNode to a String representation while also replacing the RexInputRef references - * to actual column names. Since, we compare String representations of RexNodes, two equivalent RexNode - * expressions may differ in the RexInputRef positions but otherwise the same. - * e.g. $1 = 'CA' projection (State, Country) , $2 = 'CA' projection (Country, State) - */ - private String convertRexToString(RexNode condition, RelDataType rowType) { - StringBuilder sb = new StringBuilder(); - if (condition == null) { - return null; - } - if (condition.getKind() == SqlKind.AND) { - boolean first = true; - for(RexNode pred : RelOptUtil.conjunctions(condition)) { - if (first) { - sb.append(convertRexToString(pred, rowType)); - first = false; - } else { - sb.append(" " + SqlKind.AND.toString() + " "); - sb.append(convertRexToString(pred, rowType)); - } - } - return sb.toString(); - } else if (condition.getKind() == SqlKind.OR) { - boolean first = true; - for(RexNode pred : RelOptUtil.disjunctions(condition)) { - if (first) { - sb.append(convertRexToString(pred, rowType)); - first = false; - } else { - sb.append(" " + SqlKind.OR.toString() + " "); - sb.append(convertRexToString(pred, rowType)); - } - } - return sb.toString(); - } else { - HashMap inputRefMapping = new HashMap<>(); - /* Based on the rel projection the input reference for the same column may change - * during planning. We want the cache to be agnostic to it. Hence, the entry stored - * in the cache has the input reference ($i) replaced with the column name - */ - getInputRefMapping(condition, rowType, inputRefMapping); - if (inputRefMapping.keySet().size() > 0) { - //Found input ref - replace it - String replCondition = condition.toString(); - for (String inputRef : inputRefMapping.keySet()) { - replCondition = replCondition.replace(inputRef, inputRefMapping.get(inputRef)); - } - return replCondition; - } else { - return condition.toString(); - } - } - } - - /* - * Generate the input reference to column mapping for reference replacement. Please - * look at the usage in convertRexToString() to understand why this mapping is required. - */ - private void getInputRefMapping(RexNode condition, RelDataType rowType, - HashMap mapping) { - if (condition instanceof RexCall) { - for (RexNode op : ((RexCall) condition).getOperands()) { - getInputRefMapping(op, rowType, mapping); - } - } else if (condition instanceof RexInputRef) { - mapping.put(condition.toString(), - rowType.getFieldNames().get(condition.hashCode())); - } - } - - /* - * Additional pre-processing may be required for LIKE/CAST predicates in order to compute statistics. - * e.g. A LIKE predicate should be converted to a RANGE predicate for statistics computation. MapR-DB - * does not yet support computing statistics for LIKE predicates. - */ - private RexNode convertToStatsCondition(RexNode condition, IndexDescriptor index, - IndexCallContext context, RelNode scanRel, ListtypesToProcess) { - RexBuilder builder = scanRel.getCluster().getRexBuilder(); - if (condition.getKind() == SqlKind.AND) { - final List conditions = Lists.newArrayList(); - for(RexNode pred : RelOptUtil.conjunctions(condition)) { - conditions.add(convertToStatsCondition(pred, index, context, scanRel, typesToProcess)); - } - return RexUtil.composeConjunction(builder, conditions, false); - } else if (condition.getKind() == SqlKind.OR) { - final List conditions = Lists.newArrayList(); - for(RexNode pred : RelOptUtil.disjunctions(condition)) { - conditions.add(convertToStatsCondition(pred, index, context, scanRel, typesToProcess)); - } - return RexUtil.composeDisjunction(builder, conditions, false); - } else if (condition instanceof RexCall) { - // LIKE operator - convert to a RANGE predicate, if possible - if (typesToProcess.contains(SqlKind.LIKE) - && ((RexCall) condition).getOperator().getKind() == SqlKind.LIKE) { - return convertLikeToRange((RexCall)condition, builder); - } else if (typesToProcess.contains(SqlKind.CAST) - && hasCastExpression(condition)) { - return convertCastForFIdx(((RexCall) condition), index, context, scanRel); - } - else { - return condition; - } - } - return condition; - } - - /* - * Determines whether the given expression contains a CAST expression. Assumes that the - * given expression is a valid expression. - * Returns TRUE, if it finds at least one instance of CAST operator. - */ - private boolean hasCastExpression(RexNode condition) { - if (condition instanceof RexCall) { - if (((RexCall) condition).getOperator().getKind() == SqlKind.CAST) { - return true; - } - for (RexNode op : ((RexCall) condition).getOperands()) { - if (hasCastExpression(op)) { - return true; - } - } - } - return false; - } - /* - * CAST expressions are not understood by MAPR-DB as-is. Hence, we must convert them before passing them - * onto MAPR-DB for statistics. Given a functional index, the given expression is converted into an - * expression on the `expression` column of the functional index. - */ - private RexNode convertCastForFIdx(RexCall condition, IndexDescriptor index, - IndexCallContext context, RelNode origScan) { - if (index == null) { - return condition; - } - FunctionalIndexInfo functionInfo = index.getFunctionalInfo(); - if (!functionInfo.hasFunctional()) { - return condition; - } - // The functional index has a different row-type than the original scan. Use the index row-type when - // converting the condition - RelDataType newRowType = FunctionalIndexHelper.rewriteFunctionalRowType(origScan, context, functionInfo); - RexBuilder builder = origScan.getCluster().getRexBuilder(); - return FunctionalIndexHelper.convertConditionForIndexScan(condition, - origScan, newRowType, builder, functionInfo); - } - - /* - * Helper function to perform additional pre-processing for LIKE predicates - */ - private RexNode convertLikeToRange(RexCall condition, RexBuilder builder) { - Preconditions.checkArgument(condition.getOperator().getKind() == SqlKind.LIKE, - "Unable to convertLikeToRange: argument is not a LIKE condition!"); - HBaseRegexParser parser = null; - RexNode arg = null; - RexLiteral pattern = null, escape = null; - String patternStr = null, escapeStr = null; - if (condition.getOperands().size() == 2) { - // No escape character specified - for (RexNode op : condition.getOperands()) { - if (op.getKind() == SqlKind.LITERAL) { - pattern = (RexLiteral) op; - } else { - arg = op; - } - } - // Get the PATTERN strings from the corresponding RexLiteral - if (pattern.getTypeName() == SqlTypeName.DECIMAL || - pattern.getTypeName() == SqlTypeName.INTEGER) { - patternStr = pattern.getValue().toString(); - } else if (pattern.getTypeName() == SqlTypeName.CHAR) { - patternStr = pattern.getValue2().toString(); - } - if (patternStr != null) { - parser = new HBaseRegexParser(patternStr); - } - } else if (condition.getOperands().size() == 3) { - // Escape character specified - for (RexNode op : condition.getOperands()) { - if (op.getKind() == SqlKind.LITERAL) { - // Assume first literal specifies PATTERN and the second literal specifies the ESCAPE char - if (pattern == null) { - pattern = (RexLiteral) op; - } else { - escape = (RexLiteral) op; - } - } else { - arg = op; - } - } - // Get the PATTERN and ESCAPE strings from the corresponding RexLiteral - if (pattern.getTypeName() == SqlTypeName.DECIMAL || - pattern.getTypeName() == SqlTypeName.INTEGER) { - patternStr = pattern.getValue().toString(); - } else if (pattern.getTypeName() == SqlTypeName.CHAR) { - patternStr = pattern.getValue2().toString(); - } - if (escape.getTypeName() == SqlTypeName.DECIMAL || - escape.getTypeName() == SqlTypeName.INTEGER) { - escapeStr = escape.getValue().toString(); - } else if (escape.getTypeName() == SqlTypeName.CHAR) { - escapeStr = escape.getValue2().toString(); - } - if (patternStr != null && escapeStr != null) { - parser = new HBaseRegexParser(patternStr, escapeStr.toCharArray()[0]); - } - } - if (parser != null) { - parser.parse(); - String prefix = parser.getPrefixString(); - /* - * If there is a literal prefix, convert it into an EQUALITY or RANGE predicate - */ - if (prefix != null) { - if (prefix.equals(parser.getLikeString())) { - // No WILDCARD present. This turns the LIKE predicate to EQUALITY predicate - if (arg != null) { - return builder.makeCall(SqlStdOperatorTable.EQUALS, arg, pattern); - } - } else { - // WILDCARD present. This turns the LIKE predicate to RANGE predicate - byte[] startKey = HConstants.EMPTY_START_ROW; - byte[] stopKey = HConstants.EMPTY_END_ROW; - startKey = prefix.getBytes(StandardCharsets.UTF_8); - stopKey = startKey.clone(); - boolean isMaxVal = true; - for (int i = stopKey.length - 1; i >= 0; --i) { - int nextByteValue = (0xff & stopKey[i]) + 1; - if (nextByteValue < 0xff) { - stopKey[i] = (byte) nextByteValue; - isMaxVal = false; - break; - } else { - stopKey[i] = 0; - } - } - if (isMaxVal) { - stopKey = HConstants.EMPTY_END_ROW; - } - // TODO: This maybe a potential bug since we assume UTF-8 encoding. However, we follow the - // current DB implementation. See HBaseFilterBuilder.createHBaseScanSpec "like" CASE statement - RexLiteral startKeyLiteral = builder.makeLiteral(new String(startKey, - StandardCharsets.UTF_8)); - RexLiteral stopKeyLiteral = builder.makeLiteral(new String(stopKey, - StandardCharsets.UTF_8)); - if (arg != null) { - RexNode startPred = builder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, - arg, startKeyLiteral); - RexNode stopPred = builder.makeCall(SqlStdOperatorTable.LESS_THAN, arg, stopKeyLiteral); - return builder.makeCall(SqlStdOperatorTable.AND, startPred, stopPred); - } - } - } - } - // Could not convert - return condition as-is. - return condition; - } - - /* - * Compute the selectivity of the given rowCondition. Retrieve the selectivity - * for index conditions from the cache - */ - private Pair computeSelectivity(RexNode condition, IndexDescriptor idx, double totalRows, - RelNode scanRel, Map baseConditionMap) { - double selectivity; - boolean guess = false; - if (totalRows <= 0) { - return new Pair<>(1.0, true); - } - String conditionAsStr = convertRexToString(condition, scanRel.getRowType()); - if (condition.getKind() == SqlKind.AND) { - selectivity = 1.0; - for (RexNode pred : RelOptUtil.conjunctions(condition)) { - Pair selPayload = computeSelectivity(pred, idx, totalRows, scanRel, baseConditionMap); - if (selPayload.left > 0) { - // At least one AND branch is a guess - if (selPayload.right == true) { - guess = true; - } - selectivity *= selPayload.left; - } - } - } else if (condition.getKind() == SqlKind.OR) { - selectivity = 0.0; - for (RexNode pred : RelOptUtil.disjunctions(condition)) { - Pair selPayload = computeSelectivity(pred, idx, totalRows, scanRel, baseConditionMap); - if (selPayload.left > 0.0) { - // At least one OR branch is a guess - if (selPayload.right == true) { - guess = true; - } - selectivity += selPayload.left; - } - } - // Cap selectivity of OR'ed predicates at 0.25 if at least one predicate is a guess (Calcite does the same) - if (guess && selectivity > 0.25) { - selectivity = 0.25; - } - } else { - guess = false; - if (baseConditionMap.get(conditionAsStr) != null) { - double rowCount = baseConditionMap.get(conditionAsStr); - if (rowCount != -1.0) { - selectivity = rowCount / totalRows; - } else { - // Ignore - selectivity = -1.0; - guess = true; - } - } else { - selectivity = RelMdUtil.guessSelectivity(condition); - guess = true; - } - return new Pair<>(selectivity, guess); - } - // Cap selectivity to be between 0.0 and 1.0 - selectivity = Math.min(1.0, selectivity); - selectivity = Math.max(0.0, selectivity); - logger.debug("Statistics: computeSelectivity: Cache MISS: Computed {} -> {}", conditionAsStr, selectivity); - return new Pair<>(selectivity, guess); - } - - /* - * Filters out indexes from the given collection based on the row key of indexes i.e. after filtering - * the given collection would contain only one index for each distinct row key in the collection - */ - private IndexCollection distinctFKeyIndexes(IndexCollection indexes, RelNode scanRel) { - IndexCollection distinctIdxCollection = new DrillIndexCollection(scanRel, new HashSet()); - Iterator iterator = indexes.iterator(); - Map> firstColIndexMap = new HashMap<>(); - while (iterator.hasNext()) { - IndexDescriptor index = iterator.next(); - // If index has columns - the first column is the leading column for the index - if (index.getIndexColumns() != null) { - List idxList; - String firstCol = convertLExToStr(index.getIndexColumns().get(0)); - if (firstColIndexMap.get(firstCol) != null) { - idxList = firstColIndexMap.get(firstCol); - } else { - idxList = new ArrayList<>(); - } - idxList.add(index); - firstColIndexMap.put(firstCol, idxList); - } - } - for (String firstCol : firstColIndexMap.keySet()) { - List indexesSameFirstCol = firstColIndexMap.get(firstCol); - double maxAvgRowSize = -1.0; - IndexDescriptor selectedIdx = null; - for (IndexDescriptor idx : indexesSameFirstCol) { - String tabIdxIdentifier = buildUniqueIndexIdentifier(idx); - double idxRowSize = fIStatsCache.get(tabIdxIdentifier).getAvgRowSize(); - // Prefer index with largest average row-size, breaking ties lexicographically - if (idxRowSize > maxAvgRowSize - || (idxRowSize == maxAvgRowSize - && (selectedIdx == null || idx.getIndexName().compareTo(selectedIdx.getIndexName()) < 0))) { - maxAvgRowSize = idxRowSize; - selectedIdx = idx; - } - } - assert (selectedIdx != null); - distinctIdxCollection.addIndex(selectedIdx); - } - return distinctIdxCollection; - } - - /* - * Returns the String representation for the given Logical Expression - */ - private String convertLExToStr(LogicalExpression lex) { - StringBuilder sb = new StringBuilder(); - ExpressionStringBuilder esb = new ExpressionStringBuilder(); - lex.accept(esb, sb); - return sb.toString(); - } - - /* - * Converts the given RexNode condition into a Drill logical expression. - */ - private LogicalExpression convertToLogicalExpression(RexNode condition, - RelDataType type, PlannerSettings settings, RexBuilder builder) { - LogicalExpression conditionExp; - try { - conditionExp = DrillOptiq.toDrill(new DrillParseContext(settings), type, builder, condition); - } catch (ClassCastException e) { - return null; - } - return conditionExp; - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBStatisticsPayload.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBStatisticsPayload.java deleted file mode 100644 index 493028222e2..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/planner/index/MapRDBStatisticsPayload.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.planner.index; - -public class MapRDBStatisticsPayload implements StatisticsPayload { - - double rowCount; - double leadingRowCount; - double avgRowSize; - - public MapRDBStatisticsPayload(double rowCount, double leadingRowCount, double avgRowSize) { - this.rowCount = rowCount; - this.leadingRowCount = leadingRowCount; - this.avgRowSize = avgRowSize; - } - - @Override - public String toString() { - return "MapRDBStatisticsPayload{" + - "rowCount=" + rowCount + - ", leadingRowCount=" + leadingRowCount + - ", avgRowSize=" + avgRowSize + - '}'; - } - - @Override - public double getRowCount() { - return rowCount; - } - - @Override - public double getLeadingRowCount() { - return leadingRowCount; - } - - @Override - public double getAvgRowSize() { - return avgRowSize; - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/PluginConstants.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/PluginConstants.java deleted file mode 100644 index f3558121207..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/PluginConstants.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr; - -import static org.ojai.DocumentConstants.ID_KEY; -import org.apache.drill.common.expression.SchemaPath; -import com.mapr.db.DBConstants; -import org.apache.drill.exec.planner.cost.DrillCostBase; -import org.apache.drill.exec.planner.cost.PluginCost.CheckValid; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public interface PluginConstants { - Logger logger = LoggerFactory.getLogger(PluginConstants.class); - - public final static CheckValid alwaysValid = new CheckValid() { - @Override - public boolean isValid(Integer parameter) { - return true; - } - }; - - public final static CheckValid isNonNegative = new CheckValid() { - @Override - public boolean isValid(Integer paramValue) { - if (paramValue > 0 && paramValue <= Integer.MAX_VALUE) { - return true; - } else { - logger.warn("Setting default value as the supplied parameter value is less than/equals to 0"); - return false; - } - } - }; - - String SSD = "SSD"; - String HDD = "HDD"; - SchemaPath ID_SCHEMA_PATH = SchemaPath.getSimplePath(ID_KEY); - - SchemaPath DOCUMENT_SCHEMA_PATH = SchemaPath.getSimplePath(DBConstants.DOCUMENT_FIELD); - - int JSON_TABLE_NUM_TABLETS_PER_INDEX_DEFAULT = 32; - - int JSON_TABLE_SCAN_SIZE_MB_MIN = 32; - int JSON_TABLE_SCAN_SIZE_MB_MAX = 8192; - - String JSON_TABLE_SCAN_SIZE_MB = "format-maprdb.json.scanSizeMB"; - int JSON_TABLE_SCAN_SIZE_MB_DEFAULT = 128; - - String JSON_TABLE_RESTRICTED_SCAN_SIZE_MB = "format-maprdb.json.restrictedScanSizeMB"; - int JSON_TABLE_RESTRICTED_SCAN_SIZE_MB_DEFAULT = 4096; - - String JSON_TABLE_USE_NUM_REGIONS_FOR_DISTRIBUTION_PLANNING = "format-maprdb.json.useNumRegionsForDistribution"; - boolean JSON_TABLE_USE_NUM_REGIONS_FOR_DISTRIBUTION_PLANNING_DEFAULT = false; - - String JSON_TABLE_BLOCK_SIZE = "format-maprdb.json.pluginCost.blockSize"; - int JSON_TABLE_BLOCK_SIZE_DEFAULT = 8192; - - String JSON_TABLE_MEDIA_TYPE = "format-maprdb.json.mediaType"; - String JSON_TABLE_MEDIA_TYPE_DEFAULT = SSD; - - String JSON_TABLE_SSD_BLOCK_SEQ_READ_COST = "format-maprdb.json.pluginCost.ssdBlockSequentialReadCost"; - int JSON_TABLE_SSD_BLOCK_SEQ_READ_COST_DEFAULT = 32 * DrillCostBase.BASE_CPU_COST * JSON_TABLE_BLOCK_SIZE_DEFAULT; - - // for SSD random and sequential costs are the same - String JSON_TABLE_SSD_BLOCK_RANDOM_READ_COST = "format-maprdb.json.pluginCost.ssdBlockRandomReadCost"; - int JSON_TABLE_SSD_BLOCK_RANDOM_READ_COST_DEFAULT = JSON_TABLE_SSD_BLOCK_SEQ_READ_COST_DEFAULT; - - String JSON_TABLE_AVERGE_COLUMN_SIZE = "format-maprdb.json.pluginCost.averageColumnSize"; - int JSON_TABLE_AVERGE_COLUMN_SIZE_DEFAULT = 10; - - int TABLE_BLOCK_SIZE_DEFAULT = 8192; - int TABLE_SSD_BLOCK_SEQ_READ_COST_DEFAULT = 32 * DrillCostBase.BASE_CPU_COST * TABLE_BLOCK_SIZE_DEFAULT; - int TABLE_SSD_BLOCK_RANDOM_READ_COST_DEFAULT = TABLE_SSD_BLOCK_SEQ_READ_COST_DEFAULT; - int TABLE_AVERGE_COLUMN_SIZE_DEFAULT = 10; - String JSON_TABLE_HDD_BLOCK_SEQ_READ_COST = "format-maprdb.json.pluginCost.hddBlockSequentialReadCost"; - int JSON_TABLE_HDD_BLOCK_SEQ_READ_COST_DEFAULT = 6 * JSON_TABLE_SSD_BLOCK_SEQ_READ_COST_DEFAULT; - - String JSON_TABLE_HDD_BLOCK_RANDOM_READ_COST = "format-maprdb.json.pluginCost.hddBlockRandomReadCost"; - int JSON_TABLE_HDD_BLOCK_RANDOM_READ_COST_DEFAULT = 1000 * JSON_TABLE_HDD_BLOCK_SEQ_READ_COST_DEFAULT; -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/PluginErrorHandler.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/PluginErrorHandler.java deleted file mode 100644 index 5163934060c..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/PluginErrorHandler.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr; - -import org.apache.drill.common.exceptions.UserException; -import org.apache.drill.exec.exception.SchemaChangeException; -import org.slf4j.Logger; - -public final class PluginErrorHandler { - - public static UserException unsupportedError(Logger logger, String format, Object... args) { - return UserException.unsupportedError() - .message(String.format(format, args)) - .build(logger); - } - - public static UserException dataReadError(Logger logger, Throwable t) { - return dataReadError(logger, t, null); - } - - public static UserException dataReadError(Logger logger, String format, Object... args) { - return dataReadError(null, format, args); - } - - public static UserException dataReadError(Logger logger, Throwable t, String format, Object... args) { - return UserException.dataReadError(t) - .message(format == null ? null : String.format(format, args)) - .build(logger); - } - - public static SchemaChangeException schemaChangeException(Logger logger, Throwable t, String format, Object... args) { - return new SchemaChangeException(format, t, args); - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatMatcher.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatMatcher.java deleted file mode 100644 index b2ede050f29..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatMatcher.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr; - -import java.io.IOException; - -import org.apache.drill.exec.planner.logical.DrillTable; -import org.apache.drill.exec.planner.logical.DynamicDrillTable; -import org.apache.drill.exec.store.SchemaConfig; -import org.apache.drill.exec.store.dfs.DrillFileSystem; -import org.apache.drill.exec.store.dfs.FileSelection; -import org.apache.drill.exec.store.dfs.FileSystemPlugin; -import org.apache.drill.exec.store.dfs.FormatMatcher; -import org.apache.drill.exec.store.dfs.FormatSelection; -import org.apache.hadoop.fs.FileStatus; - -import com.mapr.fs.MapRFileStatus; - -public abstract class TableFormatMatcher extends FormatMatcher { - - private final TableFormatPlugin plugin; - - public TableFormatMatcher(TableFormatPlugin plugin) { - this.plugin = plugin; - } - - @Override - public boolean supportDirectoryReads() { - return false; - } - - @Override - public DrillTable isReadable(DrillFileSystem fs, - FileSelection selection, FileSystemPlugin fsPlugin, - String storageEngineName, SchemaConfig schemaConfig) throws IOException { - FileStatus status = selection.getFirstPath(fs); - if (!isFileReadable(fs, status)) { - return null; - } - - return new DynamicDrillTable(fsPlugin, storageEngineName, schemaConfig.getUserName(), - new FormatSelection(getFormatPlugin().getConfig(), selection)); - } - - @Override - public boolean isFileReadable(DrillFileSystem fs, FileStatus status) throws IOException { - return (status instanceof MapRFileStatus) - && ((MapRFileStatus) status).isTable() - && isSupportedTable((MapRFileStatus) status); - } - - @Override - public TableFormatPlugin getFormatPlugin() { - return plugin; - } - - /** - * Returns true if the path pointed by the MapRFileStatus is a supported table - * by this format plugin. The path must point to a MapR table. - */ - protected abstract boolean isSupportedTable(MapRFileStatus status) throws IOException; -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPlugin.java deleted file mode 100644 index 24440196aab..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPlugin.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr; - -import static com.mapr.fs.jni.MapRConstants.MAPRFS_PREFIX; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.List; -import java.util.Set; - -import org.apache.drill.common.logical.FormatPluginConfig; -import org.apache.drill.common.logical.StoragePluginConfig; -import org.apache.drill.exec.physical.base.AbstractWriter; -import org.apache.drill.exec.physical.base.PhysicalOperator; -import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.store.AbstractStoragePlugin; -import org.apache.drill.exec.store.StoragePluginOptimizerRule; -import org.apache.drill.exec.store.dfs.FormatPlugin; -import org.apache.hadoop.conf.Configuration; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.collect.ImmutableSet; -import com.mapr.fs.MapRFileSystem; - -public abstract class TableFormatPlugin implements FormatPlugin { - - private final StoragePluginConfig storageConfig; - private final TableFormatPluginConfig config; - private final Configuration fsConf; - private final DrillbitContext context; - private final String name; - - private volatile AbstractStoragePlugin storagePlugin; - private final MapRFileSystem maprfs; - - protected TableFormatPlugin(String name, DrillbitContext context, Configuration fsConf, - StoragePluginConfig storageConfig, TableFormatPluginConfig formatConfig) { - this.context = context; - this.config = formatConfig; - this.storageConfig = storageConfig; - this.fsConf = fsConf; - this.name = name == null ? "maprdb" : name; - try { - this.maprfs = new MapRFileSystem(); - getMaprFS().initialize(new URI(MAPRFS_PREFIX), fsConf); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException(e); - } - } - - @Override - public boolean supportsRead() { - return true; - } - - @Override - public boolean supportsWrite() { - return false; - } - - @Override - public boolean supportsAutoPartitioning() { - return false; - } - - @Override - public Configuration getFsConf() { - return fsConf; - } - - @Override - public Set getOptimizerRules() { - return ImmutableSet.of(); - } - - @Override - public AbstractWriter getWriter(PhysicalOperator child, String location, - List partitionColumns) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public FormatPluginConfig getConfig() { - return config; - } - - @Override - public StoragePluginConfig getStorageConfig() { - return storageConfig; - } - - @Override - public DrillbitContext getContext() { - return context; - } - - @Override - public String getName() { - return name; - } - - public synchronized AbstractStoragePlugin getStoragePlugin() { - if (this.storagePlugin == null) { - this.storagePlugin = context.getStorage().resolve(storageConfig, - AbstractStoragePlugin.class); - } - return storagePlugin; - } - - @JsonIgnore - public MapRFileSystem getMaprFS() { - return maprfs; - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPluginConfig.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPluginConfig.java deleted file mode 100644 index ffd3d921509..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/TableFormatPluginConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr; - -import org.apache.drill.common.logical.FormatPluginConfig; - -public abstract class TableFormatPluginConfig implements FormatPluginConfig { - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } else if (obj == null || getClass() != obj.getClass()) { - return false; - } - return impEquals(obj); - } - - protected abstract boolean impEquals(Object obj); -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBCost.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBCost.java deleted file mode 100644 index 0c40a0286f5..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBCost.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.exec.physical.base.GroupScan; -import org.apache.drill.exec.planner.cost.PluginCost; -import org.apache.drill.exec.store.mapr.PluginConstants; -import org.apache.drill.exec.store.mapr.db.json.JsonTableGroupScan; - -public class MapRDBCost implements PluginCost { - - private int JSON_AVG_COLUMN_SIZE; - private int JSON_TABLE_BLOCK_SIZE; // bytes per block - private int JSON_BLOCK_SEQ_READ_COST; - private int JSON_BLOCK_RANDOM_READ_COST; - private int JSON_HDD_BLOCK_SEQ_READ_COST; - private int JSON_HDD_BLOCK_RANDOM_READ_COST; - private int JSON_SSD_BLOCK_SEQ_READ_COST; - private int JSON_SSD_BLOCK_RANDOM_READ_COST; - - public MapRDBCost(DrillConfig config, String mediaType) { - JSON_AVG_COLUMN_SIZE = setConfigValue(config, PluginConstants.JSON_TABLE_AVERGE_COLUMN_SIZE, - PluginConstants.JSON_TABLE_AVERGE_COLUMN_SIZE_DEFAULT, PluginConstants.alwaysValid); - JSON_TABLE_BLOCK_SIZE = setConfigValue(config, PluginConstants.JSON_TABLE_BLOCK_SIZE, - PluginConstants.JSON_TABLE_BLOCK_SIZE_DEFAULT, PluginConstants.alwaysValid); - JSON_SSD_BLOCK_SEQ_READ_COST = setConfigValue(config, PluginConstants.JSON_TABLE_SSD_BLOCK_SEQ_READ_COST, - PluginConstants.JSON_TABLE_SSD_BLOCK_SEQ_READ_COST_DEFAULT, PluginConstants.isNonNegative); - JSON_SSD_BLOCK_RANDOM_READ_COST = setConfigValue(config, PluginConstants.JSON_TABLE_SSD_BLOCK_RANDOM_READ_COST, - PluginConstants.JSON_TABLE_SSD_BLOCK_RANDOM_READ_COST_DEFAULT, new greaterThanEquals(JSON_SSD_BLOCK_SEQ_READ_COST)); - JSON_HDD_BLOCK_SEQ_READ_COST = setConfigValue(config, PluginConstants.JSON_TABLE_HDD_BLOCK_SEQ_READ_COST, - PluginConstants.JSON_TABLE_HDD_BLOCK_SEQ_READ_COST_DEFAULT, PluginConstants.isNonNegative); - JSON_HDD_BLOCK_RANDOM_READ_COST = setConfigValue(config, PluginConstants.JSON_TABLE_HDD_BLOCK_RANDOM_READ_COST, - PluginConstants.JSON_TABLE_HDD_BLOCK_RANDOM_READ_COST_DEFAULT, new greaterThanEquals(JSON_HDD_BLOCK_SEQ_READ_COST)); - JSON_BLOCK_SEQ_READ_COST = mediaType.equals(PluginConstants.SSD) ? JSON_SSD_BLOCK_SEQ_READ_COST : - JSON_HDD_BLOCK_SEQ_READ_COST; - JSON_BLOCK_RANDOM_READ_COST = mediaType.equals(PluginConstants.SSD) ? JSON_SSD_BLOCK_RANDOM_READ_COST : - JSON_HDD_BLOCK_RANDOM_READ_COST; - } - - private int setConfigValue(DrillConfig config, String configPath, - int defaultValue, CheckValid check) { - int configValue; - try { - configValue = config.getInt(configPath); - if (!check.isValid(configValue)) { configValue = defaultValue; } - } catch (Exception ex) { - // Use defaults, if config values not present or any other issue - configValue = defaultValue; - } - return configValue; - } - - @Override - public int getAverageColumnSize(GroupScan scan) { - if (scan instanceof JsonTableGroupScan) { - return JSON_AVG_COLUMN_SIZE; - } else { - return PluginConstants.TABLE_AVERGE_COLUMN_SIZE_DEFAULT; - } - } - - @Override - public int getBlockSize(GroupScan scan) { - if (scan instanceof JsonTableGroupScan) { - return JSON_TABLE_BLOCK_SIZE; - } else { - return PluginConstants.TABLE_BLOCK_SIZE_DEFAULT; - } - } - - @Override - public int getSequentialBlockReadCost(GroupScan scan) { - if (scan instanceof JsonTableGroupScan) { - return JSON_BLOCK_SEQ_READ_COST; - } else { - return PluginConstants.TABLE_SSD_BLOCK_SEQ_READ_COST_DEFAULT; - } - } - - @Override - public int getRandomBlockReadCost(GroupScan scan) { - if (scan instanceof JsonTableGroupScan) { - return JSON_BLOCK_RANDOM_READ_COST; - } else { - return PluginConstants.TABLE_SSD_BLOCK_RANDOM_READ_COST_DEFAULT; - } - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatMatcher.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatMatcher.java deleted file mode 100644 index d4c08f36e5c..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatMatcher.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import java.io.IOException; - -import com.mapr.fs.MapRFileStatus; -import com.mapr.db.index.IndexDesc; -import com.mapr.fs.tables.TableProperties; -import org.apache.drill.exec.planner.index.IndexDescriptor; -import org.apache.drill.exec.planner.index.MapRDBIndexDescriptor; -import org.apache.drill.exec.planner.logical.DrillTable; -import org.apache.drill.exec.planner.logical.DynamicDrillTable; -import org.apache.drill.exec.store.SchemaConfig; -import org.apache.drill.exec.store.dfs.DrillFileSystem; -import org.apache.drill.exec.store.dfs.FileSelection; -import org.apache.drill.exec.store.dfs.FileSystemPlugin; -import org.apache.drill.exec.store.dfs.FormatSelection; - -import org.apache.drill.exec.store.mapr.TableFormatMatcher; -import org.apache.drill.exec.store.mapr.TableFormatPlugin; - -import org.apache.drill.exec.store.mapr.db.binary.MapRDBBinaryTable; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; - -public class MapRDBFormatMatcher extends TableFormatMatcher { - - public MapRDBFormatMatcher(TableFormatPlugin plugin) { - super(plugin); - } - - @Override - protected boolean isSupportedTable(MapRFileStatus status) throws IOException { - return !getFormatPlugin() - .getMaprFS() - .getTableBasicAttrs(status.getPath()) - .getIsMarlinTable(); - } - - - /** - * Get an instance of DrillTable for a particular native secondary index - * @param fs - * @param selection - * @param fsPlugin - * @param storageEngineName - * @param userName - * @param secondaryIndexDesc - * @return - * @throws IOException - */ - public DrillTable isReadableIndex(DrillFileSystem fs, - FileSelection selection, FileSystemPlugin fsPlugin, - String storageEngineName, String userName, - IndexDescriptor secondaryIndexDesc) throws IOException { - FileStatus status = selection.getFirstPath(fs); - - if (!isFileReadable(fs, status)) { - return null; - } - - MapRDBFormatPlugin fp = (MapRDBFormatPlugin) getFormatPlugin(); - DrillTable dt = new DynamicDrillTable(fsPlugin, - storageEngineName, - userName, - new FormatSelection(fp.getConfig(), - selection)); - - // TODO: Create groupScan using index descriptor - dt.setGroupScan(fp.getGroupScan(userName, - selection, - null /* columns */, - (IndexDesc) ((MapRDBIndexDescriptor) secondaryIndexDesc).getOriginalDesc(), - null /* metadataProviderManager */)); - - return dt; - } - - @Override - public DrillTable isReadable(DrillFileSystem fs, - FileSelection selection, FileSystemPlugin fsPlugin, - String storageEngineName, SchemaConfig schemaConfig) throws IOException { - if (isFileReadable(fs, selection.getFirstPath(fs))) { - MapRDBFormatPlugin mapRDBFormatPlugin = (MapRDBFormatPlugin) getFormatPlugin(); - String tableName = mapRDBFormatPlugin.getTableName(selection); - TableProperties props = mapRDBFormatPlugin.getMaprFS().getTableProperties(new Path(tableName)); - if (props.getAttr().getJson()) { - return new DynamicDrillTable( - fsPlugin, - storageEngineName, - schemaConfig.getQueryUserCredentials().getUserName(), - new FormatSelection(mapRDBFormatPlugin.getConfig(), selection) - ); - } else { - FormatSelection formatSelection = new FormatSelection(mapRDBFormatPlugin.getConfig(), selection); - return new MapRDBBinaryTable(storageEngineName, fsPlugin, mapRDBFormatPlugin, formatSelection); - } - } - return null; - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPlugin.java deleted file mode 100644 index 516f15724f1..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPlugin.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import java.io.IOException; -import java.util.List; -import java.util.Set; - -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.logical.StoragePluginConfig; -import org.apache.drill.exec.physical.base.AbstractGroupScan; -import org.apache.drill.exec.planner.common.DrillStatsTable.TableStatistics; -import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.store.StoragePluginOptimizerRule; -import org.apache.drill.exec.store.dfs.FileSelection; -import org.apache.drill.exec.store.dfs.FormatMatcher; -import org.apache.drill.exec.store.hbase.HBaseScanSpec; -import org.apache.drill.exec.store.mapr.PluginConstants; -import org.apache.drill.exec.store.mapr.TableFormatPlugin; -import org.apache.drill.exec.store.mapr.db.binary.BinaryTableGroupScan; -import org.apache.drill.exec.store.mapr.db.json.JsonScanSpec; -import org.apache.drill.exec.store.mapr.db.json.JsonTableGroupScan; -import org.apache.drill.exec.metastore.MetadataProviderManager; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.collect.ImmutableSet; -import com.mapr.db.index.IndexDesc; -import com.mapr.fs.tables.TableProperties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MapRDBFormatPlugin extends TableFormatPlugin { - private static final Logger logger = LoggerFactory.getLogger(MapRDBFormatPlugin.class); - - private final MapRDBFormatMatcher matcher; - private final Configuration hbaseConf; - private final Connection connection; - private final MapRDBTableCache jsonTableCache; - private final int scanRangeSizeMB; - private final String mediaType; - private final MapRDBCost pluginCostModel; - private final int restrictedScanRangeSizeMB; - - public MapRDBFormatPlugin(String name, DrillbitContext context, Configuration fsConf, - StoragePluginConfig storageConfig, MapRDBFormatPluginConfig formatConfig) throws IOException { - super(name, context, fsConf, storageConfig, formatConfig); - matcher = new MapRDBFormatMatcher(this); - hbaseConf = HBaseConfiguration.create(fsConf); - hbaseConf.set(ConnectionFactory.DEFAULT_DB, ConnectionFactory.MAPR_ENGINE2); - connection = ConnectionFactory.createConnection(hbaseConf); - jsonTableCache = new MapRDBTableCache(context.getConfig()); - int scanRangeSizeMBConfig = context.getConfig().getInt(PluginConstants.JSON_TABLE_SCAN_SIZE_MB); - if (scanRangeSizeMBConfig < PluginConstants.JSON_TABLE_SCAN_SIZE_MB_MIN || - scanRangeSizeMBConfig > PluginConstants.JSON_TABLE_SCAN_SIZE_MB_MAX) { - logger.warn("Invalid scan size {} for MapR-DB tables, using default", scanRangeSizeMBConfig); - scanRangeSizeMBConfig = PluginConstants.JSON_TABLE_SCAN_SIZE_MB_DEFAULT; - } - - int restrictedScanRangeSizeMBConfig = context.getConfig().getInt(PluginConstants.JSON_TABLE_RESTRICTED_SCAN_SIZE_MB); - if (restrictedScanRangeSizeMBConfig < PluginConstants.JSON_TABLE_SCAN_SIZE_MB_MIN || - restrictedScanRangeSizeMBConfig > PluginConstants.JSON_TABLE_SCAN_SIZE_MB_MAX) { - logger.warn("Invalid restricted scan size {} for MapR-DB tables, using default", restrictedScanRangeSizeMBConfig); - restrictedScanRangeSizeMBConfig = PluginConstants.JSON_TABLE_RESTRICTED_SCAN_SIZE_MB_DEFAULT; - } - - String mediaTypeConfig = context.getConfig().getString(PluginConstants.JSON_TABLE_MEDIA_TYPE); - if (!(mediaTypeConfig.equals(PluginConstants.SSD) || - mediaTypeConfig.equals(PluginConstants.HDD))) { - logger.warn("Invalid media Type {} for MapR-DB JSON tables, using default 'SSD'", mediaTypeConfig); - mediaTypeConfig = PluginConstants.JSON_TABLE_MEDIA_TYPE_DEFAULT; - } - mediaType = mediaTypeConfig; - scanRangeSizeMB = scanRangeSizeMBConfig; - restrictedScanRangeSizeMB = restrictedScanRangeSizeMBConfig; - pluginCostModel = new MapRDBCost(context.getConfig(), mediaType); - } - - @Override - public FormatMatcher getMatcher() { - return matcher; - } - - @Override - public MapRDBFormatPluginConfig getConfig() { - return (MapRDBFormatPluginConfig)(super.getConfig()); - } - - public MapRDBTableCache getJsonTableCache() { - return jsonTableCache; - } - - @Override - @JsonIgnore - public Set getOptimizerRules() { - return ImmutableSet.of(MapRDBPushFilterIntoScan.FILTER_ON_SCAN, MapRDBPushFilterIntoScan.FILTER_ON_PROJECT, - MapRDBPushProjectIntoScan.PROJECT_ON_SCAN, MapRDBPushLimitIntoScan.LIMIT_ON_PROJECT, - MapRDBPushLimitIntoScan.LIMIT_ON_SCAN, MapRDBPushLimitIntoScan.LIMIT_ON_RKJOIN); - } - - public AbstractGroupScan getGroupScan(String userName, FileSelection selection, - List columns, IndexDesc indexDesc, MetadataProviderManager metadataProviderManager) throws IOException { - String tableName = getTableName(selection); - TableProperties props = getMaprFS().getTableProperties(new Path(tableName)); - - if (props.getAttr().getJson()) { - JsonScanSpec scanSpec = new JsonScanSpec(tableName, indexDesc, null/*condition*/); - return new JsonTableGroupScan(userName, getStoragePlugin(), this, scanSpec, columns, metadataProviderManager); - } else { - HBaseScanSpec scanSpec = new HBaseScanSpec(tableName); - return new BinaryTableGroupScan(userName, getStoragePlugin(), this, scanSpec, columns, metadataProviderManager); - } - } - - @Override - public AbstractGroupScan getGroupScan(String userName, FileSelection selection, - List columns) throws IOException { - return getGroupScan(userName, selection, columns, (IndexDesc) null /* indexDesc */, null /* metadataProviderManager */); - } - - public boolean supportsStatistics() { - return false; - } - - @Override - public TableStatistics readStatistics(FileSystem fs, Path statsTablePath) { - throw new UnsupportedOperationException("unimplemented"); - } - - @Override - public void writeStatistics(TableStatistics statistics, FileSystem fs, Path statsTablePath) throws IOException { - throw new UnsupportedOperationException("unimplemented"); - } - - @JsonIgnore - public Configuration getHBaseConf() { - return hbaseConf; - } - - @JsonIgnore - public Connection getConnection() { - return connection; - } - - public int getScanRangeSizeMB() { - return scanRangeSizeMB; - } - - public int getRestrictedScanRangeSizeMB() { - return restrictedScanRangeSizeMB; - } - - public MapRDBCost getPluginCostModel() { - return pluginCostModel; - } - - /** - * Allows to get a table name from FileSelection object - * - * @param selection File selection object - * @return string table name - */ - @JsonIgnore - public String getTableName(FileSelection selection) { - List files = selection.getFiles(); - assert (files.size() == 1); - return files.get(0).toUri().getPath(); - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java deleted file mode 100644 index 9d0605ff843..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBFormatPluginConfig.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import org.apache.drill.exec.store.mapr.TableFormatPluginConfig; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonTypeName; - -@JsonTypeName("maprdb") -@JsonInclude(Include.NON_DEFAULT) -public class MapRDBFormatPluginConfig extends TableFormatPluginConfig { - - public boolean allTextMode = false; - public boolean enablePushdown = true; - public boolean ignoreSchemaChange = false; - public boolean readAllNumbersAsDouble = false; - public boolean disableCountOptimization = false; - public boolean readTimestampWithZoneOffset = false; - /* This flag is a switch to do special handling in case of - * no columns in the query exists in the maprdb table. This flag - * can get deprecated once it is observed that this special handling - * is not regressing performance of reading maprdb table. - */ - public boolean nonExistentFieldSupport = true; - public String index = ""; - - @Override - public int hashCode() { - int result = (allTextMode ? 1231 : 1237); - result = 31 * result + (enablePushdown ? 1231 : 1237); - result = 31 * result + (ignoreSchemaChange ? 1231 : 1237); - result = 31 * result + (readAllNumbersAsDouble ? 1231 : 1237); - result = 31 * result + (disableCountOptimization ? 1231 : 1237); - result = 31 * result + (nonExistentFieldSupport ? 1231 : 1237); - result = 31 * result + (readTimestampWithZoneOffset ? 1231 : 1237); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } else if (obj == null || getClass() != obj.getClass()) { - return false; - } - - MapRDBFormatPluginConfig other = (MapRDBFormatPluginConfig) obj; - if (readAllNumbersAsDouble != other.readAllNumbersAsDouble) { - return false; - } else if (allTextMode != other.allTextMode) { - return false; - } else if (ignoreSchemaChange != other.ignoreSchemaChange) { - return false; - } else if (enablePushdown != other.enablePushdown) { - return false; - } else if (disableCountOptimization != other.disableCountOptimization) { - return false; - } else if (nonExistentFieldSupport != other.nonExistentFieldSupport) { - return false; - } else if (!index.equals(other.index)) { - return false; - } else if (readTimestampWithZoneOffset != other.readTimestampWithZoneOffset) { - return false; - } - return true; - } - - - @Override - protected boolean impEquals(Object obj) { - MapRDBFormatPluginConfig other = (MapRDBFormatPluginConfig) obj; - if (readAllNumbersAsDouble != other.readAllNumbersAsDouble) { - return false; - } else if (allTextMode != other.allTextMode) { - return false; - } else if (ignoreSchemaChange != other.ignoreSchemaChange) { - return false; - } else if (enablePushdown != other.enablePushdown) { - return false; - } else if (disableCountOptimization != other.disableCountOptimization) { - return false; - } else if (nonExistentFieldSupport != other.nonExistentFieldSupport) { - return false; - } else if (!index.equals(other.index)) { - return false; - } else if (readTimestampWithZoneOffset != other.readTimestampWithZoneOffset) { - return false; - } - return true; - } - - public boolean isReadAllNumbersAsDouble() { - return readAllNumbersAsDouble; - } - - public boolean isReadTimestampWithZoneOffset() { - return readTimestampWithZoneOffset; - } - - public boolean isAllTextMode() { - return allTextMode; - } - - public boolean disableCountOptimization() { - return disableCountOptimization; - } - - public boolean isEnablePushdown() { - return enablePushdown; - } - - public boolean isNonExistentFieldSupport() { return nonExistentFieldSupport; } - - public boolean isIgnoreSchemaChange() { - return ignoreSchemaChange; - } - - public String getIndex() { return this.index; } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBGroupScan.java deleted file mode 100644 index 18a5968415f..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBGroupScan.java +++ /dev/null @@ -1,372 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.NavigableMap; -import java.util.PriorityQueue; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.logical.StoragePluginConfig; -import org.apache.drill.exec.physical.EndpointAffinity; -import org.apache.drill.exec.physical.base.AbstractDbGroupScan; -import org.apache.calcite.rel.RelNode; -import org.apache.drill.exec.planner.index.IndexCollection; - -import org.apache.drill.exec.planner.cost.PluginCost; -import org.apache.drill.exec.planner.index.IndexDiscover; -import org.apache.drill.exec.planner.index.IndexDiscoverFactory; -import org.apache.drill.exec.planner.index.MapRDBIndexDiscover; -import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; -import org.apache.drill.exec.record.metadata.TupleMetadata; -import org.apache.drill.exec.store.AbstractStoragePlugin; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.drill.metastore.metadata.TableMetadata; -import org.apache.drill.metastore.metadata.TableMetadataProvider; -import com.google.common.base.Preconditions; -import com.google.common.base.Stopwatch; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - -public abstract class MapRDBGroupScan extends AbstractDbGroupScan { - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBGroupScan.class); - - private static final Comparator> LIST_SIZE_COMPARATOR = Comparator.comparingInt(List::size); - private static final Comparator> LIST_SIZE_COMPARATOR_REV = Collections.reverseOrder(LIST_SIZE_COMPARATOR); - - protected AbstractStoragePlugin storagePlugin; - - protected MapRDBFormatPlugin formatPlugin; - - protected MapRDBFormatPluginConfig formatPluginConfig; - - protected List columns; - - protected Map> endpointFragmentMapping; - - protected NavigableMap doNotAccessRegionsToScan; - - protected double costFactor = 1.0; - - private boolean filterPushedDown = false; - - private Stopwatch watch = Stopwatch.createUnstarted(); - - private TableMetadataProvider metadataProvider; - - private TableMetadata tableMetadata; - - public MapRDBGroupScan(MapRDBGroupScan that) { - super(that); - this.columns = that.columns; - this.formatPlugin = that.formatPlugin; - this.formatPluginConfig = that.formatPluginConfig; - this.storagePlugin = that.storagePlugin; - this.filterPushedDown = that.filterPushedDown; - this.costFactor = that.costFactor; - /* this is the only place we access the field `doNotAccessRegionsToScan` directly - * because we do not want the sub-scan spec for JSON tables to be calculated - * during the copy-constructor - */ - this.doNotAccessRegionsToScan = that.doNotAccessRegionsToScan; - this.metadataProvider = that.metadataProvider; - } - - public MapRDBGroupScan(AbstractStoragePlugin storagePlugin, - MapRDBFormatPlugin formatPlugin, List columns, String userName, - TableMetadataProvider metadataProvider) { - super(userName); - this.storagePlugin = storagePlugin; - this.formatPlugin = formatPlugin; - this.formatPluginConfig = formatPlugin.getConfig(); - this.columns = columns; - this.metadataProvider = metadataProvider; - } - - @Override - public List getOperatorAffinity() { - watch.reset(); - watch.start(); - Map endpointMap = new HashMap<>(); - for (DrillbitEndpoint ep : formatPlugin.getContext().getBits()) { - endpointMap.put(ep.getAddress(), ep); - } - - final Map affinityMap = new HashMap<>(); - for (String serverName : getRegionsToScan().values()) { - DrillbitEndpoint ep = endpointMap.get(serverName); - if (ep != null) { - EndpointAffinity affinity = affinityMap.get(ep); - if (affinity == null) { - affinityMap.put(ep, new EndpointAffinity(ep, 1)); - } else { - affinity.addAffinity(1); - } - } - } - logger.debug("Took {} µs to get operator affinity", watch.elapsed(TimeUnit.NANOSECONDS)/1000); - return Lists.newArrayList(affinityMap.values()); - } - - /** - * - * @param incomingEndpoints - */ - @Override - public void applyAssignments(List incomingEndpoints) { - watch.reset(); - watch.start(); - - final NavigableMap regionsToScan = getRegionsToScan(); - final int numSlots = incomingEndpoints.size(); - Preconditions.checkArgument(numSlots <= regionsToScan.size(), - String.format("Incoming endpoints %d is greater than number of scan regions %d", numSlots, regionsToScan.size())); - - /* - * Minimum/Maximum number of assignment per slot - */ - final int minPerEndpointSlot = (int) Math.floor((double)regionsToScan.size() / numSlots); - final int maxPerEndpointSlot = (int) Math.ceil((double)regionsToScan.size() / numSlots); - - /* - * initialize (endpoint index => HBaseSubScanSpec list) map - */ - endpointFragmentMapping = Maps.newHashMapWithExpectedSize(numSlots); - - /* - * another map with endpoint (hostname => corresponding index list) in 'incomingEndpoints' list - */ - Map> endpointHostIndexListMap = Maps.newHashMap(); - - /* - * Initialize these two maps - */ - for (int i = 0; i < numSlots; ++i) { - endpointFragmentMapping.put(i, new ArrayList(maxPerEndpointSlot)); - String hostname = incomingEndpoints.get(i).getAddress(); - Queue hostIndexQueue = endpointHostIndexListMap.get(hostname); - if (hostIndexQueue == null) { - hostIndexQueue = Lists.newLinkedList(); - endpointHostIndexListMap.put(hostname, hostIndexQueue); - } - hostIndexQueue.add(i); - } - - Set> regionsToAssignSet = Sets.newLinkedHashSet(regionsToScan.entrySet()); - - /* - * First, we assign regions which are hosted on region servers running on drillbit endpoints - */ - for (Iterator> regionsIterator = regionsToAssignSet.iterator(); regionsIterator.hasNext(); /*nothing*/) { - Entry regionEntry = regionsIterator.next(); - /* - * Test if there is a drillbit endpoint which is also an HBase RegionServer that hosts the current HBase region - */ - Queue endpointIndexlist = endpointHostIndexListMap.get(regionEntry.getValue()); - if (endpointIndexlist != null) { - Integer slotIndex = endpointIndexlist.poll(); - List endpointSlotScanList = endpointFragmentMapping.get(slotIndex); - MapRDBSubScanSpec subScanSpec = getSubScanSpec(regionEntry.getKey()); - endpointSlotScanList.add(subScanSpec); - // add to the tail of the slot list, to add more later in round robin fashion - endpointIndexlist.offer(slotIndex); - // this region has been assigned - regionsIterator.remove(); - } - } - - /* - * Build priority queues of slots, with ones which has tasks lesser than 'minPerEndpointSlot' and another which have more. - */ - PriorityQueue> minHeap = new PriorityQueue>(numSlots, LIST_SIZE_COMPARATOR); - PriorityQueue> maxHeap = new PriorityQueue>(numSlots, LIST_SIZE_COMPARATOR_REV); - for(List listOfScan : endpointFragmentMapping.values()) { - if (listOfScan.size() <= minPerEndpointSlot) { - minHeap.offer(listOfScan); - } else if (listOfScan.size() > minPerEndpointSlot){ - maxHeap.offer(listOfScan); - } - } - - /* - * Now, let's process any regions which remain unassigned and assign them to slots with minimum number of assignments. - */ - if (regionsToAssignSet.size() > 0) { - for (Entry regionEntry : regionsToAssignSet) { - List smallestList = minHeap.poll(); - MapRDBSubScanSpec subScanSpec = getSubScanSpec(regionEntry.getKey()); - smallestList.add(subScanSpec); - if (smallestList.size() < maxPerEndpointSlot) { - minHeap.offer(smallestList); - } - } - } - - /* - * While there are slots with lesser than 'minPerEndpointSlot' unit work, balance from those with more. - */ - while(minHeap.peek() != null && minHeap.peek().size() < minPerEndpointSlot) { - List smallestList = minHeap.poll(); - List largestList = maxHeap.poll(); - smallestList.add(largestList.remove(largestList.size() - 1)); - if (largestList.size() > minPerEndpointSlot) { - maxHeap.offer(largestList); - } - if (smallestList.size() < minPerEndpointSlot) { - minHeap.offer(smallestList); - } - } - - for (Entry> endpoint : endpointFragmentMapping.entrySet()) { - Collections.sort(endpoint.getValue()); - } - - /* no slot should be empty at this point */ - assert (minHeap.peek() == null || minHeap.peek().size() > 0) : String.format( - "Unable to assign tasks to some endpoints.\nEndpoints: %s.\nAssignment Map: %s.", - incomingEndpoints, endpointFragmentMapping.toString()); - - logger.debug("Built assignment map in {} µs.\nEndpoints: {}.\nAssignment Map: {}", - watch.elapsed(TimeUnit.NANOSECONDS)/1000, incomingEndpoints, endpointFragmentMapping.toString()); - } - - @Override - public int getMaxParallelizationWidth() { - return getRegionsToScan().size(); - } - - @JsonIgnore - public MapRDBFormatPlugin getFormatPlugin() { - return formatPlugin; - } - - @Override - public String getDigest() { - return toString(); - } - - @JsonProperty("storage") - public StoragePluginConfig getStorageConfig() { - return storagePlugin.getConfig(); - } - - @JsonIgnore - public AbstractStoragePlugin getStoragePlugin(){ - return storagePlugin; - } - - @Override - @JsonProperty - public List getColumns() { - return columns; - } - - @JsonIgnore - public boolean canPushdownProjects(List columns) { - return true; - } - - @JsonIgnore - public void setFilterPushedDown(boolean b) { - this.filterPushedDown = true; - } - - public String getIndexHint() { return this.formatPluginConfig.getIndex(); } - - @JsonIgnore - @Override - public boolean isFilterPushedDown() { - return filterPushedDown; - } - - protected abstract MapRDBSubScanSpec getSubScanSpec(TabletFragmentInfo key); - - public void setCostFactor(double sel) { - this.costFactor = sel; - } - - @Override - public IndexCollection getSecondaryIndexCollection(RelNode scanRel) { - IndexDiscover discover = IndexDiscoverFactory.getIndexDiscover( - getStorageConfig(), this, scanRel, MapRDBIndexDiscover.class); - - if (discover == null) { - logger.error("Null IndexDiscover was found for {}!", scanRel); - } - return discover.getTableIndex(getTableName()); - } - - @JsonIgnore - public abstract String getTableName(); - - @JsonIgnore - public int getRowKeyOrdinal() { - return 0; - } - - protected NavigableMap getRegionsToScan() { - return doNotAccessRegionsToScan; - } - - protected void resetRegionsToScan() { - this.doNotAccessRegionsToScan = null; - } - - protected void setRegionsToScan(NavigableMap regionsToScan) { - this.doNotAccessRegionsToScan = regionsToScan; - } - - @Override - public PluginCost getPluginCostModel() { - return formatPlugin.getPluginCostModel(); - } - - @JsonProperty - public TupleMetadata getSchema() { - return getTableMetadata().getSchema(); - } - - @Override - @JsonIgnore - public TableMetadataProvider getMetadataProvider() { - return metadataProvider; - } - - @Override - @JsonIgnore - public TableMetadata getTableMetadata() { - if (tableMetadata == null) { - tableMetadata = metadataProvider.getTableMetadata(); - } - return tableMetadata; - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushFilterIntoScan.java deleted file mode 100644 index 69f0ab1d317..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushFilterIntoScan.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptRuleOperand; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rex.RexNode; -import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.exec.planner.logical.DrillOptiq; -import org.apache.drill.exec.planner.logical.DrillParseContext; -import org.apache.drill.exec.planner.logical.RelOptHelper; -import org.apache.drill.exec.planner.physical.FilterPrel; -import org.apache.drill.exec.planner.physical.PrelUtil; -import org.apache.drill.exec.planner.physical.ProjectPrel; -import org.apache.drill.exec.planner.physical.ScanPrel; -import org.apache.drill.exec.store.StoragePluginOptimizerRule; -import org.apache.drill.exec.store.hbase.HBaseScanSpec; -import org.apache.drill.exec.store.mapr.db.binary.BinaryTableGroupScan; -import org.apache.drill.exec.store.mapr.db.binary.MapRDBFilterBuilder; -import org.apache.drill.exec.store.mapr.db.json.JsonConditionBuilder; -import org.apache.drill.exec.store.mapr.db.json.JsonScanSpec; -import org.apache.drill.exec.store.mapr.db.json.JsonTableGroupScan; - -import com.google.common.collect.ImmutableList; - -public abstract class MapRDBPushFilterIntoScan extends StoragePluginOptimizerRule { - - private MapRDBPushFilterIntoScan(RelOptRuleOperand operand, String description) { - super(operand, description); - } - - public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new MapRDBPushFilterIntoScan(RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), "MapRDBPushFilterIntoScan:Filter_On_Scan") { - - @Override - public void onMatch(RelOptRuleCall call) { - final FilterPrel filter = call.rel(0); - final ScanPrel scan = call.rel(1); - - final RexNode condition = filter.getCondition(); - - if (scan.getGroupScan() instanceof BinaryTableGroupScan) { - BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan(); - doPushFilterIntoBinaryGroupScan(call, filter, null, scan, groupScan, condition); - } else { - assert(scan.getGroupScan() instanceof JsonTableGroupScan); - JsonTableGroupScan groupScan = (JsonTableGroupScan)scan.getGroupScan(); - doPushFilterIntoJsonGroupScan(call, filter, null, scan, groupScan, condition); - } - } - - @Override - public boolean matches(RelOptRuleCall call) { - final ScanPrel scan = (ScanPrel) call.rel(1); - if (scan.getGroupScan() instanceof BinaryTableGroupScan || - scan.getGroupScan() instanceof JsonTableGroupScan) { - return super.matches(call); - } - return false; - } - }; - - public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new MapRDBPushFilterIntoScan(RelOptHelper.some(FilterPrel.class, RelOptHelper.some(ProjectPrel.class, RelOptHelper.any(ScanPrel.class))), "MapRDBPushFilterIntoScan:Filter_On_Project") { - - @Override - public void onMatch(RelOptRuleCall call) { - final FilterPrel filter = call.rel(0); - final ProjectPrel project = call.rel(1); - final ScanPrel scan = call.rel(2); - - // convert the filter to one that references the child of the project - final RexNode condition = RelOptUtil.pushPastProject(filter.getCondition(), project); - - if (scan.getGroupScan() instanceof BinaryTableGroupScan) { - BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan(); - doPushFilterIntoBinaryGroupScan(call, filter, project, scan, groupScan, condition); - } else { - assert(scan.getGroupScan() instanceof JsonTableGroupScan); - JsonTableGroupScan groupScan = (JsonTableGroupScan)scan.getGroupScan(); - doPushFilterIntoJsonGroupScan(call, filter, project, scan, groupScan, condition); - } - } - - @Override - public boolean matches(RelOptRuleCall call) { - final ScanPrel scan = call.rel(2); - if (scan.getGroupScan() instanceof BinaryTableGroupScan || - scan.getGroupScan() instanceof JsonTableGroupScan) { - return super.matches(call); - } - return false; - } - }; - - protected void doPushFilterIntoJsonGroupScan(RelOptRuleCall call, - FilterPrel filter, final ProjectPrel project, ScanPrel scan, - JsonTableGroupScan groupScan, RexNode condition) { - - if (groupScan.isDisablePushdown() // Do not pushdown filter if it is disabled in plugin configuration - || groupScan.isFilterPushedDown()) { // see below - /* - * The rule can get triggered again due to the transformed "scan => filter" sequence - * created by the earlier execution of this rule when we could not do a complete - * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon - * this flag to not do a re-processing of the rule on the already transformed call. - */ - return; - } - - LogicalExpression conditionExp; - try { - conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); - } catch (ClassCastException e) { - // MD-771 bug in DrillOptiq.toDrill() causes filter condition on ITEM operator to throw ClassCastException - // For such cases, we return without pushdown - return; - } - final JsonConditionBuilder jsonConditionBuilder = new JsonConditionBuilder(groupScan, conditionExp); - final JsonScanSpec newScanSpec = jsonConditionBuilder.parseTree(); - if (newScanSpec == null) { - return; // no filter pushdown ==> No transformation. - } - - final JsonTableGroupScan newGroupsScan = (JsonTableGroupScan) groupScan.clone(newScanSpec); - newGroupsScan.setFilterPushedDown(true); - - final ScanPrel newScanPrel = new ScanPrel(scan.getCluster(), filter.getTraitSet(), newGroupsScan, scan.getRowType(), scan.getTable()); - - // Depending on whether is a project in the middle, assign either scan or copy of project to childRel. - final RelNode childRel = project == null ? newScanPrel : project.copy(project.getTraitSet(), ImmutableList.of((RelNode)newScanPrel)); - - if (jsonConditionBuilder.isAllExpressionsConverted()) { - /* - * Since we could convert the entire filter condition expression into an HBase filter, - * we can eliminate the filter operator altogether. - */ - call.transformTo(childRel); - } else { - call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(childRel))); - } - } - - protected void doPushFilterIntoBinaryGroupScan(final RelOptRuleCall call, - final FilterPrel filter, - final ProjectPrel project, - final ScanPrel scan, - final BinaryTableGroupScan groupScan, - final RexNode condition) { - - if (groupScan.isFilterPushedDown()) { - /* - * The rule can get triggered again due to the transformed "scan => filter" sequence - * created by the earlier execution of this rule when we could not do a complete - * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon - * this flag to not do a re-processing of the rule on the already transformed call. - */ - return; - } - - final LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition); - final MapRDBFilterBuilder maprdbFilterBuilder = new MapRDBFilterBuilder(groupScan, conditionExp); - final HBaseScanSpec newScanSpec = maprdbFilterBuilder.parseTree(); - if (newScanSpec == null) { - return; //no filter pushdown ==> No transformation. - } - - // Pass tableStats from old groupScan so we do not go and fetch stats (an expensive operation) again from MapR DB client. - final BinaryTableGroupScan newGroupsScan = - new BinaryTableGroupScan(groupScan.getUserName(), groupScan.getStoragePlugin(), - groupScan.getFormatPlugin(), newScanSpec, groupScan.getColumns(), - groupScan.getTableStats(), groupScan.getMetadataProvider()); - newGroupsScan.setFilterPushedDown(true); - - final ScanPrel newScanPrel = new ScanPrel(scan.getCluster(), filter.getTraitSet(), newGroupsScan, scan.getRowType(), scan.getTable()); - - // Depending on whether is a project in the middle, assign either scan or copy of project to childRel. - final RelNode childRel = project == null ? newScanPrel : project.copy(project.getTraitSet(), ImmutableList.of((RelNode)newScanPrel)); - - if (maprdbFilterBuilder.isAllExpressionsConverted()) { - /* - * Since we could convert the entire filter condition expression into an HBase filter, - * we can eliminate the filter operator altogether. - */ - call.transformTo(childRel); - } else { - call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(childRel))); - } - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushLimitIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushLimitIntoScan.java deleted file mode 100644 index 325277bcc04..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushLimitIntoScan.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptRuleOperand; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rex.RexLiteral; -import org.apache.drill.exec.physical.base.GroupScan; -import org.apache.drill.exec.planner.common.DrillRelOptUtil; -import org.apache.drill.exec.planner.logical.RelOptHelper; -import org.apache.drill.exec.planner.physical.LimitPrel; -import org.apache.drill.exec.planner.physical.ProjectPrel; -import org.apache.drill.exec.planner.physical.RowKeyJoinPrel; -import org.apache.drill.exec.planner.physical.ScanPrel; -import org.apache.drill.exec.store.StoragePluginOptimizerRule; -import org.apache.drill.exec.store.hbase.HBaseScanSpec; -import org.apache.drill.exec.store.mapr.db.binary.BinaryTableGroupScan; -import org.apache.drill.exec.store.mapr.db.json.JsonTableGroupScan; -import org.apache.drill.exec.store.mapr.db.json.RestrictedJsonTableGroupScan; - -public abstract class MapRDBPushLimitIntoScan extends StoragePluginOptimizerRule { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBPushLimitIntoScan.class); - - private MapRDBPushLimitIntoScan(RelOptRuleOperand operand, String description) { - super(operand, description); - } - - public static final StoragePluginOptimizerRule LIMIT_ON_SCAN = - new MapRDBPushLimitIntoScan(RelOptHelper.some(LimitPrel.class, RelOptHelper.any(ScanPrel.class)), - "MapRDBPushLimitIntoScan:Limit_On_Scan") { - - @Override - public void onMatch(RelOptRuleCall call) { - final LimitPrel limit = call.rel(0); - final ScanPrel scan = call.rel(1); - doPushLimitIntoGroupScan(call, limit, null, scan, scan.getGroupScan()); - } - - @Override - public boolean matches(RelOptRuleCall call) { - final ScanPrel scan = call.rel(1); - final LimitPrel limit = call.rel(0); - // pushdown only apply limit but not offset, - // so if getFetch() return null no need to run this rule. - if (scan.getGroupScan().supportsLimitPushdown() - && !limit.isPushDown() && limit.getFetch() != null) { - if ((scan.getGroupScan() instanceof JsonTableGroupScan - && ((JsonTableGroupScan) scan.getGroupScan()).isIndexScan()) - || (scan.getGroupScan() instanceof RestrictedJsonTableGroupScan)) { - return true; - } - } - return false; - } - }; - - public static final StoragePluginOptimizerRule LIMIT_ON_PROJECT = - new MapRDBPushLimitIntoScan(RelOptHelper.some(LimitPrel.class, - RelOptHelper.any(ProjectPrel.class)), "MapRDBPushLimitIntoScan:Limit_On_Project") { - - @Override - public void onMatch(RelOptRuleCall call) { - final ProjectPrel project = call.rel(1); - final LimitPrel limit = call.rel(0); - RelNode child = project.getInput(); - final RelNode limitUnderProject = new LimitPrel(child.getCluster(), child.getTraitSet(), - child, limit.getOffset(), limit.getFetch()); - final RelNode newProject = new ProjectPrel(project.getCluster(), project.getTraitSet(), - limitUnderProject, project.getProjects(), project.getRowType()); - if (DrillRelOptUtil.isProjectFlatten(project)) { - //Preserve limit above the project since Flatten can produce more rows. Also mark it so we do not fire the rule again. - child = newProject; - final RelNode limitAboveProject = new LimitPrel(child.getCluster(), child.getTraitSet(), - child, limit.getOffset(), limit.getFetch(), true); - call.transformTo(limitAboveProject); - } else { - call.transformTo(newProject); - } - } - - @Override - public boolean matches(RelOptRuleCall call) { - LimitPrel limitPrel = call.rel(0); - ProjectPrel projectPrel = call.rel(1); - // pushdown only apply limit but not offset, - // so if getFetch() return null no need to run this rule. - // Do not push across Project containing CONVERT_FROMJSON for limit 0 queries. For limit 0 queries, this would - // mess up the schema since Convert_FromJson() is different from other regular functions in that it only knows - // the output schema after evaluation is performed. When input has 0 row, Drill essentially does not have a way - // to know the output type. - if (!limitPrel.isPushDown() && (limitPrel.getFetch() != null) - && (!DrillRelOptUtil.isLimit0(limitPrel.getFetch()) - || !DrillRelOptUtil.isProjectOutputSchemaUnknown(projectPrel))) { - return true; - } - return false; - } - }; - - public static final StoragePluginOptimizerRule LIMIT_ON_RKJOIN = - new MapRDBPushLimitIntoScan(RelOptHelper.some(LimitPrel.class, RelOptHelper.any(RowKeyJoinPrel.class)), - "MapRDBPushLimitIntoScan:Limit_On_RKJoin") { - - @Override - public void onMatch(RelOptRuleCall call) { - final RowKeyJoinPrel join = call.rel(1); - final LimitPrel limit = call.rel(0); - doPushLimitIntoRowKeyJoin(call, limit, null, join); - } - - @Override - public boolean matches(RelOptRuleCall call) { - final LimitPrel limit = call.rel(0); - // We do not fire this rule if fetch() is null (indicating we have to fetch all the - // remaining rows starting from offset. - return !limit.isPushDown() && limit.getFetch() != null; - } - }; - - protected void doPushLimitIntoGroupScan(RelOptRuleCall call, - LimitPrel limit, final ProjectPrel project, ScanPrel scan, GroupScan groupScan) { - try { - final GroupScan newGroupScan = getGroupScanWithLimit(groupScan, limit); - if (newGroupScan == null) { - return; - } - final ScanPrel newScan = new ScanPrel(scan.getCluster(), scan.getTraitSet(), newGroupScan, - scan.getRowType(), scan.getTable()); - final RelNode newChild; - if (project != null) { - final ProjectPrel newProject = new ProjectPrel(project.getCluster(), project.getTraitSet(), - newScan, project.getProjects(), project.getRowType()); - newChild = newProject; - } else { - newChild = newScan; - } - call.transformTo(newChild); - logger.debug("pushLimitIntoGroupScan: Converted to a new ScanPrel " + newScan.getGroupScan()); - } catch (Exception e) { - logger.warn("pushLimitIntoGroupScan: Exception while trying limit pushdown!", e); - } - } - - private GroupScan getGroupScanWithLimit(GroupScan groupScan, LimitPrel limit) { - final int offset = limit.getOffset() != null ? Math.max(0, RexLiteral.intValue(limit.getOffset())) : 0; - final int fetch = Math.max(0, RexLiteral.intValue(limit.getFetch())); - // Scan Limit uses conservative approach: use offset 0 and fetch = parent limit offset + parent limit fetch. - if (groupScan instanceof JsonTableGroupScan) { - JsonTableGroupScan jsonTableGroupScan = (JsonTableGroupScan) groupScan; - return (jsonTableGroupScan.clone(jsonTableGroupScan.getScanSpec()).applyLimit(offset + fetch)); - } else if (groupScan instanceof BinaryTableGroupScan) { - BinaryTableGroupScan binaryTableGroupScan = (BinaryTableGroupScan) groupScan; - final HBaseScanSpec oldScanSpec = binaryTableGroupScan.getHBaseScanSpec(); - final HBaseScanSpec newScanSpec = new HBaseScanSpec(oldScanSpec.getTableName(), oldScanSpec.getStartRow(), - oldScanSpec.getStopRow(), oldScanSpec.getFilter()); - return new BinaryTableGroupScan(binaryTableGroupScan.getUserName(), binaryTableGroupScan.getStoragePlugin(), - binaryTableGroupScan.getFormatPlugin(), newScanSpec, binaryTableGroupScan.getColumns(), - binaryTableGroupScan.getTableStats(), binaryTableGroupScan.getMetadataProvider()).applyLimit(offset + fetch); - } - return null; - } - - protected void doPushLimitIntoRowKeyJoin(RelOptRuleCall call, - LimitPrel limit, final ProjectPrel project, RowKeyJoinPrel join) { - final RelNode newChild; - try { - RelNode left = join.getLeft(); - RelNode right = join.getRight(); - final RelNode limitOnLeft = new LimitPrel(left.getCluster(), left.getTraitSet(), left, - limit.getOffset(), limit.getFetch()); - RowKeyJoinPrel newJoin = new RowKeyJoinPrel(join.getCluster(), join.getTraitSet(), limitOnLeft, right, - join.getCondition(), join.getJoinType()); - if (project != null) { - final ProjectPrel newProject = new ProjectPrel(project.getCluster(), project.getTraitSet(), newJoin, - project.getProjects(), project.getRowType()); - newChild = newProject; - } else { - newChild = newJoin; - } - call.transformTo(newChild); - logger.debug("pushLimitIntoRowKeyJoin: Pushed limit on left side of Join " + join.toString()); - } catch (Exception e) { - logger.warn("pushLimitIntoRowKeyJoin: Exception while trying limit pushdown!", e); - } - } -} - diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushProjectIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushProjectIntoScan.java deleted file mode 100644 index 67e40947e56..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBPushProjectIntoScan.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptRuleOperand; -import org.apache.calcite.plan.RelTrait; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.RelCollation; -import org.apache.calcite.rel.rules.ProjectRemoveRule; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexNode; -import org.apache.drill.common.exceptions.DrillRuntimeException; -import org.apache.drill.exec.planner.common.DrillRelOptUtil; -import org.apache.drill.exec.planner.logical.RelOptHelper; -import org.apache.drill.exec.planner.physical.Prel; -import org.apache.drill.exec.planner.physical.ProjectPrel; -import org.apache.drill.exec.planner.physical.ScanPrel; -import org.apache.drill.exec.store.StoragePluginOptimizerRule; -import org.apache.drill.exec.store.mapr.db.json.JsonTableGroupScan; -import org.apache.drill.exec.util.Utilities; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * Push a physical Project into Scan. Currently, this rule is only doing projection pushdown for MapRDB-JSON tables - * since it was needed for the secondary index feature which only applies to Json tables. - * For binary tables, note that the DrillPushProjectIntoScanRule is still applicable during the logical - * planning phase. - */ -public abstract class MapRDBPushProjectIntoScan extends StoragePluginOptimizerRule { - - private MapRDBPushProjectIntoScan(RelOptRuleOperand operand, String description) { - super(operand, description); - } - - public static final StoragePluginOptimizerRule PROJECT_ON_SCAN = new MapRDBPushProjectIntoScan( - RelOptHelper.some(ProjectPrel.class, RelOptHelper.any(ScanPrel.class)), "MapRDBPushProjIntoScan:Proj_On_Scan") { - @Override - public void onMatch(RelOptRuleCall call) { - final ProjectPrel project = call.rel(0); - final ScanPrel scan = call.rel(1); - - if (scan.getGroupScan() instanceof JsonTableGroupScan) { - JsonTableGroupScan groupScan = (JsonTableGroupScan) scan.getGroupScan(); - - doPushProjectIntoGroupScan(call, project, scan, groupScan); - } - } - - @Override - public boolean matches(RelOptRuleCall call) { - final ScanPrel scan = call.rel(1); - - // See class level comments above for why only JsonGroupScan is considered - if (scan.getGroupScan() instanceof JsonTableGroupScan) { - return super.matches(call); - } - return false; - } - }; - - protected void doPushProjectIntoGroupScan(RelOptRuleCall call, - ProjectPrel project, ScanPrel scan, JsonTableGroupScan groupScan) { - try { - - DrillRelOptUtil.ProjectPushInfo columnInfo = - DrillRelOptUtil.getFieldsInformation(scan.getRowType(), project.getProjects()); - if (columnInfo == null || Utilities.isStarQuery(columnInfo.getFields()) - || !groupScan.canPushdownProjects(columnInfo.getFields())) { - return; - } - RelTraitSet newTraits = call.getPlanner().emptyTraitSet(); - // Clear out collation trait - for (RelTrait trait : scan.getTraitSet()) { - if (!(trait instanceof RelCollation)) { - newTraits.plus(trait); - } - } - final ScanPrel newScan = new ScanPrel(scan.getCluster(), newTraits.plus(Prel.DRILL_PHYSICAL), - groupScan.clone(columnInfo.getFields()), - columnInfo.createNewRowType(project.getInput().getCluster().getTypeFactory()), scan.getTable()); - - List newProjects = project.getProjects().stream() - .map(n -> n.accept(columnInfo.getInputReWriter())) - .collect(Collectors.toList()); - - final ProjectPrel newProj = - new ProjectPrel(project.getCluster(), - project.getTraitSet().plus(Prel.DRILL_PHYSICAL), - newScan, - newProjects, - project.getRowType()); - - if (ProjectRemoveRule.isTrivial(newProj) && - // the old project did not involve any column renaming - sameRowTypeProjectionsFields(project.getRowType(), newScan.getRowType())) { - call.transformTo(newScan); - } else { - call.transformTo(newProj); - } - } catch (Exception e) { - throw new DrillRuntimeException(e); - } - } - - private boolean sameRowTypeProjectionsFields(RelDataType oldRowType, RelDataType newRowType) { - for (RelDataTypeField oldField : oldRowType.getFieldList()) { - String oldProjName = oldField.getName(); - boolean match = false; - for (RelDataTypeField newField : newRowType.getFieldList()) { - if (oldProjName.equals(newField.getName())) { - match = true; - break; - } - } - if (!match) { - return false; - } - } - return true; - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBRestrictedScanBatchCreator.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBRestrictedScanBatchCreator.java deleted file mode 100644 index 9238d1d3591..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBRestrictedScanBatchCreator.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.exec.ops.ExecutorFragmentContext; -import org.apache.drill.exec.physical.impl.BatchCreator; -import org.apache.drill.exec.physical.impl.ScanBatch; -import org.apache.drill.exec.record.RecordBatch; -import org.apache.drill.exec.store.RecordReader; -import org.apache.drill.exec.store.mapr.db.json.RestrictedJsonRecordReader; - -import java.util.List; - -public class MapRDBRestrictedScanBatchCreator implements BatchCreator { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBRestrictedScanBatchCreator.class); - - @Override - public ScanBatch getBatch(ExecutorFragmentContext context, RestrictedMapRDBSubScan subScan, List children) throws ExecutionSetupException { - Preconditions.checkArgument(children.isEmpty()); - List readers = Lists.newArrayList(); - for(MapRDBSubScanSpec scanSpec : subScan.getRegionScanSpecList()){ - try { - readers.add(new RestrictedJsonRecordReader(scanSpec, subScan.getFormatPlugin(), subScan.getColumns(), - context, subScan.getMaxRecordsToRead(), subScan.getSchema())); - } catch (Exception e1) { - throw new ExecutionSetupException(e1); - } - } - return new ScanBatch(subScan, context, readers, true); - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java deleted file mode 100644 index 190f535aed7..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBScanBatchCreator.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import java.util.LinkedList; -import java.util.List; - -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.exec.ops.ExecutorFragmentContext; -import org.apache.drill.exec.physical.impl.BatchCreator; -import org.apache.drill.exec.physical.impl.ScanBatch; -import org.apache.drill.exec.record.RecordBatch; -import org.apache.drill.exec.store.RecordReader; -import org.apache.drill.exec.store.hbase.HBaseRecordReader; -import org.apache.drill.exec.store.hbase.HBaseSubScan.HBaseSubScanSpec; -import org.apache.drill.exec.store.mapr.db.binary.BinaryTableGroupScan; -import org.apache.drill.exec.store.mapr.db.json.MaprDBJsonRecordReader; - -import com.google.common.base.Preconditions; - -public class MapRDBScanBatchCreator implements BatchCreator { - - @Override - public ScanBatch getBatch(ExecutorFragmentContext context, MapRDBSubScan subScan, List children) throws ExecutionSetupException { - Preconditions.checkArgument(children.isEmpty()); - List readers = new LinkedList<>(); - for (MapRDBSubScanSpec scanSpec : subScan.getRegionScanSpecList()) { - try { - if (BinaryTableGroupScan.TABLE_BINARY.equals(subScan.getTableType())) { - readers.add(new HBaseRecordReader( - subScan.getFormatPlugin().getConnection(), - getHBaseSubScanSpec(scanSpec), - subScan.getColumns())); - } else { - readers.add(new MaprDBJsonRecordReader(scanSpec, subScan.getFormatPlugin(), - subScan.getColumns(), context, subScan.getMaxRecordsToRead(), subScan.getSchema())); - } - } catch (Exception e) { - throw new ExecutionSetupException(e); - } - } - return new ScanBatch(subScan, context, readers); - } - - private HBaseSubScanSpec getHBaseSubScanSpec(MapRDBSubScanSpec scanSpec) { - return new HBaseSubScanSpec(scanSpec.getTableName(), scanSpec.getRegionServer(), - scanSpec.getStartRow(), scanSpec.getStopRow(), scanSpec.getSerializedFilter(), null); - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java deleted file mode 100644 index e9ed7db8342..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScan.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import java.util.Collections; -import java.util.Iterator; -import java.util.List; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.logical.StoragePluginConfig; -import org.apache.drill.exec.physical.base.AbstractDbSubScan; -import org.apache.drill.exec.physical.base.PhysicalOperator; -import org.apache.drill.exec.physical.base.PhysicalVisitor; -import org.apache.drill.exec.record.metadata.TupleMetadata; -import org.apache.drill.exec.store.StoragePluginRegistry; - -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.base.Preconditions; - -// Class containing information for reading a single HBase region -@JsonTypeName("maprdb-sub-scan") - -public class MapRDBSubScan extends AbstractDbSubScan { - - public static final String OPERATOR_TYPE = "MAPRDB_SUB_SCAN"; - - private final MapRDBFormatPlugin formatPlugin; - private final List regionScanSpecList; - private final List columns; - private final int maxRecordsToRead; - private final String tableType; - private final TupleMetadata schema; - - @JsonCreator - public MapRDBSubScan(@JacksonInject StoragePluginRegistry engineRegistry, - @JsonProperty("userName") String userName, - @JsonProperty("formatPluginConfig") MapRDBFormatPluginConfig formatPluginConfig, - @JsonProperty("storageConfig") StoragePluginConfig storageConfig, - @JsonProperty("regionScanSpecList") List regionScanSpecList, - @JsonProperty("columns") List columns, - @JsonProperty("maxRecordsToRead") int maxRecordsToRead, - @JsonProperty("tableType") String tableType, - @JsonProperty("schema") TupleMetadata schema) throws ExecutionSetupException { - this(userName, - engineRegistry.resolveFormat(storageConfig, formatPluginConfig, MapRDBFormatPlugin.class), - regionScanSpecList, - columns, - maxRecordsToRead, - tableType, - schema); - } - - public MapRDBSubScan(String userName, MapRDBFormatPlugin formatPlugin, - List maprSubScanSpecs, List columns, String tableType, TupleMetadata schema) { - this(userName, formatPlugin, maprSubScanSpecs, columns, -1, tableType, schema); - } - - public MapRDBSubScan(String userName, MapRDBFormatPlugin formatPlugin, - List maprSubScanSpecs, List columns, int maxRecordsToRead, String tableType, TupleMetadata schema) { - super(userName); - this.formatPlugin = formatPlugin; - this.regionScanSpecList = maprSubScanSpecs; - this.columns = columns; - this.maxRecordsToRead = maxRecordsToRead; - this.tableType = tableType; - this.schema = schema; - } - - - @JsonProperty("formatPluginConfig") - public MapRDBFormatPluginConfig getFormatPluginConfig() { - return formatPlugin.getConfig(); - } - - @JsonProperty("storageConfig") - public StoragePluginConfig getStorageConfig(){ - return formatPlugin.getStorageConfig(); - } - - @JsonProperty("regionScanSpecList") - public List getRegionScanSpecList() { - return regionScanSpecList; - } - - @JsonProperty("columns") - public List getColumns() { - return columns; - } - - @JsonProperty("maxRecordsToRead") - public int getMaxRecordsToRead() { - return maxRecordsToRead; - } - - @JsonProperty("tableType") - public String getTableType() { - return tableType; - } - - @JsonProperty("schema") - public TupleMetadata getSchema() { - return schema; - } - - @Override - public boolean isExecutable() { - return false; - } - - @Override - public T accept(PhysicalVisitor physicalVisitor, X value) throws E { - return physicalVisitor.visitSubScan(this, value); - } - - @Override - public PhysicalOperator getNewWithChildren(List children) { - Preconditions.checkArgument(children.isEmpty()); - return new MapRDBSubScan(getUserName(), formatPlugin, regionScanSpecList, columns, tableType, schema); - } - - @Override - public Iterator iterator() { - return Collections.emptyIterator(); - } - - @Override - public String getOperatorType() { - return OPERATOR_TYPE; - } - - @JsonIgnore - public MapRDBFormatPlugin getFormatPlugin() { - return formatPlugin; - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScanSpec.java deleted file mode 100644 index 7fc2e8376ca..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBSubScanSpec.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.mapr.db.index.IndexDesc; -import com.mapr.fs.jni.MapRConstants; -import com.mapr.org.apache.hadoop.hbase.util.Bytes; - -public class MapRDBSubScanSpec implements Comparable { - - protected String tableName; - protected IndexDesc indexDesc; - protected String regionServer; - protected byte[] startRow; - protected byte[] stopRow; - protected byte[] serializedFilter; - protected String userName; - - @JsonCreator - public MapRDBSubScanSpec(@JsonProperty("tableName") String tableName, - @JsonProperty("indexDesc") IndexDesc indexDesc, - @JsonProperty("regionServer") String regionServer, - @JsonProperty("startRow") byte[] startRow, - @JsonProperty("stopRow") byte[] stopRow, - @JsonProperty("serializedFilter") byte[] serializedFilter, - @JsonProperty("filterString") String filterString, - @JsonProperty("username") String userName) { - if (serializedFilter != null && filterString != null) { - throw new IllegalArgumentException("The parameters 'serializedFilter' or 'filterString' cannot be specified at the same time."); - } - this.tableName = tableName; - this.indexDesc = indexDesc; - this.regionServer = regionServer; - this.startRow = startRow; - this.stopRow = stopRow; - this.serializedFilter = serializedFilter; - this.userName = userName; - } - - /* package */ MapRDBSubScanSpec() { - // empty constructor, to be used with builder pattern; - } - - public String getTableName() { - return tableName; - } - - public IndexDesc getIndexDesc() { - return indexDesc; - } - - public MapRDBSubScanSpec setTableName(String tableName) { - this.tableName = tableName; - return this; - } - - public String getRegionServer() { - return regionServer; - } - - public MapRDBSubScanSpec setRegionServer(String regionServer) { - this.regionServer = regionServer; - return this; - } - - /** - * @return the raw (not-encoded) start row key for this sub-scan - */ - public byte[] getStartRow() { - return startRow == null ? MapRConstants.EMPTY_BYTE_ARRAY: startRow; - } - - public MapRDBSubScanSpec setStartRow(byte[] startRow) { - this.startRow = startRow; - return this; - } - - /** - * @return the raw (not-encoded) stop row key for this sub-scan - */ - public byte[] getStopRow() { - return stopRow == null ? MapRConstants.EMPTY_BYTE_ARRAY : stopRow; - } - - public MapRDBSubScanSpec setStopRow(byte[] stopRow) { - this.stopRow = stopRow; - return this; - } - - public byte[] getSerializedFilter() { - return serializedFilter; - } - - public MapRDBSubScanSpec setSerializedFilter(byte[] serializedFilter) { - this.serializedFilter = serializedFilter; - return this; - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - @Override - public String toString() { - return "MapRDBSubScanSpec [tableName=" + tableName - + ", startRow=" + (startRow == null ? null : Bytes.toStringBinary(startRow)) - + ", stopRow=" + (stopRow == null ? null : Bytes.toStringBinary(stopRow)) - + ", filter=" + (getSerializedFilter() == null ? null : Bytes.toBase64(getSerializedFilter())) - + ", regionServer=" + regionServer - + ", userName=" + userName + "]"; - } - - @Override - /* - * The semantics of the compareTo function is same as that of TabletInfoImpl. - * It compares the startRows of the two subScanSpec and returns the status - * if one is greater than the other. If the two startRows are same then it - * compares the stopRows. - */ - public int compareTo(MapRDBSubScanSpec o) { - if (o == null) { - return 1; - } else { - int result = Bytes.compareTo(this.getStartRow(), o.getStartRow()); - if (result != 0) { - return result; - } else { - result = Bytes.compareTo(this.getStopRow(), o.getStopRow()); - if (result != 0) { - if (this.getStartRow().length != 0 && this.getStopRow().length == 0) { - return 1; - } else { - return o.getStartRow().length != 0 && o.getStopRow().length == 0 ? -1 : result; - } - } else { - return 0; - } - } - } - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableCache.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableCache.java deleted file mode 100644 index e74c13776b7..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableCache.java +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import com.mapr.db.Table; -import com.mapr.db.impl.MapRDBImpl; -import com.mapr.db.index.IndexDesc; -import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.exceptions.DrillRuntimeException; -import org.apache.drill.exec.util.ImpersonationUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; - -import java.io.IOException; -import java.security.PrivilegedExceptionAction; -import java.util.concurrent.TimeUnit; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MapRDBTableCache { - - private static final Logger logger = LoggerFactory.getLogger(MapRDBTableCache.class); - - public static final String FORMAT_MAPRDB_JSON_TABLE_CACHE_ENABLED = "format-maprdb.json.tableCache.enabled"; - - public static final String FORMAT_MAPRDB_JSON_TABLE_CACHE_SIZE = "format-maprdb.json.tableCache.size"; - - public static final String FORMAT_MAPRDB_JSON_TABLE_CACHE_TIMEOUT = "format-maprdb.json.tableCache.expireTimeInMinutes"; - - private static final int MIN_TABLE_CACHE_SIZE = 1; - - private static final int MIN_TABLE_CACHE_ENTRY_TIMEOUT = 10; - - LoadingCache tableCache; - - private final boolean tableCachingEnabled; - - public MapRDBTableCache(DrillConfig config) { - tableCachingEnabled = config.getBoolean(FORMAT_MAPRDB_JSON_TABLE_CACHE_ENABLED); - if (tableCachingEnabled) { - final int tableCacheSize = Math.max((int) (config.getDouble(FORMAT_MAPRDB_JSON_TABLE_CACHE_SIZE)), MIN_TABLE_CACHE_SIZE); - final int tableCacheExpiryTime = Math.max((int) (config.getDouble(FORMAT_MAPRDB_JSON_TABLE_CACHE_TIMEOUT)), MIN_TABLE_CACHE_ENTRY_TIMEOUT); - - RemovalListener removalListener = new RemovalListener() { - public void onRemoval(RemovalNotification removal) { - Table table = removal.getValue(); - MapRDBTableCache.Key key = removal.getKey(); - logger.debug("time {} closing the tablePath {} tableHandle {} index {} userName {}", - System.nanoTime(), - key.path == null ? "null" : key.path, table == null ? "null" : table, - key.indexDesc == null ? "null" : key.indexDesc.getIndexName(), - key.ugi.getUserName() == null ? "null" : key.ugi.getUserName()); - table.close(); // close the table - } - }; - - // Common table cache for primary and index tables. Key is Pair - // For primary table, indexDesc is null. - tableCache = CacheBuilder.newBuilder(). - expireAfterAccess(tableCacheExpiryTime, TimeUnit.MINUTES). - maximumSize(tableCacheSize). - removalListener(removalListener).build(new CacheLoader() { - - @Override - public Table load(final MapRDBTableCache.Key key) throws Exception { - // getTable is already calling tableCache.get in correct user UGI context, so should be fine here. - // key.Left is Path. key.Right is indexDesc. - Table table = (key.indexDesc == null ? MapRDBImpl.getTable(key.path) : MapRDBImpl.getIndexTable(key.indexDesc)); - logger.debug("time {} opened the table for tablePath {} tableHandle {} index {} userName {}", - System.nanoTime(), - key.path == null ? "null" : key.path, - table == null ? "null" : table, - key.indexDesc == null ? "null" : key.indexDesc.getIndexName(), - key.ugi.getUserName() == null ? "null" : key.ugi.getUserName()); - return table; - } - }); - - logger.debug("table cache created with size {} and expiryTimeInMin {} ", tableCacheSize, tableCacheExpiryTime); - } - } - - - /** - * getTable given primary table path and indexDesc. - * returns Table for corresponding index table if indexDesc is not null. - * returns Table for primary table if indexDesc is null. - * - * @param tablePath primary table path - * @param indexDesc index table descriptor - */ - public Table getTable(final Path tablePath, final IndexDesc indexDesc, final String userName) throws DrillRuntimeException { - - final Table dbTableHandle; - final UserGroupInformation proxyUserUgi = ImpersonationUtil.createProxyUgi(userName); - - try { - dbTableHandle = proxyUserUgi.doAs(new PrivilegedExceptionAction() { - public Table run() throws Exception { - - if (logger.isTraceEnabled()) { - logger.trace("Getting MaprDB Table handle for proxy user: " + UserGroupInformation.getCurrentUser()); - } - - if (tableCachingEnabled) { - Table table = tableCache.get(new MapRDBTableCache.Key(tablePath, indexDesc)); - logger.trace("time {} get the tablePath {} tableHandle {} index {} userName {} currentUser {}", - System.nanoTime(), tablePath == null ? "null" : tablePath, - table == null ? "null" : table, - indexDesc == null ? "null" : indexDesc.getIndexName(), - userName == null ? "null" : userName, - UserGroupInformation.getCurrentUser() == null ? "null" : UserGroupInformation.getCurrentUser()); - return table; - } else { - return indexDesc == null ? MapRDBImpl.getTable(tablePath) : MapRDBImpl.getIndexTable(indexDesc); - } - } - }); - } catch (Exception e) { - throw new DrillRuntimeException("Error getting table: " + tablePath.toString() + (indexDesc == null ? "" : (", " + - "IndexDesc: " + indexDesc.toString())), e); - } - - return dbTableHandle; - } - - /** - * getTable given primary table name. - * returns Table for primary table with given name. - * - * @param tableName primary table path - */ - public Table getTable(String tableName, String userName) { - return getTable(new Path(tableName), null, userName); - } - - /** - * getTable given primary table path. - * returns Table for primary table with given path. - * - * @param tablePath primary table path - */ - public Table getTable(Path tablePath, String userName) { - return getTable(tablePath, null, userName); - } - - /** - * getTable given primary table name and indexDesc. - * returns Table for corresponding index table if indexDesc is not null. - * returns Table for primary table if indexDesc is null. - * - * @param tableName primary table name - * @param indexDesc index table Descriptor - */ - public Table getTable(String tableName, IndexDesc indexDesc, String userName) { - return getTable(new Path(tableName), indexDesc, userName); - } - - /** - * closeTable - * - * @param table table to be closed. - */ - public void closeTable(Table table) { - if (!tableCachingEnabled && table != null) { - table.close(); - } - } - - /** - * Key for {@link MapRDBTableCache} to store table path, {@link IndexDesc} and UGI. - */ - static class Key { - final Path path; - - final IndexDesc indexDesc; - - final UserGroupInformation ugi; - - Key(Path path, IndexDesc indexDesc) throws IOException { - this.path = path; - this.indexDesc = indexDesc; - this.ugi = UserGroupInformation.getCurrentUser(); - } - - public int hashCode() { - - final int IdxDescHashCode = (indexDesc == null) ? 0 : indexDesc.getIndexFid().hashCode(); - return (path.hashCode() + IdxDescHashCode + ugi.hashCode()); - } - - static boolean isEqual(Object a, Object b) { - return a == b || a != null && a.equals(b); - } - - public boolean equals(Object obj) { - if (obj == this) { - return true; - } else if (obj != null && obj instanceof MapRDBTableCache.Key) { - MapRDBTableCache.Key that = (MapRDBTableCache.Key) obj; - return isEqual(this.path, that.path) - && isEqual(this.indexDesc, that.indexDesc) - && isEqual(this.ugi, that.ugi); - } else { - return false; - } - } - - public String toString() { - return "(Path: " + this.path.toString() + - ", UGI: " + this.ugi.toString() + - ", IndexDesc: " + (this.indexDesc == null ? "" : this.indexDesc.toString()) + ")"; - } - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableStats.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableStats.java deleted file mode 100644 index 5e1e2e74811..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/MapRDBTableStats.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.mapr.TableMappingRulesFactory; - -import com.mapr.fs.hbase.HBaseAdminImpl; - -public class MapRDBTableStats { - private static volatile HBaseAdminImpl admin = null; - - private long numRows; - - public MapRDBTableStats(Configuration conf, String tablePath) throws Exception { - if (admin == null) { - synchronized (MapRDBTableStats.class) { - if (admin == null) { - Configuration config = conf; - admin = new HBaseAdminImpl(config, TableMappingRulesFactory.create(conf)); - } - } - } - numRows = admin.getNumRows(tablePath); - } - - public long getNumRows() { - return numRows; - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/RestrictedMapRDBSubScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/RestrictedMapRDBSubScan.java deleted file mode 100644 index 7fa8898c27b..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/RestrictedMapRDBSubScan.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import java.util.ArrayList; -import java.util.List; - -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; - -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.logical.StoragePluginConfig; -import org.apache.drill.exec.physical.impl.join.RowKeyJoin; -import org.apache.drill.exec.record.metadata.TupleMetadata; -import org.apache.drill.exec.store.StoragePluginRegistry; - -/** - * A RestrictedMapRDBSubScan is intended for skip-scan (as opposed to sequential scan) operations - * where the set of rowkeys is obtained from a corresponding RowKeyJoin instance -*/ -@JsonTypeName("maprdb-restricted-subscan") -public class RestrictedMapRDBSubScan extends MapRDBSubScan { - - @JsonCreator - public RestrictedMapRDBSubScan(@JacksonInject StoragePluginRegistry engineRegistry, - @JsonProperty("userName") String userName, - @JsonProperty("formatPluginConfig") MapRDBFormatPluginConfig formatPluginConfig, - @JsonProperty("storageConfig") StoragePluginConfig storageConfig, - @JsonProperty("regionScanSpecList") List regionScanSpecList, - @JsonProperty("columns") List columns, - @JsonProperty("maxRecordsToRead") int maxRecordsToRead, - @JsonProperty("tableType") String tableType, - @JsonProperty("schema") TupleMetadata schema) throws ExecutionSetupException { - this(userName, - engineRegistry.resolveFormat(storageConfig, formatPluginConfig, MapRDBFormatPlugin.class), - regionScanSpecList, columns, maxRecordsToRead, tableType, schema); - } - - public RestrictedMapRDBSubScan(String userName, MapRDBFormatPlugin formatPlugin, - List maprDbSubScanSpecs, - List columns, int maxRecordsToRead, String tableType, TupleMetadata schema) { - super(userName, formatPlugin, new ArrayList<>(), columns, maxRecordsToRead, tableType, schema); - - for(RestrictedMapRDBSubScanSpec restrictedSpec : maprDbSubScanSpecs) { - getRegionScanSpecList().add(restrictedSpec); - } - - } - - @Override - public void addJoinForRestrictedSubScan(RowKeyJoin rjbatch) { - // currently, all subscan specs are sharing the same join batch instance - for (MapRDBSubScanSpec s : getRegionScanSpecList()) { - assert (s instanceof RestrictedMapRDBSubScanSpec); - ((RestrictedMapRDBSubScanSpec)s).setJoinForSubScan(rjbatch); - } - } - - @Override - public boolean isRestrictedSubScan() { - return true; - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/RestrictedMapRDBSubScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/RestrictedMapRDBSubScanSpec.java deleted file mode 100644 index 1faf5551a35..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/RestrictedMapRDBSubScanSpec.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import com.mapr.db.impl.IdCodec; - -import org.apache.commons.lang3.tuple.Pair; -import org.apache.drill.exec.physical.impl.join.RowKeyJoin; -import org.apache.drill.exec.record.AbstractRecordBatch.BatchState; -import org.apache.drill.exec.vector.ValueVector; - -import com.fasterxml.jackson.annotation.JsonIgnore; - -import java.nio.ByteBuffer; - -/** - * A RestrictedMapRDBSubScanSpec encapsulates a join instance which contains the ValueVectors of row keys and - * is associated with this sub-scan and also exposes an iterator type interface over the row key vectors. - */ -public class RestrictedMapRDBSubScanSpec extends MapRDBSubScanSpec { - - /** - * The RowKeyJoin instance (specific to one minor fragment) which will supply this - * subscan with the set of rowkeys. For efficiency, we keep a reference to this - * join rather than making another copy of the rowkeys. - */ - private RowKeyJoin rjbatch = null; - - /** - * The following are needed to maintain internal state of iteration over the set - * of row keys - */ - private ValueVector rowKeyVector = null; // the current row key value vector - private int currentIndex = 0; // the current index within the row key vector - private int maxOccupiedIndex = -1; // max occupied index within a row key vector - - public RestrictedMapRDBSubScanSpec(String tableName, String regionServer, byte[] serializedFilter, String userName) { - super(tableName, null, regionServer, null, null, serializedFilter, null, userName); - } - /* package */ RestrictedMapRDBSubScanSpec() { - // empty constructor, to be used with builder pattern; - } - - public void setJoinForSubScan(RowKeyJoin rjbatch) { - this.rjbatch = rjbatch; - } - - @JsonIgnore - public RowKeyJoin getJoinForSubScan() { - return rjbatch; - } - - @JsonIgnore - private void init(Pair b) { - this.maxOccupiedIndex = b.getRight(); - this.rowKeyVector = b.getLeft(); - this.currentIndex = 0; - } - - /** - * Return {@code true} if a valid rowkey batch is available, {@code false} otherwise - */ - @JsonIgnore - public boolean readyToGetRowKey() { - return rjbatch != null && rjbatch.hasRowKeyBatch(); - } - - /** - * Return {@code true} if the row key join is in the build schema phase - */ - @JsonIgnore - public boolean isBuildSchemaPhase() { - return rjbatch.getBatchState() == BatchState.BUILD_SCHEMA; - } - - /** - * Returns {@code true} if the iteration has more row keys. - * (In other words, returns {@code true} if {@link #nextRowKey} would - * return a non-null row key) - * @return {@code true} if the iteration has more row keys - */ - @JsonIgnore - public boolean hasRowKey() { - if (rowKeyVector != null && currentIndex <= maxOccupiedIndex) { - return true; - } - - if (rjbatch != null) { - Pair currentBatch = rjbatch.nextRowKeyBatch(); - - // note that the hash table could be null initially during the BUILD_SCHEMA phase - if (currentBatch != null) { - init(currentBatch); - return true; - } - } - - return false; - } - - @JsonIgnore - public int getMaxRowKeysToBeRead() { - if (rjbatch != null) { - Pair currentBatch = rjbatch.nextRowKeyBatch(); - - // note that the currentBatch could be null initially during the BUILD_SCHEMA phase - if (currentBatch != null) { - init(currentBatch); - } - } - return maxOccupiedIndex + 1; - } - - /** - * Returns number of rowKeys that can be read. - * Number of rowKeys returned will be numRowKeysToRead at the most i.e. it - * will be less than numRowKeysToRead if only that many exist in the currentBatch. - */ - @JsonIgnore - public int hasRowKeys(int numRowKeysToRead) { - int numKeys = 0; - - // if there is pending rows from the current batch, read them first - // in chunks of numRowsToRead rows - if (rowKeyVector != null && currentIndex <= maxOccupiedIndex) { - numKeys = Math.min(numRowKeysToRead, maxOccupiedIndex - currentIndex + 1); - return numKeys; - } - - // otherwise, get the next batch of rowkeys - if (rjbatch != null) { - Pair currentBatch = rjbatch.nextRowKeyBatch(); - - // note that the currentBatch could be null initially during the BUILD_SCHEMA phase - if (currentBatch != null) { - init(currentBatch); - numKeys = Math.min(numRowKeysToRead, maxOccupiedIndex - currentIndex + 1); - } - } - - return numKeys; - } - - /** - * Returns ids of rowKeys to be read. - * Number of rowKey ids returned will be numRowKeysToRead at the most i.e. it - * will be less than numRowKeysToRead if only that many exist in the currentBatch. - */ - @JsonIgnore - public ByteBuffer[] getRowKeyIdsToRead(int numRowKeysToRead) { - - int numKeys = hasRowKeys(numRowKeysToRead); - if (numKeys == 0) { - return null; - } - - int index = 0; - final ByteBuffer[] rowKeyIds = new ByteBuffer[numKeys]; - - while (index < numKeys) { - Object o = rowKeyVector.getAccessor().getObject(currentIndex + index); - rowKeyIds[index++] = IdCodec.encode(o.toString()); - } - - updateRowKeysRead(numKeys); - return rowKeyIds; - } - - /** - * updates the index to reflect number of keys read. - */ - @JsonIgnore - public void updateRowKeysRead(int numKeys) { - currentIndex += numKeys; - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/TabletFragmentInfo.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/TabletFragmentInfo.java deleted file mode 100644 index 0fb2e394704..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/TabletFragmentInfo.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db; - -import org.apache.hadoop.hbase.HRegionInfo; - -import com.mapr.db.impl.TabletInfoImpl; - -public class TabletFragmentInfo implements Comparable { - - final private HRegionInfo regionInfo; - final private TabletInfoImpl tabletInfoImpl; - - public TabletFragmentInfo(HRegionInfo regionInfo) { - this(null, regionInfo); - } - - public TabletFragmentInfo(TabletInfoImpl tabletInfoImpl) { - this(tabletInfoImpl, null); - } - - TabletFragmentInfo(TabletInfoImpl tabletInfoImpl, HRegionInfo regionInfo) { - this.regionInfo = regionInfo; - this.tabletInfoImpl = tabletInfoImpl; - } - - public HRegionInfo getRegionInfo() { - return regionInfo; - } - - public TabletInfoImpl getTabletInfoImpl() { - return tabletInfoImpl; - } - - public boolean containsRow(byte[] row) { - return tabletInfoImpl != null ? tabletInfoImpl.containsRow(row) : - regionInfo.containsRow(row); - } - - public byte[] getStartKey() { - return tabletInfoImpl != null ? tabletInfoImpl.getStartRow() : - regionInfo.getStartKey(); - } - - public byte[] getEndKey() { - return tabletInfoImpl != null ? tabletInfoImpl.getStopRow() : - regionInfo.getEndKey(); - } - - @Override - public int compareTo(TabletFragmentInfo o) { - return tabletInfoImpl != null ? tabletInfoImpl.compareTo(o.tabletInfoImpl) : - regionInfo.compareTo(o.regionInfo); - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((regionInfo == null) ? 0 : regionInfo.hashCode()); - result = prime * result + ((tabletInfoImpl == null) ? 0 : tabletInfoImpl.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - TabletFragmentInfo other = (TabletFragmentInfo) obj; - if (regionInfo == null) { - if (other.regionInfo != null) { - return false; - } - } else if (!regionInfo.equals(other.regionInfo)) { - return false; - } - if (tabletInfoImpl == null) { - if (other.tabletInfoImpl != null) { - return false; - } - } else if (!tabletInfoImpl.equals(other.tabletInfoImpl)) { - return false; - } - return true; - } - - @Override - public String toString() { - return "TabletFragmentInfo [regionInfo=" + regionInfo + ", tabletInfoImpl=" + tabletInfoImpl - + "]"; - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java deleted file mode 100644 index a53177bcb64..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/BinaryTableGroupScan.java +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.binary; - -import static org.apache.drill.exec.store.mapr.db.util.CommonFns.isNullOrEmpty; - -import java.util.List; -import java.util.TreeMap; - -import com.fasterxml.jackson.annotation.JsonCreator; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rex.RexNode; -import org.apache.drill.common.exceptions.DrillRuntimeException; -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.exec.physical.base.GroupScan; -import org.apache.drill.exec.physical.base.PhysicalOperator; -import org.apache.drill.exec.physical.base.ScanStats; -import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty; -import org.apache.drill.exec.record.metadata.TupleMetadata; -import org.apache.drill.exec.store.AbstractStoragePlugin; -import org.apache.drill.exec.planner.index.Statistics; -import org.apache.drill.exec.store.StoragePluginRegistry; -import org.apache.drill.exec.store.dfs.FileSystemConfig; -import org.apache.drill.exec.store.hbase.DrillHBaseConstants; -import org.apache.drill.exec.store.hbase.HBaseScanSpec; -import org.apache.drill.exec.store.hbase.HBaseUtils; -import org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin; -import org.apache.drill.exec.store.mapr.db.MapRDBFormatPluginConfig; -import org.apache.drill.exec.store.mapr.db.MapRDBGroupScan; -import org.apache.drill.exec.store.mapr.db.MapRDBSubScan; -import org.apache.drill.exec.store.mapr.db.MapRDBSubScanSpec; -import org.apache.drill.exec.store.mapr.db.MapRDBTableStats; -import org.apache.drill.exec.store.mapr.db.TabletFragmentInfo; -import org.apache.drill.exec.metastore.store.FileSystemMetadataProviderManager; -import org.apache.drill.exec.metastore.MetadataProviderManager; -import org.apache.drill.metastore.metadata.TableMetadataProvider; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.RegionLocator; - -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.base.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@JsonTypeName("maprdb-binary-scan") -public class BinaryTableGroupScan extends MapRDBGroupScan implements DrillHBaseConstants { - private static final Logger logger = LoggerFactory.getLogger(BinaryTableGroupScan.class); - - public static final String TABLE_BINARY = "binary"; - - private final HBaseScanSpec hbaseScanSpec; - - private HTableDescriptor hTableDesc; - - private MapRDBTableStats tableStats; - - @JsonCreator - public BinaryTableGroupScan(@JsonProperty("userName") final String userName, - @JsonProperty("hbaseScanSpec") HBaseScanSpec scanSpec, - @JsonProperty("storage") FileSystemConfig storagePluginConfig, - @JsonProperty("format") MapRDBFormatPluginConfig formatPluginConfig, - @JsonProperty("columns") List columns, - @JsonProperty("schema") TupleMetadata schema, - @JacksonInject StoragePluginRegistry pluginRegistry) throws ExecutionSetupException { - this(userName, pluginRegistry.resolve(storagePluginConfig, AbstractStoragePlugin.class), - pluginRegistry.resolveFormat(storagePluginConfig, formatPluginConfig, MapRDBFormatPlugin.class), - scanSpec, columns, null /* tableStats */, FileSystemMetadataProviderManager.getMetadataProviderForSchema(schema)); - } - - public BinaryTableGroupScan(String userName, AbstractStoragePlugin storagePlugin, - MapRDBFormatPlugin formatPlugin, HBaseScanSpec scanSpec, List columns, - MetadataProviderManager metadataProviderManager) { - this(userName, storagePlugin, formatPlugin, scanSpec, - columns, null /* tableStats */, FileSystemMetadataProviderManager.getMetadataProvider(metadataProviderManager)); - } - - public BinaryTableGroupScan(String userName, AbstractStoragePlugin storagePlugin, - MapRDBFormatPlugin formatPlugin, HBaseScanSpec scanSpec, - List columns, MapRDBTableStats tableStats, TableMetadataProvider metadataProvider) { - super(storagePlugin, formatPlugin, columns, userName, metadataProvider); - this.hbaseScanSpec = scanSpec; - this.tableStats = tableStats; - init(); - } - - /** - * Private constructor, used for cloning. - * @param that The HBaseGroupScan to clone - */ - private BinaryTableGroupScan(BinaryTableGroupScan that) { - super(that); - this.hbaseScanSpec = that.hbaseScanSpec; - this.endpointFragmentMapping = that.endpointFragmentMapping; - this.hTableDesc = that.hTableDesc; - this.tableStats = that.tableStats; - } - - @Override - public GroupScan clone(List columns) { - BinaryTableGroupScan newScan = new BinaryTableGroupScan(this); - newScan.columns = columns == null ? ALL_COLUMNS : columns; - HBaseUtils.verifyColumns(columns, hTableDesc); - return newScan; - } - - private void init() { - logger.debug("Getting region locations"); - TableName tableName = TableName.valueOf(hbaseScanSpec.getTableName()); - try (Admin admin = formatPlugin.getConnection().getAdmin(); - RegionLocator locator = formatPlugin.getConnection().getRegionLocator(tableName)) { - hTableDesc = admin.getTableDescriptor(tableName); - // Fetch tableStats only once and cache it. - if (tableStats == null) { - tableStats = new MapRDBTableStats(getHBaseConf(), hbaseScanSpec.getTableName()); - } - boolean foundStartRegion = false; - final TreeMap regionsToScan = new TreeMap<>(); - List regionLocations = locator.getAllRegionLocations(); - for (HRegionLocation regionLocation : regionLocations) { - HRegionInfo regionInfo = regionLocation.getRegionInfo(); - if (!foundStartRegion && hbaseScanSpec.getStartRow() != null && hbaseScanSpec.getStartRow().length != 0 && !regionInfo.containsRow(hbaseScanSpec.getStartRow())) { - continue; - } - foundStartRegion = true; - regionsToScan.put(new TabletFragmentInfo(regionInfo), regionLocation.getHostname()); - if (hbaseScanSpec.getStopRow() != null && hbaseScanSpec.getStopRow().length != 0 && regionInfo.containsRow(hbaseScanSpec.getStopRow())) { - break; - } - } - setRegionsToScan(regionsToScan); - } catch (Exception e) { - throw new DrillRuntimeException("Error getting region info for table: " + hbaseScanSpec.getTableName(), e); - } - HBaseUtils.verifyColumns(columns, hTableDesc); - } - - - @Override - protected MapRDBSubScanSpec getSubScanSpec(TabletFragmentInfo tfi) { - HBaseScanSpec spec = hbaseScanSpec; - return new MapRDBSubScanSpec( - spec.getTableName(), - null /* indexFid */, - getRegionsToScan().get(tfi), - (!isNullOrEmpty(spec.getStartRow()) && tfi.containsRow(spec.getStartRow())) ? spec.getStartRow() : tfi.getStartKey(), - (!isNullOrEmpty(spec.getStopRow()) && tfi.containsRow(spec.getStopRow())) ? spec.getStopRow() : tfi.getEndKey(), - spec.getSerializedFilter(), - null, - getUserName()); - } - - @Override - public MapRDBSubScan getSpecificScan(int minorFragmentId) { - assert minorFragmentId < endpointFragmentMapping.size() : String.format( - "Mappings length [%d] should be greater than minor fragment id [%d] but it isn't.", endpointFragmentMapping.size(), - minorFragmentId); - return new MapRDBSubScan(getUserName(), formatPlugin, endpointFragmentMapping.get(minorFragmentId), columns, TABLE_BINARY, getSchema()); - } - - @Override - public ScanStats getScanStats() { - //TODO: look at stats for this. - long rowCount = (long) ((hbaseScanSpec.getFilter() != null ? .5 : 1) * tableStats.getNumRows()); - int avgColumnSize = 10; - int numColumns = (columns == null || columns.isEmpty()) ? 100 : columns.size(); - return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, rowCount, 1, avgColumnSize * numColumns * rowCount); - } - - @Override - @JsonIgnore - public PhysicalOperator getNewWithChildren(List children) { - Preconditions.checkArgument(children.isEmpty()); - return new BinaryTableGroupScan(this); - } - - @JsonIgnore - public Configuration getHBaseConf() { - return getFormatPlugin().getHBaseConf(); - } - - @Override - @JsonIgnore - public String getTableName() { - return getHBaseScanSpec().getTableName(); - } - - @JsonIgnore - public MapRDBTableStats getTableStats() { - return tableStats; - } - - @Override - public String toString() { - return "BinaryTableGroupScan [ScanSpec=" - + hbaseScanSpec + ", columns=" - + columns + "]"; - } - - @JsonProperty - public HBaseScanSpec getHBaseScanSpec() { - return hbaseScanSpec; - } - - @Override - public void setRowCount(RexNode condition, double count, double capRowCount) { - throw new UnsupportedOperationException("setRowCount() not implemented for BinaryTableGroupScan"); - } - - @Override - public double getRowCount(RexNode condition, RelNode scanRel) { - return Statistics.ROWCOUNT_UNKNOWN; - } - - @Override - public Statistics getStatistics() { - throw new UnsupportedOperationException("getStatistics() not implemented for BinaryTableGroupScan"); - } - - @Override - @JsonIgnore - public boolean isIndexScan() { - return false; - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBBinaryTable.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBBinaryTable.java deleted file mode 100644 index 23aa43b2953..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBBinaryTable.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.binary; - - -import org.apache.drill.exec.store.dfs.FileSystemPlugin; -import org.apache.drill.exec.store.dfs.FormatSelection; -import org.apache.drill.exec.store.hbase.AbstractHBaseDrillTable; -import org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin; - -public class MapRDBBinaryTable extends AbstractHBaseDrillTable { - - public MapRDBBinaryTable(String storageEngineName, FileSystemPlugin storagePlugin, MapRDBFormatPlugin formatPlugin, - FormatSelection selection) { - super(storageEngineName, storagePlugin, selection); - setTableDesc(formatPlugin.getConnection(), formatPlugin.getTableName(selection.getSelection())); - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java deleted file mode 100644 index 996286c9ead..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.binary; - -import java.util.Arrays; -import java.util.List; - -import org.apache.drill.common.FunctionNames; -import org.apache.drill.common.expression.BooleanOperator; -import org.apache.drill.common.expression.FunctionCall; -import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.expression.visitors.AbstractExprVisitor; -import org.apache.drill.exec.store.hbase.DrillHBaseConstants; -import org.apache.drill.exec.store.hbase.HBaseRegexParser; -import org.apache.drill.exec.store.hbase.HBaseScanSpec; -import org.apache.drill.exec.store.hbase.HBaseUtils; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.filter.BinaryComparator; -import org.apache.hadoop.hbase.filter.ByteArrayComparable; -import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; -import org.apache.hadoop.hbase.filter.Filter; -import org.apache.hadoop.hbase.filter.NullComparator; -import org.apache.hadoop.hbase.filter.RegexStringComparator; -import org.apache.hadoop.hbase.filter.RowFilter; -import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; - -public class MapRDBFilterBuilder extends AbstractExprVisitor implements DrillHBaseConstants { - - final private BinaryTableGroupScan groupScan; - - final private LogicalExpression le; - - private boolean allExpressionsConverted = true; - - private static Boolean nullComparatorSupported; - - public MapRDBFilterBuilder(BinaryTableGroupScan groupScan, LogicalExpression le) { - this.groupScan = groupScan; - this.le = le; - } - - public HBaseScanSpec parseTree() { - HBaseScanSpec parsedSpec = le.accept(this, null); - if (parsedSpec != null) { - parsedSpec = mergeScanSpecs(FunctionNames.AND, this.groupScan.getHBaseScanSpec(), parsedSpec); - /* - * If RowFilter is THE filter attached to the scan specification, - * remove it since its effect is also achieved through startRow and stopRow. - */ - Filter filter = parsedSpec.getFilter(); - if (filter instanceof RowFilter && - ((RowFilter)filter).getOperator() != CompareOp.NOT_EQUAL && - ((RowFilter)filter).getComparator() instanceof BinaryComparator) { - parsedSpec = new HBaseScanSpec(parsedSpec.getTableName(), parsedSpec.getStartRow(), parsedSpec.getStopRow(), null); - } - } - return parsedSpec; - } - - public boolean isAllExpressionsConverted() { - return allExpressionsConverted; - } - - @Override - public HBaseScanSpec visitUnknown(LogicalExpression e, Void value) throws RuntimeException { - allExpressionsConverted = false; - return null; - } - - @Override - public HBaseScanSpec visitBooleanOperator(BooleanOperator op, Void value) throws RuntimeException { - return visitFunctionCall(op, value); - } - - @Override - public HBaseScanSpec visitFunctionCall(FunctionCall call, Void value) throws RuntimeException { - HBaseScanSpec nodeScanSpec = null; - String functionName = call.getName(); - List args = call.args(); - - if (MaprDBCompareFunctionsProcessor.isCompareFunction(functionName)) { - /* - * HBASE-10848: Bug in HBase versions (0.94.[0-18], 0.96.[0-2], 0.98.[0-1]) - * causes a filter with NullComparator to fail. Enable only if specified in - * the configuration (after ensuring that the HBase cluster has the fix). - */ - if (nullComparatorSupported == null) { - nullComparatorSupported = groupScan.getHBaseConf().getBoolean("drill.hbase.supports.null.comparator", false); - } - - MaprDBCompareFunctionsProcessor processor = MaprDBCompareFunctionsProcessor.createFunctionsProcessorInstance(call, nullComparatorSupported); - if (processor.isSuccess()) { - nodeScanSpec = createHBaseScanSpec(call, processor); - } - } else { - switch (functionName) { - case FunctionNames.AND: - case FunctionNames.OR: - HBaseScanSpec firstScanSpec = args.get(0).accept(this, null); - for (int i = 1; i < args.size(); ++i) { - HBaseScanSpec nextScanSpec = args.get(i).accept(this, null); - if (firstScanSpec != null && nextScanSpec != null) { - nodeScanSpec = mergeScanSpecs(functionName, firstScanSpec, nextScanSpec); - } else { - allExpressionsConverted = false; - if (FunctionNames.AND.equals(functionName)) { - nodeScanSpec = firstScanSpec == null ? nextScanSpec : firstScanSpec; - } - } - firstScanSpec = nodeScanSpec; - } - break; - } - } - - if (nodeScanSpec == null) { - allExpressionsConverted = false; - } - - return nodeScanSpec; - } - - private HBaseScanSpec mergeScanSpecs(String functionName, HBaseScanSpec leftScanSpec, HBaseScanSpec rightScanSpec) { - Filter newFilter = null; - byte[] startRow = HConstants.EMPTY_START_ROW; - byte[] stopRow = HConstants.EMPTY_END_ROW; - - switch (functionName) { - case FunctionNames.AND: - newFilter = HBaseUtils.andFilterAtIndex(leftScanSpec.getFilter(), -1, rightScanSpec.getFilter()); //HBaseUtils.LAST_FILTER - startRow = HBaseUtils.maxOfStartRows(leftScanSpec.getStartRow(), rightScanSpec.getStartRow()); - stopRow = HBaseUtils.minOfStopRows(leftScanSpec.getStopRow(), rightScanSpec.getStopRow()); - break; - case FunctionNames.OR: - newFilter = HBaseUtils.orFilterAtIndex(leftScanSpec.getFilter(), -1, rightScanSpec.getFilter()); //HBaseUtils.LAST_FILTER - startRow = HBaseUtils.minOfStartRows(leftScanSpec.getStartRow(), rightScanSpec.getStartRow()); - stopRow = HBaseUtils.maxOfStopRows(leftScanSpec.getStopRow(), rightScanSpec.getStopRow()); - } - return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, newFilter); - } - - private HBaseScanSpec createHBaseScanSpec(FunctionCall call, MaprDBCompareFunctionsProcessor processor) { - String functionName = processor.getFunctionName(); - SchemaPath field = processor.getPath(); - byte[] fieldValue = processor.getValue(); - boolean sortOrderAscending = processor.isSortOrderAscending(); - boolean isRowKey = field.getRootSegmentPath().equals(ROW_KEY); - if (!(isRowKey - || (!field.getRootSegment().isLastPath() - && field.getRootSegment().getChild().isLastPath() - && field.getRootSegment().getChild().isNamed()) - ) - ) { - /* - * if the field in this function is neither the row_key nor a qualified HBase column, return. - */ - return null; - } - - if (processor.isRowKeyPrefixComparison()) { - return createRowKeyPrefixScanSpec(call, processor); - } - - CompareOp compareOp = null; - boolean isNullTest = false; - ByteArrayComparable comparator = new BinaryComparator(fieldValue); - byte[] startRow = HConstants.EMPTY_START_ROW; - byte[] stopRow = HConstants.EMPTY_END_ROW; - switch (functionName) { - case FunctionNames.EQ: - compareOp = CompareOp.EQUAL; - if (isRowKey) { - startRow = fieldValue; - /* stopRow should be just greater than 'value'*/ - stopRow = Arrays.copyOf(fieldValue, fieldValue.length+1); - compareOp = CompareOp.EQUAL; - } - break; - case FunctionNames.NE: - compareOp = CompareOp.NOT_EQUAL; - break; - case FunctionNames.GE: - if (sortOrderAscending) { - compareOp = CompareOp.GREATER_OR_EQUAL; - if (isRowKey) { - startRow = fieldValue; - } - } else { - compareOp = CompareOp.LESS_OR_EQUAL; - if (isRowKey) { - // stopRow should be just greater than 'value' - stopRow = Arrays.copyOf(fieldValue, fieldValue.length+1); - } - } - break; - case FunctionNames.GT: - if (sortOrderAscending) { - compareOp = CompareOp.GREATER; - if (isRowKey) { - // startRow should be just greater than 'value' - startRow = Arrays.copyOf(fieldValue, fieldValue.length+1); - } - } else { - compareOp = CompareOp.LESS; - if (isRowKey) { - stopRow = fieldValue; - } - } - break; - case FunctionNames.LE: - if (sortOrderAscending) { - compareOp = CompareOp.LESS_OR_EQUAL; - if (isRowKey) { - // stopRow should be just greater than 'value' - stopRow = Arrays.copyOf(fieldValue, fieldValue.length+1); - } - } else { - compareOp = CompareOp.GREATER_OR_EQUAL; - if (isRowKey) { - startRow = fieldValue; - } - } - break; - case FunctionNames.LT: - if (sortOrderAscending) { - compareOp = CompareOp.LESS; - if (isRowKey) { - stopRow = fieldValue; - } - } else { - compareOp = CompareOp.GREATER; - if (isRowKey) { - // startRow should be just greater than 'value' - startRow = Arrays.copyOf(fieldValue, fieldValue.length+1); - } - } - break; - case FunctionNames.IS_NULL: - case "isNull": - case "is null": - if (isRowKey) { - return null; - } - isNullTest = true; - compareOp = CompareOp.EQUAL; - comparator = new NullComparator(); - break; - case FunctionNames.IS_NOT_NULL: - case "isNotNull": - case "is not null": - if (isRowKey) { - return null; - } - compareOp = CompareOp.NOT_EQUAL; - comparator = new NullComparator(); - break; - case "like": - /* - * Convert the LIKE operand to Regular Expression pattern so that we can - * apply RegexStringComparator() - */ - HBaseRegexParser parser = new HBaseRegexParser(call).parse(); - compareOp = CompareOp.EQUAL; - comparator = new RegexStringComparator(parser.getRegexString()); - - /* - * We can possibly do better if the LIKE operator is on the row_key - */ - if (isRowKey) { - String prefix = parser.getPrefixString(); - if (prefix != null) { // group 3 is literal - /* - * If there is a literal prefix, it can help us prune the scan to a sub range - */ - if (prefix.equals(parser.getLikeString())) { - /* The operand value is literal. This turns the LIKE operator to EQUAL operator */ - startRow = stopRow = fieldValue; - compareOp = null; - } else { - startRow = prefix.getBytes(StandardCharsets.UTF_8); - stopRow = startRow.clone(); - boolean isMaxVal = true; - for (int i = stopRow.length - 1; i >= 0; --i) { - int nextByteValue = (0xff & stopRow[i]) + 1; - if (nextByteValue < 0xff) { - stopRow[i] = (byte) nextByteValue; - isMaxVal = false; - break; - } else { - stopRow[i] = 0; - } - } - if (isMaxVal) { - stopRow = HConstants.EMPTY_END_ROW; - } - } - } - } - break; - } - - if (compareOp != null || startRow != HConstants.EMPTY_START_ROW || stopRow != HConstants.EMPTY_END_ROW) { - Filter filter = null; - if (isRowKey) { - if (compareOp != null) { - filter = new RowFilter(compareOp, comparator); - } - } else { - byte[] family = HBaseUtils.getBytes(field.getRootSegment().getPath()); - byte[] qualifier = HBaseUtils.getBytes(field.getRootSegment().getChild().getNameSegment().getPath()); - filter = new SingleColumnValueFilter(family, qualifier, compareOp, comparator); - ((SingleColumnValueFilter)filter).setLatestVersionOnly(true); - if (!isNullTest) { - ((SingleColumnValueFilter)filter).setFilterIfMissing(true); - } - } - return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, filter); - } - // else - return null; - } - - private HBaseScanSpec createRowKeyPrefixScanSpec(FunctionCall call, - MaprDBCompareFunctionsProcessor processor) { - byte[] startRow = processor.getRowKeyPrefixStartRow(); - byte[] stopRow = processor.getRowKeyPrefixStopRow(); - Filter filter = processor.getRowKeyPrefixFilter(); - - if (startRow != HConstants.EMPTY_START_ROW || - stopRow != HConstants.EMPTY_END_ROW || - filter != null) { - return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, filter); - } - - // else - return null; - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MaprDBCompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MaprDBCompareFunctionsProcessor.java deleted file mode 100644 index c5b237d8745..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MaprDBCompareFunctionsProcessor.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.binary; - -import io.netty.buffer.ByteBuf; -import org.apache.drill.common.expression.FunctionCall; -import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.ValueExpressions; -import org.apache.drill.exec.store.hbase.CompareFunctionsProcessor; -import org.apache.hadoop.hbase.util.Order; -import org.apache.hadoop.hbase.util.PositionedByteRange; -import org.apache.hadoop.hbase.util.SimplePositionedMutableByteRange; - -class MaprDBCompareFunctionsProcessor extends CompareFunctionsProcessor { - - public MaprDBCompareFunctionsProcessor(String functionName) { - super(functionName); - } - - public static MaprDBCompareFunctionsProcessor createFunctionsProcessorInstance(FunctionCall call, boolean nullComparatorSupported) { - String functionName = call.getName(); - MaprDBCompareFunctionsProcessor evaluator = new MaprDBCompareFunctionsProcessor(functionName); - - return createFunctionsProcessorInstanceInternal(call, nullComparatorSupported, evaluator); - } - - @Override - protected ByteBuf getByteBuf(LogicalExpression valueArg, String encodingType) { - switch (encodingType) { - case "UTF8_OB": - case "UTF8_OBD": - if (valueArg instanceof ValueExpressions.QuotedString) { - int stringLen = ((ValueExpressions.QuotedString) valueArg).value.getBytes(StandardCharsets.UTF_8).length; - ByteBuf bb = newByteBuf(stringLen + 2, true); - PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, stringLen + 2); - if (encodingType.endsWith("_OBD")) { - org.apache.hadoop.hbase.util.OrderedBytes.encodeString(br, ((ValueExpressions.QuotedString) valueArg).value, - Order.DESCENDING); - setSortOrderAscending(false); - } else { - org.apache.hadoop.hbase.util.OrderedBytes.encodeString(br, ((ValueExpressions.QuotedString) valueArg).value, - Order.ASCENDING); - } - return bb; - } - } - return null; - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/AllTextValueWriter.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/AllTextValueWriter.java deleted file mode 100644 index bbb2aec04ea..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/AllTextValueWriter.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import java.nio.ByteBuffer; - -import org.apache.drill.exec.vector.complex.impl.MapOrListWriterImpl; -import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapOrListWriter; -import org.ojai.DocumentReader; - -import com.mapr.org.apache.hadoop.hbase.util.Bytes; - -import io.netty.buffer.DrillBuf; - -public class AllTextValueWriter extends OjaiValueWriter { - - public AllTextValueWriter(DrillBuf buffer) { - super(buffer); - } - - protected void writeTimeStamp(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writeString(writer, fieldName, reader.getTimestamp().toUTCString()); - } - - protected void writeTime(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writeString(writer, fieldName, reader.getTime().toTimeStr()); - } - - protected void writeDate(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writeString(writer, fieldName, reader.getDate().toDateStr()); - } - - protected void writeDouble(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writeString(writer, fieldName, String.valueOf(reader.getDouble())); - } - - protected void writeFloat(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writeString(writer, fieldName, String.valueOf(reader.getFloat())); - } - - protected void writeLong(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writeString(writer, fieldName, String.valueOf(reader.getLong())); - } - - protected void writeInt(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writeString(writer, fieldName, String.valueOf(reader.getInt())); - } - - protected void writeShort(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writeString(writer, fieldName, String.valueOf(reader.getShort())); - } - - protected void writeByte(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writeString(writer, fieldName, String.valueOf(reader.getByte())); - } - - protected void writeBoolean(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writeString(writer, fieldName, String.valueOf(reader.getBoolean())); - } - - protected void writeBinary(MapOrListWriter writer, String fieldName, ByteBuffer buf) { - writeString(writer, fieldName, Bytes.toString(buf)); - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java deleted file mode 100644 index cc56bc35b6c..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import static com.mapr.db.rowcol.DBValueBuilderImpl.KeyValueBuilder; - -import org.apache.drill.common.FunctionNames; -import org.apache.drill.common.expression.FunctionCall; -import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; -import org.apache.drill.common.expression.ValueExpressions.DateExpression; -import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression; -import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression; -import org.apache.drill.common.expression.ValueExpressions.DoubleExpression; -import org.apache.drill.common.expression.ValueExpressions.FloatExpression; -import org.apache.drill.common.expression.ValueExpressions.IntExpression; -import org.apache.drill.common.expression.ValueExpressions.LongExpression; -import org.apache.drill.common.expression.ValueExpressions.QuotedString; -import org.apache.drill.common.expression.ValueExpressions.TimeExpression; -import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression; -import org.apache.drill.common.expression.ValueExpressions.VarDecimalExpression; -import org.apache.drill.common.expression.visitors.AbstractExprVisitor; -import org.joda.time.LocalTime; -import org.ojai.Value; -import org.ojai.types.ODate; -import org.ojai.types.OTime; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.mapr.db.util.SqlHelper; - -import org.ojai.types.OTimestamp; - -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; - -class CompareFunctionsProcessor extends AbstractExprVisitor { - - private String functionName; - private Boolean success; - protected Value value; - protected SchemaPath path; - - public CompareFunctionsProcessor(String functionName) { - this.functionName = functionName; - this.success = false; - this.value = null; - } - - public static boolean isCompareFunction(String functionName) { - return COMPARE_FUNCTIONS_TRANSPOSE_MAP.keySet().contains(functionName); - } - - @Override - public Boolean visitUnknown(LogicalExpression e, LogicalExpression valueArg) throws RuntimeException { - return false; - } - - /** - * Converts specified function call to be pushed into maprDB JSON scan. - * - * @param call function call to be pushed - * @return CompareFunctionsProcessor instance which contains converted function call - */ - public static CompareFunctionsProcessor process(FunctionCall call) { - return processWithEvaluator(call, new CompareFunctionsProcessor(call.getName())); - } - - /** - * Converts specified function call to be pushed into maprDB JSON scan. - * For the case when timestamp value is used, it is converted to UTC timezone - * before converting to {@link OTimestamp} instance. - * - * @param call function call to be pushed - * @return CompareFunctionsProcessor instance which contains converted function call - */ - public static CompareFunctionsProcessor processWithTimeZoneOffset(FunctionCall call) { - CompareFunctionsProcessor processor = new CompareFunctionsProcessor(call.getName()) { - @Override - protected boolean visitTimestampExpr(SchemaPath path, TimeStampExpression valueArg) { - // converts timestamp value from local time zone to UTC since the record reader - // reads the timestamp in local timezone if the readTimestampWithZoneOffset flag is enabled - Instant localInstant = Instant.ofEpochMilli(valueArg.getTimeStamp()); - ZonedDateTime utcZonedDateTime = localInstant.atZone(ZoneId.of("UTC")); - ZonedDateTime convertedZonedDateTime = utcZonedDateTime.withZoneSameLocal(ZoneId.systemDefault()); - long timeStamp = convertedZonedDateTime.toInstant().toEpochMilli(); - - this.value = KeyValueBuilder.initFrom(new OTimestamp(timeStamp)); - this.path = path; - return true; - } - }; - return processWithEvaluator(call, processor); - } - - private static CompareFunctionsProcessor processWithEvaluator(FunctionCall call, CompareFunctionsProcessor evaluator) { - String functionName = call.getName(); - LogicalExpression nameArg = call.arg(0); - LogicalExpression valueArg = call.argCount() >= 2 ? call.arg(1) : null; - - if (VALUE_EXPRESSION_CLASSES.contains(nameArg.getClass())) { - LogicalExpression swapArg = valueArg; - valueArg = nameArg; - nameArg = swapArg; - evaluator.functionName = COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(functionName); - } - if (nameArg != null) { - evaluator.success = nameArg.accept(evaluator, valueArg); - } - - return evaluator; - } - - public boolean isSuccess() { - // TODO Auto-generated method stub - return success; - } - - public SchemaPath getPath() { - return path; - } - - public Value getValue() { - return value; - } - - public String getFunctionName() { - return functionName; - } - - @Override - public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) throws RuntimeException { - // If valueArg is null, this might be a IS NULL/IS NOT NULL type of query - if (valueArg == null) { - this.path = path; - return true; - } - - if (valueArg instanceof QuotedString) { - this.value = SqlHelper.decodeStringAsValue(((QuotedString) valueArg).value); - this.path = path; - return true; - } - - if (valueArg instanceof IntExpression) { - this.value = KeyValueBuilder.initFrom(((IntExpression)valueArg).getInt()); - this.path = path; - return true; - } - - if (valueArg instanceof FloatExpression) { - this.value = KeyValueBuilder.initFrom(((FloatExpression)valueArg).getFloat()); - this.path = path; - return true; - } - - if (valueArg instanceof BooleanExpression) { - this.value = KeyValueBuilder.initFrom(((BooleanExpression)valueArg).getBoolean()); - this.path = path; - return true; - } - - if (valueArg instanceof Decimal28Expression) { - this.value = KeyValueBuilder.initFrom(((Decimal28Expression)valueArg).getBigDecimal()); - this.path = path; - return true; - } - - if (valueArg instanceof Decimal38Expression) { - this.value = KeyValueBuilder.initFrom(((Decimal38Expression)valueArg).getBigDecimal()); - this.path = path; - return true; - } - - if (valueArg instanceof DoubleExpression) { - this.value = KeyValueBuilder.initFrom(((DoubleExpression)valueArg).getDouble()); - this.path = path; - return true; - } - - if (valueArg instanceof LongExpression) { - this.value = KeyValueBuilder.initFrom(((LongExpression)valueArg).getLong()); - this.path = path; - return true; - } - - if (valueArg instanceof DateExpression) { - long d = ((DateExpression)valueArg).getDate(); - final long MILLISECONDS_IN_A_DAY = (long)1000 * 60 * 60 * 24; - int daysSinceEpoch = (int)(d / MILLISECONDS_IN_A_DAY); - this.value = KeyValueBuilder.initFrom(ODate.fromDaysSinceEpoch(daysSinceEpoch)); - this.path = path; - return true; - } - - if (valueArg instanceof TimeExpression) { - int t = ((TimeExpression)valueArg).getTime(); - LocalTime lT = LocalTime.fromMillisOfDay(t); - this.value = KeyValueBuilder.initFrom(new OTime(lT.getHourOfDay(), lT.getMinuteOfHour(), lT.getSecondOfMinute(), lT.getMillisOfSecond())); - this.path = path; - return true; - } - - // MaprDB does not support decimals completely, therefore double value is used. - // See com.mapr.db.impl.ConditionImpl.is(FieldPath path, QueryCondition.Op op, BigDecimal value) method - if (valueArg instanceof VarDecimalExpression) { - this.value = KeyValueBuilder.initFrom(((VarDecimalExpression) valueArg).getBigDecimal().doubleValue()); - this.path = path; - return true; - } - - if (valueArg instanceof TimeStampExpression) { - return visitTimestampExpr(path, (TimeStampExpression) valueArg); - } - return false; - } - - protected boolean visitTimestampExpr(SchemaPath path, TimeStampExpression valueArg) { - this.value = KeyValueBuilder.initFrom(new OTimestamp(valueArg.getTimeStamp())); - this.path = path; - return true; - } - - private static final ImmutableSet> VALUE_EXPRESSION_CLASSES; - static { - ImmutableSet.Builder> builder = ImmutableSet.builder(); - VALUE_EXPRESSION_CLASSES = builder - .add(BooleanExpression.class) - .add(DateExpression.class) - .add(DoubleExpression.class) - .add(FloatExpression.class) - .add(IntExpression.class) - .add(LongExpression.class) - .add(QuotedString.class) - .add(TimeExpression.class) - .add(VarDecimalExpression.class) - .build(); - } - - private static final ImmutableMap COMPARE_FUNCTIONS_TRANSPOSE_MAP; - static { - ImmutableMap.Builder builder = ImmutableMap.builder(); - COMPARE_FUNCTIONS_TRANSPOSE_MAP = builder - // unary functions - .put(FunctionNames.IS_NOT_NULL, FunctionNames.IS_NOT_NULL) - .put("isNotNull", "isNotNull") - .put("is not null", "is not null") - .put(FunctionNames.IS_NULL, FunctionNames.IS_NULL) - .put("isNull", "isNull") - .put("is null", "is null") - // binary functions - .put("like", "like") - .put(FunctionNames.EQ, FunctionNames.EQ) - .put(FunctionNames.NE, FunctionNames.NE) - .put(FunctionNames.GE, FunctionNames.LE) - .put(FunctionNames.GT, FunctionNames.LT) - .put(FunctionNames.LE, FunctionNames.GE) - .put(FunctionNames.LT, FunctionNames.GT) - .build(); - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/DocumentReaderVectorWriter.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/DocumentReaderVectorWriter.java deleted file mode 100644 index 1c9deadc0c3..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/DocumentReaderVectorWriter.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.mapr.db.ojai.DBDocumentReaderBase; - -/** - * Base class for writing a single OJAI Document to Drill's Value Vectors. - */ -abstract class DocumentReaderVectorWriter { - protected static final Logger logger = LoggerFactory.getLogger(DocumentReaderVectorWriter.class); - - protected final OjaiValueWriter valueWriter; - - protected DocumentReaderVectorWriter(final OjaiValueWriter valueWriter) { - this.valueWriter = valueWriter; - } - - protected abstract void writeDBDocument(final VectorContainerWriter writer, final DBDocumentReaderBase reader) - throws SchemaChangeException; - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/FieldPathHelper.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/FieldPathHelper.java deleted file mode 100644 index d8d379295e2..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/FieldPathHelper.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import java.util.Stack; - -import org.apache.drill.common.expression.PathSegment; -import org.apache.drill.common.expression.SchemaPath; -import org.ojai.FieldPath; -import org.ojai.FieldSegment; - -public class FieldPathHelper { - - /** - * Returns {@link SchemaPath} equivalent of the specified {@link FieldPath}. - */ - public static SchemaPath fieldPath2SchemaPath(FieldPath fieldPath) { - Stack fieldSegments = new Stack(); - FieldSegment seg = fieldPath.getRootSegment(); - while (seg != null) { - fieldSegments.push(seg); - seg = seg.getChild(); - } - - PathSegment child = null; - while (!fieldSegments.isEmpty()) { - seg = fieldSegments.pop(); - if (seg.isNamed()) { - child = new PathSegment.NameSegment(((FieldSegment.NameSegment)seg).getName(), child); - } else { - child = new PathSegment.ArraySegment(((FieldSegment.IndexSegment)seg).getIndex(), child); - } - } - return new SchemaPath((PathSegment.NameSegment)child); - } - - /** - * Returns {@link FieldPath} equivalent of the specified {@link SchemaPath}. - */ - public static FieldPath schemaPath2FieldPath(SchemaPath column) { - Stack pathSegments = new Stack(); - PathSegment seg = column.getRootSegment(); - while (seg != null) { - pathSegments.push(seg); - seg = seg.getChild(); - } - - FieldSegment child = null; - while (!pathSegments.isEmpty()) { - seg = pathSegments.pop(); - if (seg.isNamed()) { - child = new FieldSegment.NameSegment(((PathSegment.NameSegment)seg).getPath(), child, false); - } else { - child = new FieldSegment.IndexSegment(((PathSegment.ArraySegment)seg).getIndex(), child); - } - } - return new FieldPath((FieldSegment.NameSegment) child); - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/FieldTransferVectorWriter.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/FieldTransferVectorWriter.java deleted file mode 100644 index 67bbcb3c40b..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/FieldTransferVectorWriter.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import static org.apache.drill.exec.store.mapr.PluginErrorHandler.dataReadError; - -import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.vector.complex.impl.MapOrListWriterImpl; -import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; -import org.ojai.DocumentReader.EventType; - -import com.mapr.db.ojai.DBDocumentReaderBase; - -/** - * This implementation of DocumentReaderVectorWriter does field by field transfer the OJAI Document - * to Drill Value Vectors. - */ -class FieldTransferVectorWriter extends DocumentReaderVectorWriter { - - protected FieldTransferVectorWriter(final OjaiValueWriter valueWriter) { - super(valueWriter); - } - - @Override - protected void writeDBDocument(VectorContainerWriter vectorWriter, DBDocumentReaderBase reader) - throws SchemaChangeException { - MapOrListWriterImpl writer = new MapOrListWriterImpl(vectorWriter.rootAsMap()); - if (reader.next() != EventType.START_MAP) { - throw dataReadError(logger, "The document did not start with START_MAP!"); - } - valueWriter.writeToListOrMap(writer, reader); - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/IdOnlyVectorWriter.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/IdOnlyVectorWriter.java deleted file mode 100644 index 2c9f7621673..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/IdOnlyVectorWriter.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import static org.apache.drill.exec.store.mapr.PluginErrorHandler.schemaChangeException; -import static org.ojai.DocumentConstants.ID_KEY; - -import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; -import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; -import org.ojai.Value; - -import com.mapr.db.impl.IdCodec; -import com.mapr.db.ojai.DBDocumentReaderBase; - -/** - * This implementation of DocumentReaderVectorWriter writes only the "_id" field from the OJAI - * Document to Drill Value Vectors. This is useful for "_id" only queries. - */ -class IdOnlyVectorWriter extends DocumentReaderVectorWriter { - - protected IdOnlyVectorWriter(final OjaiValueWriter valueWriter) { - super(valueWriter); - } - - @Override - public void writeDBDocument(VectorContainerWriter vectorWriter, DBDocumentReaderBase reader) - throws SchemaChangeException { - MapWriter writer = vectorWriter.rootAsMap(); - - Value id = reader.getId(); - try { - switch(id.getType()) { - case STRING: - valueWriter.writeString(writer, ID_KEY, id.getString()); - break; - case BINARY: - valueWriter.writeBinary(writer, ID_KEY, id.getBinary()); - break; - default: - throw new UnsupportedOperationException(id.getType() + - " is not a supported type for _id field."); - } - } catch (IllegalStateException | IllegalArgumentException e) { - throw schemaChangeException(logger, e, "Possible schema change at _id: '%s'", IdCodec.asString(id)); - } - - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java deleted file mode 100644 index 1e93cf70901..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import java.util.List; - -import org.apache.drill.common.FunctionNames; -import org.apache.drill.common.expression.BooleanOperator; -import org.apache.drill.common.expression.FunctionCall; -import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.expression.visitors.AbstractExprVisitor; -import org.apache.drill.exec.store.hbase.DrillHBaseConstants; -import org.ojai.Value; -import org.ojai.store.QueryCondition; -import org.ojai.store.QueryCondition.Op; - -import com.mapr.db.impl.MapRDBImpl; - -public class JsonConditionBuilder extends AbstractExprVisitor implements DrillHBaseConstants { - - final private JsonTableGroupScan groupScan; - - final private LogicalExpression le; - - private boolean allExpressionsConverted = true; - - public JsonConditionBuilder(JsonTableGroupScan groupScan, - LogicalExpression conditionExp) { - this.groupScan = groupScan; - this.le = conditionExp; - } - - public JsonScanSpec parseTree() { - JsonScanSpec parsedSpec = le.accept(this, null); - if (parsedSpec != null) { - parsedSpec.mergeScanSpec(FunctionNames.AND, this.groupScan.getScanSpec()); - } - return parsedSpec; - } - - public boolean isAllExpressionsConverted() { - // TODO Auto-generated method stub - return allExpressionsConverted; - } - - @Override - public JsonScanSpec visitSchemaPath(SchemaPath path, Void value) throws RuntimeException { - String fieldPath = FieldPathHelper.schemaPath2FieldPath(path).asPathString(); - QueryCondition cond = MapRDBImpl.newCondition().is(fieldPath, Op.EQUAL, true); - return new JsonScanSpec(groupScan.getTableName(), - groupScan.getIndexDesc(), - cond.build()); - } - - @Override - public JsonScanSpec visitUnknown(LogicalExpression e, Void value) throws RuntimeException { - allExpressionsConverted = false; - return null; - } - - @Override - public JsonScanSpec visitBooleanOperator(BooleanOperator op, Void value) throws RuntimeException { - return visitFunctionCall(op, value); - } - - @Override - public JsonScanSpec visitFunctionCall(FunctionCall call, Void value) throws RuntimeException { - JsonScanSpec nodeScanSpec = null; - String functionName = call.getName(); - List args = call.args(); - - if (CompareFunctionsProcessor.isCompareFunction(functionName)) { - CompareFunctionsProcessor processor; - if (groupScan.getFormatPlugin().getConfig().isReadTimestampWithZoneOffset()) { - processor = CompareFunctionsProcessor.processWithTimeZoneOffset(call); - } else { - processor = CompareFunctionsProcessor.process(call); - } - if (processor.isSuccess()) { - nodeScanSpec = createJsonScanSpec(call, processor); - } - } else { - switch(functionName) { - case FunctionNames.AND: - case FunctionNames.OR: - nodeScanSpec = args.get(0).accept(this, null); - for (int i = 1; i < args.size(); ++i) { - JsonScanSpec nextScanSpec = args.get(i).accept(this, null); - if (nodeScanSpec != null && nextScanSpec != null) { - nodeScanSpec.mergeScanSpec(functionName, nextScanSpec); - } else { - allExpressionsConverted = false; - if (FunctionNames.AND.equals(functionName)) { - nodeScanSpec = nodeScanSpec == null ? nextScanSpec : nodeScanSpec; - } - } - } - break; - - case "ojai_sizeof": - case "ojai_typeof": - case "ojai_nottypeof": - case "ojai_matches": - case "ojai_notmatches": - case "ojai_condition": { - final OjaiFunctionsProcessor processor = OjaiFunctionsProcessor.process(call); - if (processor != null) { - return new JsonScanSpec(groupScan.getTableName(), groupScan.getIndexDesc(), - processor.getCondition()); - } - } - } - } - - if (nodeScanSpec == null) { - allExpressionsConverted = false; - } - - return nodeScanSpec; - } - - private void setIsCondition(QueryCondition c, - String str, - QueryCondition.Op op, - Value v) { - switch (v.getType()) { - case BOOLEAN: - c.is(str, op, v.getBoolean()); - break; - case STRING: - c.is(str, op, v.getString()); - break; - case BYTE: - c.is(str, op, v.getByte()); - break; - case SHORT: - c.is(str, op, v.getShort()); - break; - case INT: - c.is(str, op, v.getInt()); - break; - case LONG: - c.is(str, op, v.getLong()); - break; - case FLOAT: - c.is(str, op, v.getFloat()); - break; - case DOUBLE: - c.is(str, op, v.getDouble()); - break; - case DECIMAL: - c.is(str, op, v.getDecimal()); - break; - case DATE: - c.is(str, op, v.getDate()); - break; - case TIME: - c.is(str, op, v.getTime()); - break; - case TIMESTAMP: - c.is(str, op, v.getTimestamp()); - break; - case BINARY: - c.is(str, op, v.getBinary()); - break; - // XXX/TODO: Map, Array? - default: - break; - } - } - - private JsonScanSpec createJsonScanSpec(FunctionCall call, - CompareFunctionsProcessor processor) { - String functionName = processor.getFunctionName(); - String fieldPath = FieldPathHelper.schemaPath2FieldPath(processor.getPath()).asPathString(); - Value fieldValue = processor.getValue(); - - QueryCondition cond = null; - switch (functionName) { - case FunctionNames.EQ: - cond = MapRDBImpl.newCondition(); - setIsCondition(cond, fieldPath, Op.EQUAL, fieldValue); - break; - - case FunctionNames.NE: - cond = MapRDBImpl.newCondition(); - setIsCondition(cond, fieldPath, Op.NOT_EQUAL, fieldValue); - break; - - case FunctionNames.LT: - cond = MapRDBImpl.newCondition(); - setIsCondition(cond, fieldPath, Op.LESS, fieldValue); - break; - - case FunctionNames.LE: - cond = MapRDBImpl.newCondition(); - setIsCondition(cond, fieldPath, Op.LESS_OR_EQUAL, fieldValue); - break; - - case FunctionNames.GT: - cond = MapRDBImpl.newCondition(); - setIsCondition(cond, fieldPath, Op.GREATER, fieldValue); - break; - - case FunctionNames.GE: - cond = MapRDBImpl.newCondition(); - setIsCondition(cond, fieldPath, Op.GREATER_OR_EQUAL, fieldValue); - break; - - case FunctionNames.IS_NULL: - // 'field is null' should be transformed to 'field not exists OR typeof(field) = NULL' - QueryCondition orCond = MapRDBImpl.newCondition().or(); - cond = orCond.notExists(fieldPath).typeOf(fieldPath, Value.Type.NULL).close(); - break; - - case FunctionNames.IS_NOT_NULL: - // 'field is not null should be transformed to 'field exists AND typeof(field) != NULL' - QueryCondition andCond = MapRDBImpl.newCondition().and(); - cond = andCond.exists(fieldPath).notTypeOf(fieldPath, Value.Type.NULL).close(); - break; - - case FunctionNames.IS_TRUE: - cond = MapRDBImpl.newCondition().is(fieldPath, Op.EQUAL, true); - break; - - case FunctionNames.IS_NOT_FALSE: - cond = MapRDBImpl.newCondition().is(fieldPath, Op.NOT_EQUAL, false); - break; - - case FunctionNames.IS_FALSE: - cond = MapRDBImpl.newCondition().is(fieldPath, Op.EQUAL, false); - break; - - case FunctionNames.IS_NOT_TRUE: - cond = MapRDBImpl.newCondition().is(fieldPath, Op.NOT_EQUAL, true); - break; - - case "like": - cond = MapRDBImpl.newCondition().like(fieldPath, fieldValue.getString()); - break; - - default: - } - - if (cond != null) { - return new JsonScanSpec(groupScan.getTableName(), - groupScan.getIndexDesc(), - cond.build()); - } - - return null; - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonScanSpec.java deleted file mode 100644 index 1a9878f178f..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonScanSpec.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import java.nio.ByteBuffer; -import java.util.List; - -import org.apache.drill.common.FunctionNames; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HConstants; -import org.ojai.store.QueryCondition; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.mapr.db.impl.ConditionImpl; -import com.mapr.db.impl.ConditionNode.RowkeyRange; -import com.mapr.db.impl.MapRDBImpl; -import com.mapr.db.index.IndexDesc; - -public class JsonScanSpec { - protected String tableName; - protected IndexDesc indexDesc; - protected QueryCondition condition; - protected byte[] startRow; - protected byte[] stopRow; - - @JsonCreator - public JsonScanSpec(@JsonProperty("tableName") String tableName, - @JsonProperty("indexDesc") IndexDesc indexDesc, - @JsonProperty("condition") QueryCondition condition) { - this.tableName = tableName; - this.indexDesc = indexDesc; - this.condition = condition; - if (this.condition != null) { - List rkRanges = ((ConditionImpl)this.condition).getRowkeyRanges(); - if (rkRanges.size() > 0) { - startRow = rkRanges.get(0).getStartRow(); - stopRow = rkRanges.get(rkRanges.size() - 1).getStopRow(); - } else { - startRow = HConstants.EMPTY_START_ROW; - stopRow = HConstants.EMPTY_END_ROW; - } - } - } - - public String getTableName() { - return this.tableName; - } - - public IndexDesc getIndexDesc() { - return this.indexDesc; - } - - public void setStartRow(byte []startRow) { - this.startRow = startRow; - } - - public void setStopRow(byte []stopRow) { - this.stopRow = stopRow; - } - - public byte[] getStartRow() { - return this.startRow; - } - - public byte[] getStopRow() { - return this.stopRow; - } - - public byte[] getSerializedFilter() { - if (this.condition != null) { - ByteBuffer bbuf = ((ConditionImpl)this.condition).getDescriptor().getSerialized(); - byte[] serFilter = new byte[bbuf.limit() - bbuf.position()]; - bbuf.get(serFilter); - return serFilter; - } - return null; - } - - public void setCondition(QueryCondition condition) { - this.condition = condition; - } - - @JsonIgnore - public QueryCondition getCondition() { - return this.condition; - } - - public boolean isSecondaryIndex() { - return (this.indexDesc != null); - } - - @JsonIgnore - public Path getPrimaryTablePath() { - return (this.indexDesc == null) ? null : new Path(this.indexDesc.getPrimaryTablePath()); - } - - @JsonIgnore - public String getIndexName() { - return (this.indexDesc == null) ? null : this.indexDesc.getIndexName(); - } - - public void mergeScanSpec(String functionName, JsonScanSpec scanSpec) { - if (this.condition != null && scanSpec.getCondition() != null) { - QueryCondition newCond = MapRDBImpl.newCondition(); - switch (functionName) { - case FunctionNames.AND: - newCond.and(); - break; - case FunctionNames.OR: - newCond.or(); - break; - default: - assert(false); - } - - newCond.condition(this.condition) - .condition(scanSpec.getCondition()) - .close() - .build(); - - this.condition = newCond; - } else if (scanSpec.getCondition() != null){ - this.condition = scanSpec.getCondition(); - } - } - - @Override - public String toString() { - String fidInfo = (getIndexDesc() != null)? ", indexName=" + getIndexName() : ""; - return "JsonScanSpec [tableName=" + tableName - + ", condition=" + (condition == null ? null : condition.toString()) - + fidInfo - + "]"; - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonSubScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonSubScanSpec.java deleted file mode 100644 index 1e7eb31246c..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonSubScanSpec.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import java.nio.ByteBuffer; - -import org.apache.drill.exec.store.mapr.db.MapRDBSubScanSpec; -import org.ojai.store.QueryCondition; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.mapr.db.impl.ConditionImpl; -import com.mapr.db.impl.MapRDBImpl; -import com.mapr.db.index.IndexDesc; - -/** - * This class is a helper extension of {@link MapRDBSubScanSpec} class and does not - * get serialized or deserialized. - */ -public class JsonSubScanSpec extends MapRDBSubScanSpec { - - protected QueryCondition condition; - - public JsonSubScanSpec(String tableName, IndexDesc indexDesc, String regionServer, - QueryCondition scanRangeCondition, QueryCondition userCondition, - byte[] startRow, byte[] stopRow, String userName) { - super(tableName, indexDesc, regionServer, startRow, stopRow, null, null, userName); - - condition = MapRDBImpl.newCondition().and(); - - if (userCondition != null && !userCondition.isEmpty()) { - condition.condition(userCondition); - } - if (scanRangeCondition != null && !scanRangeCondition.isEmpty()) { - condition.condition(scanRangeCondition); - } - - condition.close().build(); - } - - public void setCondition(QueryCondition cond) { - condition = cond; - } - - @JsonIgnore - public QueryCondition getCondition() { - return this.condition; - } - - @Override - public byte[] getSerializedFilter() { - if (this.condition != null) { - ByteBuffer bbuf = ((ConditionImpl)this.condition).getDescriptor().getSerialized(); - byte[] serFilter = new byte[bbuf.limit() - bbuf.position()]; - bbuf.get(serFilter); - return serFilter; - } - - return null; - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java deleted file mode 100644 index b59267b37a0..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableGroupScan.java +++ /dev/null @@ -1,773 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import static org.apache.drill.exec.planner.index.Statistics.ROWCOUNT_HUGE; -import static org.apache.drill.exec.planner.index.Statistics.ROWCOUNT_UNKNOWN; -import static org.apache.drill.exec.planner.index.Statistics.AVG_ROWSIZE_UNKNOWN; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.TreeMap; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.mapr.db.impl.ConditionImpl; -import com.mapr.db.impl.ConditionNode.RowkeyRange; - -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rex.RexNode; -import org.apache.drill.common.exceptions.DrillRuntimeException; -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.expression.FieldReference; -import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.exec.physical.base.GroupScan; -import org.apache.drill.exec.physical.base.IndexGroupScan; -import org.apache.drill.exec.physical.base.PhysicalOperator; -import org.apache.drill.exec.physical.base.ScanStats; -import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty; -import org.apache.drill.exec.record.metadata.TupleMetadata; -import org.apache.drill.exec.store.AbstractStoragePlugin; -import org.apache.drill.exec.planner.index.IndexDescriptor; -import org.apache.drill.exec.planner.index.MapRDBIndexDescriptor; -import org.apache.drill.exec.planner.index.MapRDBStatisticsPayload; -import org.apache.drill.exec.planner.index.Statistics; -import org.apache.drill.exec.planner.index.MapRDBStatistics; -import org.apache.drill.exec.planner.cost.PluginCost; -import org.apache.drill.exec.planner.physical.PartitionFunction; -import org.apache.drill.exec.planner.physical.PrelUtil; -import org.apache.drill.exec.store.StoragePluginRegistry; -import org.apache.drill.exec.store.dfs.FileSystemConfig; -import org.apache.drill.exec.store.dfs.FileSystemPlugin; -import org.apache.drill.exec.store.mapr.PluginConstants; -import org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin; -import org.apache.drill.exec.store.mapr.db.MapRDBFormatPluginConfig; -import org.apache.drill.exec.store.mapr.db.MapRDBGroupScan; -import org.apache.drill.exec.store.mapr.db.MapRDBSubScan; -import org.apache.drill.exec.store.mapr.db.MapRDBSubScanSpec; -import org.apache.drill.exec.store.mapr.db.MapRDBTableStats; -import org.apache.drill.exec.store.mapr.db.TabletFragmentInfo; -import org.apache.drill.exec.util.Utilities; -import org.apache.drill.exec.metastore.store.FileSystemMetadataProviderManager; -import org.apache.drill.exec.metastore.MetadataProviderManager; -import org.apache.drill.metastore.metadata.TableMetadataProvider; -import org.ojai.store.QueryCondition; - -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.base.Preconditions; -import com.mapr.db.MetaTable; -import com.mapr.db.Table; -import com.mapr.db.impl.TabletInfoImpl; -import com.mapr.db.index.IndexDesc; -import com.mapr.db.scan.ScanRange; - -@JsonTypeName("maprdb-json-scan") -public class JsonTableGroupScan extends MapRDBGroupScan implements IndexGroupScan { - static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonTableGroupScan.class); - - public static final int STAR_COLS = 100; - public static final String TABLE_JSON = "json"; - /* - * The maintains a mapping of . These RowCounts take precedence over - * anything computed using stats. Currently, it is used for picking index plans with the - * index_selectivity_factor. We forcibly set the full table rows as HUGE in this - * map when the selectivity of the index is lower than index_selectivity_factor. During costing, the table - * rowCount is returned as HUGE instead of the correct rowcount. This results in the planner choosing - * the cheaper index plans! - * NOTE: Full table rowCounts are specified with the NULL condition. e.g. forcedRowCountMap - */ - protected Map forcedRowCountMap; - /* - * This stores the statistics associated with this GroupScan. Please note that the stats must be initialized - * before using it to compute filter row counts based on query conditions. - */ - protected MapRDBStatistics stats; - protected JsonScanSpec scanSpec; - protected double fullTableRowCount; - protected double fullTableEstimatedSize; - - /** - * need only read maxRecordsToRead records. - */ - protected int maxRecordsToRead = -1; - - /** - * Forced parallelization width - */ - protected int parallelizationWidth = -1; - - @JsonCreator - public JsonTableGroupScan(@JsonProperty("userName") final String userName, - @JsonProperty("scanSpec") JsonScanSpec scanSpec, - @JsonProperty("storage") FileSystemConfig storagePluginConfig, - @JsonProperty("format") MapRDBFormatPluginConfig formatPluginConfig, - @JsonProperty("columns") List columns, - @JsonProperty("schema") TupleMetadata schema, - @JacksonInject StoragePluginRegistry pluginRegistry) throws ExecutionSetupException { - this(userName, pluginRegistry.resolve(storagePluginConfig, AbstractStoragePlugin.class), - pluginRegistry.resolveFormat(storagePluginConfig, formatPluginConfig, MapRDBFormatPlugin.class), - scanSpec, columns, new MapRDBStatistics(), FileSystemMetadataProviderManager.getMetadataProviderForSchema(schema)); - } - - public JsonTableGroupScan(String userName, AbstractStoragePlugin storagePlugin, - MapRDBFormatPlugin formatPlugin, JsonScanSpec scanSpec, List columns, - MetadataProviderManager metadataProviderManager) { - this(userName, storagePlugin, formatPlugin, scanSpec, columns, - new MapRDBStatistics(), FileSystemMetadataProviderManager.getMetadataProvider(metadataProviderManager)); - } - - public JsonTableGroupScan(String userName, AbstractStoragePlugin storagePlugin, - MapRDBFormatPlugin formatPlugin, JsonScanSpec scanSpec, List columns, - MapRDBStatistics stats, TableMetadataProvider metadataProvider) { - super(storagePlugin, formatPlugin, columns, userName, metadataProvider); - this.scanSpec = scanSpec; - this.stats = stats; - this.forcedRowCountMap = new HashMap<>(); - init(); - } - - /** - * Private constructor, used for cloning. - * @param that The HBaseGroupScan to clone - */ - protected JsonTableGroupScan(JsonTableGroupScan that) { - super(that); - this.scanSpec = that.scanSpec; - this.endpointFragmentMapping = that.endpointFragmentMapping; - this.stats = that.stats; - this.fullTableRowCount = that.fullTableRowCount; - this.fullTableEstimatedSize = that.fullTableEstimatedSize; - this.forcedRowCountMap = that.forcedRowCountMap; - this.maxRecordsToRead = that.maxRecordsToRead; - this.parallelizationWidth = that.parallelizationWidth; - init(); - } - - @Override - public GroupScan clone(List columns) { - JsonTableGroupScan newScan = new JsonTableGroupScan(this); - newScan.columns = columns; - return newScan; - } - - public GroupScan clone(JsonScanSpec scanSpec) { - JsonTableGroupScan newScan = new JsonTableGroupScan(this); - newScan.scanSpec = scanSpec; - newScan.resetRegionsToScan(); // resetting will force recalculation - return newScan; - } - - private void init() { - try { - // Get the fullTableRowCount only once i.e. if not already obtained before. - if (fullTableRowCount == 0) { - final Table t = this.formatPlugin.getJsonTableCache().getTable( - scanSpec.getTableName(), scanSpec.getIndexDesc(), getUserName()); - final MetaTable metaTable = t.getMetaTable(); - // For condition null, we get full table stats. - com.mapr.db.scan.ScanStats stats = metaTable.getScanStats(); - fullTableRowCount = stats.getEstimatedNumRows(); - fullTableEstimatedSize = stats.getEstimatedSize(); - // MapRDB client can return invalid rowCount i.e. 0, especially right after table - // creation. It takes 15 minutes before table stats are obtained and cached in client. - // If we get 0 rowCount, fallback to getting rowCount using old admin API. - if (fullTableRowCount == 0) { - PluginCost pluginCostModel = formatPlugin.getPluginCostModel(); - final int avgColumnSize = pluginCostModel.getAverageColumnSize(this); - final int numColumns = (columns == null || columns.isEmpty() || Utilities.isStarQuery(columns)) ? STAR_COLS : columns.size(); - MapRDBTableStats tableStats = new MapRDBTableStats(formatPlugin.getFsConf(), scanSpec.getTableName()); - fullTableRowCount = tableStats.getNumRows(); - fullTableEstimatedSize = fullTableRowCount * numColumns * avgColumnSize; - } - } - } catch (Exception e) { - throw new DrillRuntimeException("Error getting region info for table: " + - scanSpec.getTableName() + (scanSpec.getIndexDesc() == null ? "" : (", index: " + scanSpec.getIndexName())), e); - } - } - - @Override - protected NavigableMap getRegionsToScan() { - return getRegionsToScan(formatPlugin.getScanRangeSizeMB()); - } - - protected NavigableMap getRegionsToScan(int scanRangeSizeMB) { - // If regionsToScan already computed, just return. - double estimatedRowCount = ROWCOUNT_UNKNOWN; - if (doNotAccessRegionsToScan == null) { - final Table t = this.formatPlugin.getJsonTableCache().getTable( - scanSpec.getTableName(), scanSpec.getIndexDesc(), getUserName()); - final MetaTable metaTable = t.getMetaTable(); - - QueryCondition scanSpecCondition = scanSpec.getCondition(); - List scanRanges = (scanSpecCondition == null) - ? metaTable.getScanRanges(scanRangeSizeMB) - : metaTable.getScanRanges(scanSpecCondition, scanRangeSizeMB); - logger.debug("getRegionsToScan() with scanSpec {}: table={}, index={}, condition={}, sizeMB={}, #ScanRanges={}", - System.identityHashCode(scanSpec), scanSpec.getTableName(), scanSpec.getIndexName(), - scanSpec.getCondition() == null ? "null" : scanSpec.getCondition(), scanRangeSizeMB, - scanRanges == null ? "null" : scanRanges.size()); - final TreeMap regionsToScan = new TreeMap<>(); - if (isIndexScan()) { - String idxIdentifier = stats.buildUniqueIndexIdentifier(scanSpec.getIndexDesc().getPrimaryTablePath(), - scanSpec.getIndexDesc().getIndexName()); - if (stats.isStatsAvailable()) { - estimatedRowCount = stats.getRowCount(scanSpec.getCondition(), idxIdentifier); - } - } else { - if (stats.isStatsAvailable()) { - estimatedRowCount = stats.getRowCount(scanSpec.getCondition(), null); - } - } - // If limit pushdown has occurred - factor it in the rowcount - if (this.maxRecordsToRead > 0) { - estimatedRowCount = Math.min(estimatedRowCount, this.maxRecordsToRead); - } - // If the estimated row count > 0 then scan ranges must be > 0 - Preconditions.checkState(estimatedRowCount == ROWCOUNT_UNKNOWN - || estimatedRowCount == 0 || (scanRanges != null && scanRanges.size() > 0), - String.format("#Scan ranges should be greater than 0 since estimated rowcount=[%f]", estimatedRowCount)); - if (scanRanges != null && scanRanges.size() > 0) { - // set the start-row of the scanspec as the start-row of the first scan range - ScanRange firstRange = scanRanges.get(0); - QueryCondition firstCondition = firstRange.getCondition(); - byte[] firstStartRow = ((ConditionImpl) firstCondition).getRowkeyRanges().get(0).getStartRow(); - scanSpec.setStartRow(firstStartRow); - - // set the stop-row of ScanSpec as the stop-row of the last scan range - ScanRange lastRange = scanRanges.get(scanRanges.size() - 1); - QueryCondition lastCondition = lastRange.getCondition(); - List rowkeyRanges = ((ConditionImpl) lastCondition).getRowkeyRanges(); - byte[] lastStopRow = rowkeyRanges.get(rowkeyRanges.size() - 1).getStopRow(); - scanSpec.setStopRow(lastStopRow); - - for (ScanRange range : scanRanges) { - TabletInfoImpl tabletInfoImpl = (TabletInfoImpl) range; - regionsToScan.put(new TabletFragmentInfo(tabletInfoImpl), range.getLocations()[0]); - } - } - setRegionsToScan(regionsToScan); - } - return doNotAccessRegionsToScan; - } - - @Override - protected MapRDBSubScanSpec getSubScanSpec(final TabletFragmentInfo tfi) { - // XXX/TODO check filter/Condition - final JsonScanSpec spec = scanSpec; - final String serverHostName = getRegionsToScan().get(tfi); - QueryCondition condition = tfi.getTabletInfoImpl().getCondition(); - byte[] startRow = condition == null ? null : ((ConditionImpl) condition).getRowkeyRanges().get(0).getStartRow(); - byte[] stopRow = condition == null ? null : ((ConditionImpl) condition).getRowkeyRanges().get(0).getStopRow(); - JsonSubScanSpec subScanSpec = new JsonSubScanSpec( - spec.getTableName(), - spec.getIndexDesc(), - serverHostName, - tfi.getTabletInfoImpl().getCondition(), - spec.getCondition(), - startRow, - stopRow, - getUserName()); - return subScanSpec; - } - - @Override - public MapRDBSubScan getSpecificScan(int minorFragmentId) { - assert minorFragmentId < endpointFragmentMapping.size() : String.format( - "Mappings length [%d] should be greater than minor fragment id [%d] but it isn't.", endpointFragmentMapping.size(), - minorFragmentId); - return new MapRDBSubScan(getUserName(), formatPlugin, endpointFragmentMapping.get(minorFragmentId), columns, maxRecordsToRead, TABLE_JSON, getSchema()); - } - - @Override - public ScanStats getScanStats() { - if (isIndexScan()) { - return indexScanStats(); - } - return fullTableScanStats(); - } - - private ScanStats fullTableScanStats() { - PluginCost pluginCostModel = formatPlugin.getPluginCostModel(); - final int avgColumnSize = pluginCostModel.getAverageColumnSize(this); - final int numColumns = (columns == null || columns.isEmpty()) ? STAR_COLS : columns.size(); - // index will be NULL for FTS - double rowCount = stats.getRowCount(scanSpec.getCondition(), null); - // rowcount based on _id predicate. If NO _id predicate present in condition, then the - // rowcount should be same as totalRowCount. Equality b/w the two rowcounts should not be - // construed as NO _id predicate since stats are approximate. - double leadingRowCount = stats.getLeadingRowCount(scanSpec.getCondition(), null); - double avgRowSize = stats.getAvgRowSize(null, true); - double totalRowCount = stats.getRowCount(null, null); - logger.debug("GroupScan {} with stats {}: rowCount={}, condition={}, totalRowCount={}, fullTableRowCount={}", - System.identityHashCode(this), System.identityHashCode(stats), rowCount, - scanSpec.getCondition() == null ? "null" : scanSpec.getCondition(), - totalRowCount, fullTableRowCount); - // If UNKNOWN, or DB stats sync issues(manifests as 0 rows) use defaults. - if (rowCount == ROWCOUNT_UNKNOWN || rowCount == 0) { - rowCount = (scanSpec.getSerializedFilter() != null ? .5 : 1) * fullTableRowCount; - } - // If limit pushdown has occurred - factor it in the rowcount - if (this.maxRecordsToRead > 0) { - rowCount = Math.min(rowCount, this.maxRecordsToRead); - } - if (totalRowCount == ROWCOUNT_UNKNOWN || totalRowCount == 0) { - logger.debug("did not get valid totalRowCount, will take this: {}", fullTableRowCount); - totalRowCount = fullTableRowCount; - } - if (avgRowSize == AVG_ROWSIZE_UNKNOWN || avgRowSize == 0) { - avgRowSize = fullTableEstimatedSize/fullTableRowCount; - } - double totalBlocks = getNumOfBlocks(totalRowCount, fullTableEstimatedSize, avgRowSize, pluginCostModel); - double numBlocks = Math.min(totalBlocks, getNumOfBlocks(leadingRowCount, fullTableEstimatedSize, avgRowSize, pluginCostModel)); - double diskCost = numBlocks * pluginCostModel.getSequentialBlockReadCost(this); - /* - * Table scan cost made INFINITE in order to pick index plans. Use the MAX possible rowCount for - * costing purposes. - * NOTE: Full table rowCounts are specified with the NULL condition. - * e.g. forcedRowCountMap - */ - if (forcedRowCountMap.get(null) != null && //Forced full table rowcount and it is HUGE - forcedRowCountMap.get(null) == ROWCOUNT_HUGE ) { - rowCount = ROWCOUNT_HUGE; - diskCost = ROWCOUNT_HUGE; - } - - logger.debug("JsonGroupScan:{} rowCount:{}, avgRowSize:{}, blocks:{}, totalBlocks:{}, diskCost:{}", - this.getOperatorId(), rowCount, avgRowSize, numBlocks, totalBlocks, diskCost); - return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, rowCount, 1, diskCost); - } - - private double getNumOfBlocks(double rowCount, double sizeFromDisk, - double avgRowSize, PluginCost pluginCostModel) { - if (rowCount == ROWCOUNT_UNKNOWN || rowCount == 0) { - return Math.ceil(sizeFromDisk / pluginCostModel.getBlockSize(this)); - } else { - return Math.ceil(rowCount * avgRowSize / pluginCostModel.getBlockSize(this)); - } - } - - private ScanStats indexScanStats() { - if (!this.getIndexHint().equals("") && - this.getIndexHint().equals(getIndexDesc().getIndexName())) { - logger.debug("JsonIndexGroupScan:{} forcing index {} by making tiny cost", this, this.getIndexHint()); - return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, 1,1, 0); - } - - int totalColNum = STAR_COLS; - PluginCost pluginCostModel = formatPlugin.getPluginCostModel(); - final int avgColumnSize = pluginCostModel.getAverageColumnSize(this); - boolean filterPushed = (scanSpec.getSerializedFilter() != null); - if (scanSpec != null && scanSpec.getIndexDesc() != null) { - totalColNum = scanSpec.getIndexDesc().getIncludedFields().size() - + scanSpec.getIndexDesc().getIndexedFields().size() + 1; - } - int numColumns = (columns == null || columns.isEmpty()) ? totalColNum: columns.size(); - String idxIdentifier = stats.buildUniqueIndexIdentifier(scanSpec.getIndexDesc().getPrimaryTablePath(), - scanSpec.getIndexDesc().getIndexName()); - double rowCount = stats.getRowCount(scanSpec.getCondition(), idxIdentifier); - // rowcount based on index leading columns predicate. - double leadingRowCount = stats.getLeadingRowCount(scanSpec.getCondition(), idxIdentifier); - double avgRowSize = stats.getAvgRowSize(idxIdentifier, false); - // If UNKNOWN, use defaults - if (rowCount == ROWCOUNT_UNKNOWN || rowCount == 0) { - rowCount = (filterPushed ? 0.0005f : 0.001f) * fullTableRowCount / scanSpec.getIndexDesc().getIndexedFields().size(); - } - // If limit pushdown has occurred - factor it in the rowcount - if (this.maxRecordsToRead > 0) { - rowCount = Math.min(rowCount, this.maxRecordsToRead); - } - if (leadingRowCount == ROWCOUNT_UNKNOWN || leadingRowCount == 0) { - leadingRowCount = rowCount; - } - if (avgRowSize == AVG_ROWSIZE_UNKNOWN || avgRowSize == 0) { - avgRowSize = avgColumnSize * numColumns; - } - double rowsFromDisk = leadingRowCount; - if (!filterPushed) { - // both start and stop rows are empty, indicating this is a full scan so - // use the total rows for calculating disk i/o - rowsFromDisk = fullTableRowCount; - } - double totalBlocks = Math.ceil((avgRowSize * fullTableRowCount)/pluginCostModel.getBlockSize(this)); - double numBlocks = Math.ceil(((avgRowSize * rowsFromDisk)/pluginCostModel.getBlockSize(this))); - numBlocks = Math.min(totalBlocks, numBlocks); - double diskCost = numBlocks * pluginCostModel.getSequentialBlockReadCost(this); - logger.debug("index_plan_info: JsonIndexGroupScan:{} - indexName:{}: rowCount:{}, avgRowSize:{}, blocks:{}, totalBlocks:{}, rowsFromDisk {}, diskCost:{}", - System.identityHashCode(this), scanSpec.getIndexDesc().getIndexName(), rowCount, avgRowSize, numBlocks, totalBlocks, rowsFromDisk, diskCost); - return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, rowCount, 1, diskCost); - } - - @Override - @JsonIgnore - public PhysicalOperator getNewWithChildren(List children) { - Preconditions.checkArgument(children.isEmpty()); - return new JsonTableGroupScan(this); - } - - @Override - @JsonIgnore - public String getTableName() { - return scanSpec.getTableName(); - } - - public IndexDesc getIndexDesc() { - return scanSpec.getIndexDesc(); - } - - public boolean isDisablePushdown() { - return !formatPluginConfig.isEnablePushdown(); - } - - @Override - @JsonIgnore - public boolean canPushdownProjects(List columns) { - return formatPluginConfig.isEnablePushdown(); - } - - @Override - public String toString() { - return "JsonTableGroupScan [ScanSpec=" + scanSpec + ", columns=" + columns - + (maxRecordsToRead > 0 ? ", limit=" + maxRecordsToRead : "") - + (getMaxParallelizationWidth() > 0 ? ", maxwidth=" + getMaxParallelizationWidth() : "") + "]"; - } - - public JsonScanSpec getScanSpec() { - return scanSpec; - } - - @Override - public boolean supportsSecondaryIndex() { - return true; - } - - @Override - @JsonIgnore - public boolean isIndexScan() { - return scanSpec != null && scanSpec.isSecondaryIndex(); - } - - @Override - public boolean supportsRestrictedScan() { - return true; - } - - - @Override - public RestrictedJsonTableGroupScan getRestrictedScan(List columns) { - RestrictedJsonTableGroupScan newScan = new RestrictedJsonTableGroupScan(this.getUserName(), - (FileSystemPlugin) this.getStoragePlugin(), - this.getFormatPlugin(), - this.getScanSpec(), - this.getColumns(), - this.getStatistics(), - this.getSchema()); - - newScan.columns = columns; - return newScan; - } - - /** - * Get the estimated average rowsize. DO NOT call this API directly. - * Call the stats API instead which modifies the counts based on preference options. - * @param index to use for generating the estimate - * @return row count post filtering - */ - public MapRDBStatisticsPayload getAverageRowSizeStats(IndexDescriptor index) { - IndexDesc indexDesc = null; - double avgRowSize = AVG_ROWSIZE_UNKNOWN; - - if (index != null) { - indexDesc = (IndexDesc)((MapRDBIndexDescriptor)index).getOriginalDesc(); - } - // If no index is specified, get it from the primary table - if (indexDesc == null && scanSpec.isSecondaryIndex()) { - throw new UnsupportedOperationException("getAverageRowSizeStats should be invoked on primary table"); - } - - // Get the index table or primary table and use the DB API to get the estimated number of rows. For size estimates, - // we assume that all the columns would be read from the disk. - final Table table = this.formatPlugin.getJsonTableCache().getTable(scanSpec.getTableName(), indexDesc, getUserName()); - - if (table != null) { - final MetaTable metaTable = table.getMetaTable(); - if (metaTable != null) { - avgRowSize = metaTable.getAverageRowSize(); - } - } - logger.debug("index_plan_info: getEstimatedRowCount obtained from DB Client for {}: indexName: {}, indexInfo: {}, " + - "avgRowSize: {}, estimatedSize {}", this, (indexDesc == null ? "null" : indexDesc.getIndexName()), - (indexDesc == null ? "null" : indexDesc.getIndexInfo()), avgRowSize, fullTableEstimatedSize); - return new MapRDBStatisticsPayload(ROWCOUNT_UNKNOWN, ROWCOUNT_UNKNOWN, avgRowSize); - } - - /** - * Get the estimated statistics after applying the {@link RexNode} condition. DO NOT call this API directly. - * Call the stats API instead which modifies the counts based on preference options. - * @param condition filter to apply - * @param index to use for generating the estimate - * @param scanRel the current scan rel - * @return row count post filtering - */ - public MapRDBStatisticsPayload getFirstKeyEstimatedStats(QueryCondition condition, IndexDescriptor index, RelNode scanRel) { - IndexDesc indexDesc = null; - if (index != null) { - indexDesc = (IndexDesc)((MapRDBIndexDescriptor)index).getOriginalDesc(); - } - return getFirstKeyEstimatedStatsInternal(condition, indexDesc, scanRel); - } - - /** - * Get the estimated statistics after applying the {@link QueryCondition} condition - * @param condition filter to apply - * @param index to use for generating the estimate - * @param scanRel the current scan rel - * @return {@link MapRDBStatisticsPayload} statistics - */ - private MapRDBStatisticsPayload getFirstKeyEstimatedStatsInternal(QueryCondition condition, IndexDesc index, RelNode scanRel) { - - // If no index is specified, get it from the primary table - if (index == null && scanSpec.isSecondaryIndex()) { - // If stats not cached get it from the table. - // table = MapRDB.getTable(scanSpec.getPrimaryTablePath()); - throw new UnsupportedOperationException("getFirstKeyEstimatedStats should be invoked on primary table"); - } - - // Get the index table or primary table and use the DB API to get the estimated number of rows. For size estimates, - // we assume that all the columns would be read from the disk. - final Table table = this.formatPlugin.getJsonTableCache().getTable(scanSpec.getTableName(), index, getUserName()); - - if (table != null) { - // Factor reflecting confidence in the DB estimates. If a table has few tablets, the tablet-level stats - // might be off. The decay scalingFactor will reduce estimates when one tablet represents a significant percentage - // of the entire table. - double scalingFactor = 1.0; - boolean isFullScan = false; - final MetaTable metaTable = table.getMetaTable(); - com.mapr.db.scan.ScanStats stats = (condition == null) - ? metaTable.getScanStats() : metaTable.getScanStats(condition); - if (index == null && condition != null) { - // Given table condition might not be on leading column. Check if the rowcount matches full table rows. - // In that case no leading key present or does not prune enough. Treat it like so. - com.mapr.db.scan.ScanStats noConditionPTabStats = metaTable.getScanStats(); - if (stats.getEstimatedNumRows() == noConditionPTabStats.getEstimatedNumRows()) { - isFullScan = true; - } - } - // Use the scalingFactor only when a condition filters out rows from the table. If no condition is present, all rows - // should be selected. So the scalingFactor should not reduce the returned rows - if (condition != null && !isFullScan) { - double forcedScalingFactor = PrelUtil.getSettings(scanRel.getCluster()).getIndexStatsRowCountScalingFactor(); - // Accuracy is defined as 1 - Error where Error = # Boundary Tablets (2) / # Total Matching Tablets. - // For 2 or less matching tablets, the error is assumed to be 50%. The Sqrt gives the decaying scalingFactor - if (stats.getTabletCount() > 2) { - double accuracy = 1.0 - (2.0/stats.getTabletCount()); - scalingFactor = Math.min(1.0, 1.0 / Math.sqrt(1.0 / accuracy)); - } else { - scalingFactor = 0.5; - } - if (forcedScalingFactor < 1.0 - && metaTable.getScanStats().getTabletCount() < PluginConstants.JSON_TABLE_NUM_TABLETS_PER_INDEX_DEFAULT) { - // User forced confidence scalingFactor for small tables (assumed as less than 32 tablets (~512 MB)) - scalingFactor = forcedScalingFactor; - } - } - logger.info("index_plan_info: getEstimatedRowCount obtained from DB Client for {}: indexName: {}, indexInfo: {}, " + - "condition: {} rowCount: {}, avgRowSize: {}, estimatedSize {}, tabletCount {}, totalTabletCount {}, " + - "scalingFactor {}", - this, (index == null ? "null" : index.getIndexName()), (index == null ? "null" : index.getIndexInfo()), - (condition == null ? "null" : condition.toString()), stats.getEstimatedNumRows(), - (stats.getEstimatedNumRows() == 0 ? 0 : stats.getEstimatedSize()/stats.getEstimatedNumRows()), - stats.getEstimatedSize(), stats.getTabletCount(), metaTable.getScanStats().getTabletCount(), scalingFactor); - return new MapRDBStatisticsPayload(scalingFactor * stats.getEstimatedNumRows(), scalingFactor * stats.getEstimatedNumRows(), - ((stats.getEstimatedNumRows() == 0 ? 0 : (double)stats.getEstimatedSize()/stats.getEstimatedNumRows()))); - } else { - logger.info("index_plan_info: getEstimatedRowCount: {} indexName: {}, indexInfo: {}, " + - "condition: {} rowCount: UNKNOWN, avgRowSize: UNKNOWN", this, (index == null ? "null" : index.getIndexName()), - (index == null ? "null" : index.getIndexInfo()), (condition == null ? "null" : condition.toString())); - return new MapRDBStatisticsPayload(ROWCOUNT_UNKNOWN, ROWCOUNT_UNKNOWN, AVG_ROWSIZE_UNKNOWN); - } - } - - - /** - * Set the row count resulting from applying the {@link RexNode} condition. Forced row counts will take - * precedence over stats row counts - * @param condition filter to apply - * @param count row count - * @param capRowCount row count limit - */ - @Override - @JsonIgnore - public void setRowCount(RexNode condition, double count, double capRowCount) { - forcedRowCountMap.put(condition, count); - } - - @Override - public void setStatistics(Statistics statistics) { - assert statistics instanceof MapRDBStatistics : String.format( - "Passed unexpected statistics instance. Expects MAPR-DB Statistics instance"); - this.stats = ((MapRDBStatistics) statistics); - } - - /** - * Get the row count after applying the {@link RexNode} condition - * @param condition filter to apply - * @param scanRel the current scan rel - * @return row count post filtering - */ - @Override - @JsonIgnore - public double getRowCount(RexNode condition, RelNode scanRel) { - // Do not use statistics if row count is forced. Forced rowcounts take precedence over stats - double rowcount; - if (forcedRowCountMap.get(condition) != null) { - return forcedRowCountMap.get(condition); - } - if (scanSpec.getIndexDesc() != null) { - String idxIdentifier = stats.buildUniqueIndexIdentifier(scanSpec.getIndexDesc().getPrimaryTablePath(), - scanSpec.getIndexName()); - rowcount = stats.getRowCount(condition, idxIdentifier, scanRel); - } else { - rowcount = stats.getRowCount(condition, null, scanRel); - } - // Stats might NOT have the full rows (e.g. table is newly populated and DB stats APIs return it after - // 15 mins). Use the table rows as populated using the (expensive but accurate) Hbase API if needed. - if (condition == null && (rowcount == 0 || rowcount == ROWCOUNT_UNKNOWN)) { - rowcount = fullTableRowCount; - logger.debug("getRowCount: Stats not available yet! Use Admin APIs full table rowcount {}", - fullTableRowCount); - } - return rowcount; - } - - @Override - public boolean isDistributed() { - // getMaxParallelizationWidth gets information about all regions to scan and is expensive. - // This option is meant to be used only for unit tests. - boolean useNumRegions = storagePlugin.getContext().getConfig().getBoolean(PluginConstants.JSON_TABLE_USE_NUM_REGIONS_FOR_DISTRIBUTION_PLANNING); - double fullTableSize; - - if (useNumRegions) { - return getMaxParallelizationWidth() > 1; - } - - // This function gets called multiple times during planning. To avoid performance - // bottleneck, estimate degree of parallelization using stats instead of actually getting information - // about all regions. - double rowCount, rowSize; - double scanRangeSize = storagePlugin.getContext().getConfig().getInt(PluginConstants.JSON_TABLE_SCAN_SIZE_MB) * 1024 * 1024; - - if (scanSpec.getIndexDesc() != null) { - String idxIdentifier = stats.buildUniqueIndexIdentifier(scanSpec.getIndexDesc().getPrimaryTablePath(), scanSpec.getIndexName()); - rowCount = stats.getRowCount(scanSpec.getCondition(), idxIdentifier); - rowSize = stats.getAvgRowSize(idxIdentifier, false); - } else { - rowCount = stats.getRowCount(scanSpec.getCondition(), null); - rowSize = stats.getAvgRowSize(null, false); - } - - if (rowCount == ROWCOUNT_UNKNOWN || rowCount == 0 || - rowSize == AVG_ROWSIZE_UNKNOWN || rowSize == 0) { - fullTableSize = (scanSpec.getSerializedFilter() != null ? .5 : 1) * this.fullTableEstimatedSize; - } else { - fullTableSize = rowCount * rowSize; - } - - return (long) fullTableSize / scanRangeSize > 1; - } - - @Override - public MapRDBStatistics getStatistics() { - return stats; - } - - @Override - @JsonIgnore - public void setColumns(List columns) { - this.columns = columns; - } - - @Override - @JsonIgnore - public List getColumns() { - return columns; - } - - @Override - @JsonIgnore - public PartitionFunction getRangePartitionFunction(List refList) { - return new JsonTableRangePartitionFunction(refList, scanSpec.getTableName(), this.getUserName(), this.getFormatPlugin()); - } - - /** - * Convert a given {@link LogicalExpression} condition into a {@link QueryCondition} condition - * @param condition expressed as a {@link LogicalExpression} - * @return {@link QueryCondition} condition equivalent to the given expression - */ - @JsonIgnore - public QueryCondition convertToQueryCondition(LogicalExpression condition) { - final JsonConditionBuilder jsonConditionBuilder = new JsonConditionBuilder(this, condition); - final JsonScanSpec newScanSpec = jsonConditionBuilder.parseTree(); - if (newScanSpec != null) { - return newScanSpec.getCondition(); - } else { - return null; - } - } - - /** - * Checks if Json table reader supports limit push down. - * - * @return true if limit push down is supported, false otherwise - */ - @Override - public boolean supportsLimitPushdown() { - if (maxRecordsToRead < 0) { - return true; - } - return false; // limit is already pushed. No more pushdown of limit - } - - @Override - public GroupScan applyLimit(int maxRecords) { - maxRecordsToRead = Math.max(maxRecords, 1); - return this; - } - - @Override - public int getMaxParallelizationWidth() { - if (this.parallelizationWidth > 0) { - return this.parallelizationWidth; - } - return super.getMaxParallelizationWidth(); - } - - @Override - public void setParallelizationWidth(int width) { - if (width > 0) { - this.parallelizationWidth = width; - logger.debug("Forced parallelization width = {}", width); - } - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableRangePartitionFunction.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableRangePartitionFunction.java deleted file mode 100644 index 3b438189f48..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonTableRangePartitionFunction.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import java.util.List; -import java.util.Objects; - -import org.apache.drill.common.expression.FieldReference; -import org.apache.drill.exec.planner.physical.AbstractRangePartitionFunction; -import org.apache.drill.exec.record.VectorWrapper; -import org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin; -import org.apache.drill.exec.vector.ValueVector; -import org.ojai.store.QueryCondition; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.mapr.db.Table; -import com.mapr.db.impl.ConditionImpl; -import com.mapr.db.impl.IdCodec; -import com.mapr.db.impl.ConditionNode.RowkeyRange; -import com.mapr.db.scan.ScanRange; -import com.mapr.fs.jni.MapRConstants; -import com.mapr.org.apache.hadoop.hbase.util.Bytes; - -@JsonTypeName("jsontable-range-partition-function") -public class JsonTableRangePartitionFunction extends AbstractRangePartitionFunction { - private static final Logger logger = LoggerFactory.getLogger(JsonTableRangePartitionFunction.class); - - @JsonProperty("refList") - protected List refList; - - @JsonProperty("tableName") - protected String tableName; - - @JsonIgnore - protected String userName; - - @JsonIgnore - protected ValueVector partitionKeyVector = null; - - // List of start keys of the scan ranges for the table. - @JsonProperty - protected List startKeys = null; - - // List of stop keys of the scan ranges for the table. - @JsonProperty - protected List stopKeys = null; - - @JsonCreator - public JsonTableRangePartitionFunction( - @JsonProperty("refList") List refList, - @JsonProperty("tableName") String tableName, - @JsonProperty("startKeys") List startKeys, - @JsonProperty("stopKeys") List stopKeys) { - this.refList = refList; - this.tableName = tableName; - this.startKeys = startKeys; - this.stopKeys = stopKeys; - } - - public JsonTableRangePartitionFunction(List refList, - String tableName, String userName, MapRDBFormatPlugin formatPlugin) { - this.refList = refList; - this.tableName = tableName; - this.userName = userName; - initialize(formatPlugin); - } - - @JsonProperty("refList") - @Override - public List getPartitionRefList() { - return refList; - } - - @Override - public void setup(List> partitionKeys) { - if (partitionKeys.size() != 1) { - throw new UnsupportedOperationException( - "Range partitioning function supports exactly one partition column; encountered " + partitionKeys.size()); - } - - VectorWrapper v = partitionKeys.get(0); - - partitionKeyVector = v.getValueVector(); - - Preconditions.checkArgument(partitionKeyVector != null, "Found null partitionKeVector."); - } - - @Override - public int hashCode() { - return Objects.hash(refList, tableName, userName, partitionKeyVector, startKeys, stopKeys); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj instanceof JsonTableRangePartitionFunction) { - JsonTableRangePartitionFunction rpf = (JsonTableRangePartitionFunction) obj; - List thisPartRefList = this.getPartitionRefList(); - List otherPartRefList = rpf.getPartitionRefList(); - if (thisPartRefList.size() != otherPartRefList.size()) { - return false; - } - for (int refIdx = 0; refIdx < thisPartRefList.size(); refIdx++) { - if (!thisPartRefList.get(refIdx).equals(otherPartRefList.get(refIdx))) { - return false; - } - } - return true; - } - return false; - } - - @Override - public int eval(int index, int numPartitions) { - - String key = partitionKeyVector.getAccessor().getObject(index).toString(); - byte[] encodedKey = IdCodec.encodeAsBytes(key); - - int tabletId = -1; - - // Do a 'range' binary search through the list of start-stop keys to find nearest range. Assumption is - // that the list of start keys is ordered (this is ensured by MetaTable.getScanRanges()) - // TODO: This search should ideally be delegated to MapR-DB once an appropriate API is available - // to optimize this - int low = 0, high = startKeys.size() - 1; - while (low <= high) { - int mid = low + (high-low)/2; - - byte[] start = startKeys.get(mid); - byte[] stop = stopKeys.get(mid); - - // Check if key is present in the mid interval of [start, stop]. - // Account for empty byte array start/stop - if ((Bytes.compareTo(encodedKey, start) >= 0 || - Bytes.equals(start, MapRConstants.EMPTY_BYTE_ARRAY) - ) && - (Bytes.compareTo(encodedKey, stop) < 0 || - Bytes.equals(stop, MapRConstants.EMPTY_BYTE_ARRAY) - ) - ) { - tabletId = mid; - break; - } - - if (Bytes.compareTo(encodedKey, start) >= 0) { - // key is greater, ignore left side - low = mid + 1; - } else { - // key is smaller, ignore right side - high = mid - 1; - } - } - - if (tabletId < 0) { - tabletId = 0; - logger.warn("Key {} was not found in any of the start-stop ranges. Using default tabletId {}", key, tabletId); - } - - int partitionId = tabletId % numPartitions; - - logger.trace("Key = {}, tablet id = {}, partition id = {}", key, tabletId, partitionId); - - return partitionId; - } - - - public void initialize(MapRDBFormatPlugin plugin) { - - // get the table handle from the table cache - Table table = plugin.getJsonTableCache().getTable(tableName, userName); - - // Get all scan ranges for the primary table. - // The reason is the row keys could typically belong to any one of the tablets of the table, so - // there is no use trying to get only limited set of scan ranges. - // NOTE: here we use the restrictedScanRangeSizeMB because the range partitioning should be parallelized - // based on the number of scan ranges on the RestrictedJsonTableGroupScan. - List ranges = table.getMetaTable().getScanRanges(plugin.getRestrictedScanRangeSizeMB()); - - this.startKeys = Lists.newArrayList(); - this.stopKeys = Lists.newArrayList(); - - logger.debug("Num scan ranges for table {} = {}", table.getName(), ranges.size()); - - int count = 0; - for (ScanRange r : ranges) { - QueryCondition condition = r.getCondition(); - List rowkeyRanges = ((ConditionImpl)condition).getRowkeyRanges(); - byte[] start = rowkeyRanges.get(0).getStartRow(); - byte[] stop = rowkeyRanges.get(rowkeyRanges.size() - 1).getStopRow(); - - Preconditions.checkNotNull(start, String.format("Encountered a null start key at position %d for scan range condition %s.", count, condition.toString())); - Preconditions.checkNotNull(stop, String.format("Encountered a null stop key at position %d for scan range condition %s.", count, condition.toString())); - - if (count > 0) { - // after the first start key, rest should be non-empty - Preconditions.checkState( !(Bytes.equals(start, MapRConstants.EMPTY_BYTE_ARRAY)), String.format("Encountered an empty start key at position %d", count)); - } - - if (count < ranges.size() - 1) { - // except for the last stop key, rest should be non-empty - Preconditions.checkState( !(Bytes.equals(stop, MapRConstants.EMPTY_BYTE_ARRAY)), String.format("Encountered an empty stop key at position %d", count)); - } - - startKeys.add(start); - stopKeys.add(stop); - count++; - } - - // check validity; only need to check one of the lists since they are populated together - Preconditions.checkArgument(startKeys.size() > 0, "Found empty list of start/stopKeys."); - - Preconditions.checkState(startKeys.size() == ranges.size(), - String.format("Mismatch between the lengths: num start keys = %d, num scan ranges = %d", startKeys.size(), ranges.size())); - - Preconditions.checkState(stopKeys.size() == ranges.size(), - String.format("Mismatch between the lengths: num stop keys = %d, num scan ranges = %d", stopKeys.size(), ranges.size())); - - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java deleted file mode 100644 index 287c764fc1b..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/MaprDBJsonRecordReader.java +++ /dev/null @@ -1,557 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - - -import com.mapr.db.Table; -import com.mapr.db.Table.TableOption; -import com.mapr.db.exceptions.DBException; -import com.mapr.db.impl.IdCodec; -import com.mapr.db.impl.MapRDBImpl; -import com.mapr.db.index.IndexDesc; -import com.mapr.db.ojai.DBDocumentReaderBase; -import com.mapr.db.util.ByteBufs; -import io.netty.buffer.DrillBuf; -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.exceptions.UserException; -import org.apache.drill.common.expression.PathSegment; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.exec.ExecConstants; -import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.expr.fn.impl.DateUtility; -import org.apache.drill.exec.ops.FragmentContext; -import org.apache.drill.exec.ops.OperatorContext; -import org.apache.drill.exec.ops.OperatorStats; -import org.apache.drill.exec.physical.impl.OutputMutator; -import org.apache.drill.exec.record.metadata.TupleMetadata; -import org.apache.drill.exec.store.AbstractRecordReader; -import org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin; -import org.apache.drill.exec.store.mapr.db.MapRDBSubScanSpec; -import org.apache.drill.exec.util.EncodedSchemaPathSet; -import org.apache.drill.exec.vector.BaseValueVector; -import org.apache.drill.exec.vector.complex.fn.JsonReaderUtils; -import org.apache.drill.exec.vector.complex.impl.MapOrListWriterImpl; -import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; -import org.apache.hadoop.fs.Path; -import org.ojai.Document; -import org.ojai.DocumentReader; -import org.ojai.DocumentStream; -import org.ojai.FieldPath; -import org.ojai.FieldSegment; -import org.ojai.store.QueryCondition; -import org.ojai.util.FieldProjector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Preconditions; -import com.google.common.base.Stopwatch; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; - -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.Stack; -import java.util.concurrent.TimeUnit; - -import static org.apache.drill.exec.store.mapr.PluginConstants.DOCUMENT_SCHEMA_PATH; -import static org.apache.drill.exec.store.mapr.PluginErrorHandler.dataReadError; -import static org.ojai.DocumentConstants.ID_FIELD; - -public class MaprDBJsonRecordReader extends AbstractRecordReader { - private static final Logger logger = LoggerFactory.getLogger(MaprDBJsonRecordReader.class); - protected enum SchemaState {SCHEMA_UNKNOWN, SCHEMA_INIT, SCHEMA_CHANGE}; - - protected static final FieldPath[] ID_ONLY_PROJECTION = { ID_FIELD }; - - protected Table table; - protected QueryCondition condition; - - /** - * A set of projected FieldPaths that are pushed into MapR-DB Scanner. - * This set is a superset of the fields returned by {@link #getColumns()} when - * projection pass-through is in effect. In such cases, {@link #getColumns()} - * returns only those fields which are required by Drill to run its operators. - */ - private FieldPath[] scannedFields; - - private OperatorContext operatorContext; - protected VectorContainerWriter vectorWriter; - private DBDocumentReaderBase reader; - Document document; - protected OutputMutator vectorWriterMutator; - - private DrillBuf buffer; - - private DocumentStream documentStream; - - private Iterator documentReaderIterators; - private Iterator documentIterator; - - private boolean includeId; - private boolean idOnly; - private SchemaState schemaState; - private boolean projectWholeDocument; - private FieldProjector projector; - - private final boolean unionEnabled; - private final boolean readNumbersAsDouble; - private final boolean readTimestampWithZoneOffset; - private boolean disablePushdown; - private final boolean allTextMode; - private final boolean ignoreSchemaChange; - private final boolean disableCountOptimization; - private final boolean nonExistentColumnsProjection; - private final TupleMetadata schema; - - protected final MapRDBSubScanSpec subScanSpec; - protected final MapRDBFormatPlugin formatPlugin; - - protected OjaiValueWriter valueWriter; - protected DocumentReaderVectorWriter documentWriter; - protected int maxRecordsToRead = -1; - protected DBDocumentReaderBase lastDocumentReader; - protected Document lastDocument; - - public MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, MapRDBFormatPlugin formatPlugin, - List projectedColumns, FragmentContext context, int maxRecords, TupleMetadata schema) { - this(subScanSpec, formatPlugin, projectedColumns, context, schema); - this.maxRecordsToRead = maxRecords; - this.lastDocumentReader = null; - this.lastDocument = null; - this.schemaState = SchemaState.SCHEMA_UNKNOWN; - } - - protected MaprDBJsonRecordReader(MapRDBSubScanSpec subScanSpec, MapRDBFormatPlugin formatPlugin, - List projectedColumns, FragmentContext context, TupleMetadata schema) { - buffer = context.getManagedBuffer(); - final Path tablePath = new Path(Preconditions.checkNotNull(subScanSpec, - "MapRDB reader needs a sub-scan spec").getTableName()); - this.subScanSpec = subScanSpec; - this.formatPlugin = formatPlugin; - this.schema = schema; - final IndexDesc indexDesc = subScanSpec.getIndexDesc(); - byte[] serializedFilter = subScanSpec.getSerializedFilter(); - condition = null; - - if (serializedFilter != null) { - condition = com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(serializedFilter)); - } - - disableCountOptimization = formatPlugin.getConfig().disableCountOptimization(); - // Below call will set the scannedFields and includeId correctly - setColumns(projectedColumns); - unionEnabled = context.getOptions().getOption(ExecConstants.ENABLE_UNION_TYPE); - readNumbersAsDouble = formatPlugin.getConfig().isReadAllNumbersAsDouble(); - readTimestampWithZoneOffset = formatPlugin.getConfig().isReadTimestampWithZoneOffset(); - allTextMode = formatPlugin.getConfig().isAllTextMode(); - ignoreSchemaChange = formatPlugin.getConfig().isIgnoreSchemaChange(); - disablePushdown = !formatPlugin.getConfig().isEnablePushdown(); - nonExistentColumnsProjection = formatPlugin.getConfig().isNonExistentFieldSupport(); - - // Do not use cached table handle for two reasons. - // cached table handles default timeout is 60 min after which those handles will become stale. - // Since execution can run for longer than 60 min, we want to get a new table handle and use it - // instead of the one from cache. - // Since we are setting some table options, we do not want to use shared handles. - // - // Call it here instead of setup since this will make sure it's called under correct UGI block when impersonation - // is enabled and table is used with and without views. - table = (indexDesc == null ? MapRDBImpl.getTable(tablePath) : MapRDBImpl.getIndexTable(indexDesc)); - - if (condition != null) { - logger.debug("Created record reader with query condition {}", condition.toString()); - } else { - logger.debug("Created record reader with query condition NULL"); - } - } - - @Override - protected Collection transformColumns(Collection columns) { - Set transformed = Sets.newLinkedHashSet(); - Set encodedSchemaPathSet = Sets.newLinkedHashSet(); - - if (disablePushdown) { - transformed.add(SchemaPath.STAR_COLUMN); - includeId = true; - } else { - if (isStarQuery()) { - transformed.add(SchemaPath.STAR_COLUMN); - includeId = true; - if (isSkipQuery() && !disableCountOptimization) { - // `SELECT COUNT(*)` query - idOnly = true; - scannedFields = ID_ONLY_PROJECTION; - } - } else { - Set scannedFieldsSet = Sets.newTreeSet(); - Set projectedFieldsSet = null; - - for (SchemaPath column : columns) { - if (EncodedSchemaPathSet.isEncodedSchemaPath(column)) { - encodedSchemaPathSet.add(column); - } else { - transformed.add(column); - if (!DOCUMENT_SCHEMA_PATH.equals(column)) { - FieldPath fp = getFieldPathForProjection(column); - scannedFieldsSet.add(fp); - } else { - projectWholeDocument = true; - } - } - } - if (projectWholeDocument) { - // we do not want to project the fields from the encoded field path list - // hence make a copy of the scannedFieldsSet here for projection. - projectedFieldsSet = new ImmutableSet.Builder() - .addAll(scannedFieldsSet).build(); - } - - if (encodedSchemaPathSet.size() > 0) { - Collection decodedSchemaPaths = EncodedSchemaPathSet.decode(encodedSchemaPathSet); - // now we look at the fields which are part of encoded field set and either - // add them to scanned set or clear the scanned set if all fields were requested. - for (SchemaPath column : decodedSchemaPaths) { - if (column.equals(SchemaPath.STAR_COLUMN)) { - includeId = true; - scannedFieldsSet.clear(); - break; - } - scannedFieldsSet.add(getFieldPathForProjection(column)); - } - } - - if (scannedFieldsSet.size() > 0) { - if (includesIdField(scannedFieldsSet)) { - includeId = true; - } - scannedFields = scannedFieldsSet.toArray(new FieldPath[scannedFieldsSet.size()]); - } - - if (disableCountOptimization) { - idOnly = (scannedFields == null); - } - - if (projectWholeDocument) { - projector = new FieldProjector(projectedFieldsSet); - } - - } - } - return transformed; - } - - protected FieldPath[] getScannedFields() { - return scannedFields; - } - - protected boolean getIdOnly() { - return idOnly; - } - - protected Table getTable() { - return table; - } - - protected boolean getIgnoreSchemaChange() { - return ignoreSchemaChange; - } - - @Override - public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException { - this.vectorWriter = new VectorContainerWriter(output, unionEnabled); - this.vectorWriterMutator = output; - this.operatorContext = context; - - try { - table.setOption(TableOption.EXCLUDEID, !includeId); - documentStream = table.find(condition, scannedFields); - documentIterator = documentStream.iterator(); - setupWriter(); - } catch (DBException ex) { - throw new ExecutionSetupException(ex); - } - } - - /** - * Setup the valueWriter and documentWriters based on config options. - */ - private void setupWriter() { - if (allTextMode) { - if (readTimestampWithZoneOffset) { - valueWriter = new AllTextValueWriter(buffer) { - /** - * Applies local time zone offset to timestamp value read using specified {@code reader}. - * - * @param writer writer to store string representation of timestamp value - * @param fieldName name of the field - * @param reader document reader - */ - @Override - protected void writeTimeStamp(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - String formattedTimestamp = Instant.ofEpochMilli(reader.getTimestampLong()) - .atZone(ZoneId.systemDefault()).format(DateUtility.UTC_FORMATTER); - writeString(writer, fieldName, formattedTimestamp); - } - }; - } else { - valueWriter = new AllTextValueWriter(buffer); - } - } else if (readNumbersAsDouble) { - if (readTimestampWithZoneOffset) { - valueWriter = new NumbersAsDoubleValueWriter(buffer) { - @Override - protected void writeTimeStamp(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writeTimestampWithLocalZoneOffset(writer, fieldName, reader); - } - }; - } else { - valueWriter = new NumbersAsDoubleValueWriter(buffer); - } - } else { - if (readTimestampWithZoneOffset) { - valueWriter = new OjaiValueWriter(buffer) { - @Override - protected void writeTimeStamp(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writeTimestampWithLocalZoneOffset(writer, fieldName, reader); - } - }; - } else { - valueWriter = new OjaiValueWriter(buffer); - } - } - - if (projectWholeDocument) { - documentWriter = new ProjectionPassthroughVectorWriter(valueWriter, projector, includeId); - } else if (isSkipQuery()) { - documentWriter = new RowCountVectorWriter(valueWriter); - } else if (idOnly) { - documentWriter = new IdOnlyVectorWriter(valueWriter); - } else { - documentWriter = new FieldTransferVectorWriter(valueWriter); - } - } - - /** - * Applies local time zone offset to timestamp value read using specified {@code reader}. - * - * @param writer writer to store timestamp value - * @param fieldName name of the field - * @param reader document reader - */ - private void writeTimestampWithLocalZoneOffset(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - Instant utcInstant = Instant.ofEpochMilli(reader.getTimestampLong()); - ZonedDateTime localZonedDateTime = utcInstant.atZone(ZoneId.systemDefault()); - ZonedDateTime convertedZonedDateTime = localZonedDateTime.withZoneSameLocal(ZoneId.of("UTC")); - long timeStamp = convertedZonedDateTime.toInstant().toEpochMilli(); - writer.timeStamp(fieldName).writeTimeStamp(timeStamp); - } - - @Override - public int next() { - Stopwatch watch = Stopwatch.createUnstarted(); - watch.start(); - - vectorWriter.allocate(); - vectorWriter.reset(); - - int recordCount = 0; - reader = null; - document = null; - - int maxRecordsForThisBatch = this.maxRecordsToRead >= 0? - Math.min(BaseValueVector.INITIAL_VALUE_ALLOCATION, this.maxRecordsToRead) : BaseValueVector.INITIAL_VALUE_ALLOCATION; - - try { - // If the last document caused a SchemaChange create a new output schema for this scan batch - if (schemaState == SchemaState.SCHEMA_CHANGE && !ignoreSchemaChange) { - // Clear the ScanBatch vector container writer/mutator in order to be able to generate the new schema - vectorWriterMutator.clear(); - vectorWriter = new VectorContainerWriter(vectorWriterMutator, unionEnabled); - logger.debug("Encountered schema change earlier use new writer {}", vectorWriter.toString()); - document = lastDocument; - setupWriter(); - if (recordCount < maxRecordsForThisBatch) { - vectorWriter.setPosition(recordCount); - if (document != null) { - reader = (DBDocumentReaderBase) document.asReader(); - documentWriter.writeDBDocument(vectorWriter, reader); - recordCount++; - } - } - } - } catch (SchemaChangeException e) { - String err_row = reader.getId().asJsonString(); - if (ignoreSchemaChange) { - logger.warn("{}. Dropping row '{}' from result.", e.getMessage(), err_row); - logger.debug("Stack trace:", e); - } else { - /* We should not encounter a SchemaChangeException here since this is the first document for this - * new schema. Something is very wrong - cannot handle any further! - */ - throw dataReadError(logger, e, "SchemaChangeException for row '%s'.", err_row); - } - } - schemaState = SchemaState.SCHEMA_INIT; - while(recordCount < maxRecordsForThisBatch) { - vectorWriter.setPosition(recordCount); - try { - document = nextDocument(); - if (document == null) { - break; // no more documents for this reader - } else { - documentWriter.writeDBDocument(vectorWriter, (DBDocumentReaderBase) document.asReader()); - } - recordCount++; - } catch (UserException e) { - throw UserException.unsupportedError(e) - .addContext(String.format("Table: %s, document id: '%s'", - table.getPath(), - document.asReader() == null ? null : - IdCodec.asString(((DBDocumentReaderBase)document.asReader()).getId()))) - .build(logger); - } catch (SchemaChangeException e) { - String err_row = ((DBDocumentReaderBase)document.asReader()).getId().asJsonString(); - if (ignoreSchemaChange) { - logger.warn("{}. Dropping row '{}' from result.", e.getMessage(), err_row); - logger.debug("Stack trace:", e); - } else { - /* Save the current document reader for next iteration. The recordCount is not updated so we - * would start from this reader on the next next() call - */ - lastDocument = document; - schemaState = SchemaState.SCHEMA_CHANGE; - break; - } - } - } - - if (nonExistentColumnsProjection && recordCount > 0) { - if (schema == null || schema.isEmpty()) { - JsonReaderUtils.ensureAtLeastOneField(vectorWriter, getColumns(), allTextMode, Collections.emptyList()); - } else { - JsonReaderUtils.writeColumnsUsingSchema(vectorWriter, getColumns(), schema, allTextMode); - } - } - vectorWriter.setValueCount(recordCount); - if (maxRecordsToRead > 0) { - maxRecordsToRead -= recordCount; - } - logger.debug("Took {} ms to get {} records", watch.elapsed(TimeUnit.MILLISECONDS), recordCount); - return recordCount; - } - - protected DBDocumentReaderBase nextDocumentReader() { - final OperatorStats operatorStats = operatorContext == null ? null : operatorContext.getStats(); - try { - if (operatorStats != null) { - operatorStats.startWait(); - } - try { - if (!documentReaderIterators.hasNext()) { - return null; - } else { - return (DBDocumentReaderBase) documentReaderIterators.next(); - } - } finally { - if (operatorStats != null) { - operatorStats.stopWait(); - } - } - } catch (DBException e) { - throw dataReadError(logger, e); - } - } - - protected Document nextDocument() { - final OperatorStats operatorStats = operatorContext == null ? null : operatorContext.getStats(); - try { - if (operatorStats != null) { - operatorStats.startWait(); - } - try { - if (!documentIterator.hasNext()) { - return null; - } else { - return documentIterator.next(); - } - } finally { - if (operatorStats != null) { - operatorStats.stopWait(); - } - } - } catch (DBException e) { - throw dataReadError(logger, e); - } - } - /* - * Extracts contiguous named segments from the SchemaPath, starting from the - * root segment and build the FieldPath from it for projection. - * - * This is due to bug 22726 and 22727, which cause DB's DocumentReaders to - * behave incorrectly for sparse lists, hence we avoid projecting beyond the - * first encountered ARRAY field and let Drill handle the projection. - */ - private static FieldPath getFieldPathForProjection(SchemaPath column) { - Stack pathSegments = new Stack<>(); - PathSegment seg = column.getRootSegment(); - while (seg != null && seg.isNamed()) { - pathSegments.push((PathSegment.NameSegment) seg); - seg = seg.getChild(); - } - FieldSegment.NameSegment child = null; - while (!pathSegments.isEmpty()) { - child = new FieldSegment.NameSegment(pathSegments.pop().getPath(), child, false); - } - return new FieldPath(child); - } - - public static boolean includesIdField(Collection projected) { - return Iterables.tryFind(projected, path -> Preconditions.checkNotNull(path).equals(ID_FIELD)).isPresent(); - } - - @Override - public void close() { - if (documentStream != null) { - documentStream.close(); - } - if (table != null) { - table.close(); - } - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("MaprDBJsonRecordReader[Table=") - .append(table != null ? table.getPath() : null); - if (reader != null) { - sb.append(", Document ID=") - .append(IdCodec.asString(reader.getId())); - } - sb.append(", reader=") - .append(reader) - .append(']'); - return sb.toString(); - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/NumbersAsDoubleValueWriter.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/NumbersAsDoubleValueWriter.java deleted file mode 100644 index d7d38cbfa79..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/NumbersAsDoubleValueWriter.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import org.apache.drill.exec.vector.complex.impl.MapOrListWriterImpl; -import org.ojai.DocumentReader; - -import io.netty.buffer.DrillBuf; - -public class NumbersAsDoubleValueWriter extends OjaiValueWriter { - - public NumbersAsDoubleValueWriter(DrillBuf buffer) { - super(buffer); - } - - protected void writeFloat(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writer.float8(fieldName).writeFloat8(reader.getFloat()); - } - - protected void writeLong(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writer.float8(fieldName).writeFloat8(reader.getLong()); - } - - protected void writeInt(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writer.float8(fieldName).writeFloat8(reader.getInt()); - } - - protected void writeShort(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writer.float8(fieldName).writeFloat8(reader.getShort()); - } - - protected void writeByte(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writer.float8(fieldName).writeFloat8(reader.getByte()); - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/OjaiFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/OjaiFunctionsProcessor.java deleted file mode 100644 index 2a43797cf42..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/OjaiFunctionsProcessor.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import org.apache.commons.codec.binary.Base64; - -import org.apache.drill.common.expression.FunctionCall; -import org.apache.drill.common.expression.LogicalExpression; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.expression.ValueExpressions.IntExpression; -import org.apache.drill.common.expression.ValueExpressions.LongExpression; -import org.apache.drill.common.expression.ValueExpressions.QuotedString; -import org.apache.drill.common.expression.visitors.AbstractExprVisitor; -import com.google.common.collect.ImmutableMap; -import org.ojai.Value; -import org.ojai.store.QueryCondition; - -import com.mapr.db.impl.ConditionImpl; -import com.mapr.db.impl.MapRDBImpl; - -import java.nio.ByteBuffer; - -class OjaiFunctionsProcessor extends AbstractExprVisitor { - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OjaiFunctionsProcessor.class); - private QueryCondition queryCond; - - private OjaiFunctionsProcessor() { - } - - private static String getStackTrace() { - final Throwable throwable = new Throwable(); - final StackTraceElement[] ste = throwable.getStackTrace(); - final StringBuilder sb = new StringBuilder(); - for (int i = 1; i < ste.length; ++i) { - sb.append(ste[i].toString()); - sb.append('\n'); - } - - return sb.toString(); - } - - @Override - public Void visitUnknown(LogicalExpression e, Void valueArg) throws RuntimeException { - logger.debug("visitUnknown() e class " + e.getClass()); - logger.debug(getStackTrace()); - return null; - } - - private static class Ref { - T value; - } - - private static SchemaPath getSchemaPathArg(LogicalExpression expr) { - final Ref ref = new Ref<>(); - expr.accept(new OjaiFunctionsProcessor() { - @Override - public Void visitSchemaPath(SchemaPath e, Void v) { - ref.value = e; - return null; - } - }, null); - - return ref.value; - } - - private static String getStringArg(LogicalExpression expr) { - final Ref ref = new Ref<>(); - expr.accept(new OjaiFunctionsProcessor() { - @Override - public Void visitQuotedStringConstant(QuotedString e, Void v) { - ref.value = e; - return null; - } - }, null); - - return ref.value != null ? ref.value.getString() : null; - } - - private static int getIntArg(LogicalExpression expr) { - final Ref ref = new Ref<>(); - expr.accept(new OjaiFunctionsProcessor() { - @Override - public Void visitIntConstant(IntExpression e, Void v) { - ref.value = new Integer(e.getInt()); - return null; - } - }, null); - - return ref.value != null ? ref.value.intValue() : 0; - } - - private static long getLongArg(LogicalExpression expr) { - final Ref ref = new Ref<>(); - expr.accept(new OjaiFunctionsProcessor() { - @Override - public Void visitIntConstant(IntExpression e, Void v) { - ref.value = new Long(e.getInt()); - return null; - } - - @Override - public Void visitLongConstant(LongExpression e, Void v) { - ref.value = e.getLong(); - return null; - } - }, null); - - return ref.value != null ? ref.value.longValue() : 0; - } - - private final static ImmutableMap STRING_TO_RELOP; - static { - ImmutableMap.Builder builder = ImmutableMap.builder(); - STRING_TO_RELOP = builder - .put("=", QueryCondition.Op.EQUAL) - .put("<>", QueryCondition.Op.NOT_EQUAL) - .put("<", QueryCondition.Op.LESS) - .put("<=", QueryCondition.Op.LESS_OR_EQUAL) - .put(">", QueryCondition.Op.GREATER) - .put(">=", QueryCondition.Op.GREATER_OR_EQUAL) - .build(); - } - - @Override - public Void visitFunctionCall(FunctionCall call, Void v) throws RuntimeException { - final String functionName = call.getName(); - final String fieldName = FieldPathHelper.schemaPath2FieldPath(getSchemaPathArg(call.arg(0))).asPathString(); - switch(functionName) { - case "ojai_sizeof": { - // ojai_sizeof(field, "", ) - final String relOp = getStringArg(call.arg(1)); - final long size = getLongArg(call.arg(2)); - queryCond = MapRDBImpl.newCondition() - .sizeOf(fieldName, STRING_TO_RELOP.get(relOp), size) - .build(); - break; - } - - case "ojai_typeof": - case "ojai_nottypeof": { - // ojai_[not]typeof(field, ); - final int typeCode = getIntArg(call.arg(1)); - final Value.Type typeValue = Value.Type.valueOf(typeCode); - queryCond = MapRDBImpl.newCondition(); - if (functionName.equals("ojai_typeof")) { - queryCond.typeOf(fieldName, typeValue); - } else { - queryCond.notTypeOf(fieldName, typeValue); - } - queryCond.build(); - break; - } - - case "ojai_matches": - case "ojai_notmatches": { - // ojai_[not]matches(field, ); - final String regex = getStringArg(call.arg(1)); - if (functionName.equals("ojai_matches")) { - queryCond = MapRDBImpl.newCondition() - .matches(fieldName, regex); - } else { - queryCond = MapRDBImpl.newCondition() - .notMatches(fieldName, regex); - } - queryCond.build(); - break; - } - - case "ojai_condition": { - // ojai_condition(field, ); - final String condString = getStringArg(call.arg(1)); - final byte[] condBytes = Base64.decodeBase64(condString); - final ByteBuffer condBuffer = ByteBuffer.wrap(condBytes); - queryCond = ConditionImpl.parseFrom(condBuffer); - break; - } - - default: - throw new IllegalArgumentException("unrecognized functionName " + functionName); - } // switch(functionName) - - return null; - } - - public static OjaiFunctionsProcessor process(FunctionCall call) { - final OjaiFunctionsProcessor processor = new OjaiFunctionsProcessor(); - - call.accept(processor, null); - return processor; - } - - public QueryCondition getCondition() { - return queryCond; - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/OjaiValueWriter.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/OjaiValueWriter.java deleted file mode 100644 index 57aad06de94..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/OjaiValueWriter.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import static org.apache.drill.exec.store.mapr.PluginErrorHandler.schemaChangeException; -import static org.apache.drill.exec.store.mapr.PluginErrorHandler.unsupportedError; - -import java.nio.ByteBuffer; - -import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.vector.complex.impl.MapOrListWriterImpl; -import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapOrListWriter; -import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; -import org.ojai.DocumentReader; -import org.ojai.DocumentReader.EventType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.mapr.org.apache.hadoop.hbase.util.Bytes; - -import io.netty.buffer.DrillBuf; - -public class OjaiValueWriter { - protected static final Logger logger = LoggerFactory.getLogger(OjaiValueWriter.class); - - protected static final long MILLISECONDS_IN_A_DAY = (long)1000 * 60 * 60 * 24; - - protected DrillBuf buffer; - - public OjaiValueWriter(DrillBuf buffer) { - this.buffer = buffer; - } - - /* - * Precondition to call this function is that the DBDocumentReader has already emitted START_MAP/START_ARRAY event. - */ - protected void writeToListOrMap(MapOrListWriterImpl writer, DocumentReader reader) throws SchemaChangeException { - String fieldName = null; - writer.start(); - outside: while (true) { - EventType event = reader.next(); - if (event == null - || event == EventType.END_MAP - || event == EventType.END_ARRAY) { - break outside; - } else if (reader.inMap()) { - fieldName = reader.getFieldName(); - } - - try { - switch (event) { - case NULL: - break; // not setting the field will leave it as null - case BINARY: - writeBinary(writer, fieldName, reader.getBinary()); - break; - case BOOLEAN: - writeBoolean(writer, fieldName, reader); - break; - case STRING: - writeString(writer, fieldName, reader.getString()); - break; - case BYTE: - writeByte(writer, fieldName, reader); - break; - case SHORT: - writeShort(writer, fieldName, reader); - break; - case INT: - writeInt(writer, fieldName, reader); - break; - case LONG: - writeLong(writer, fieldName, reader); - break; - case FLOAT: - writeFloat(writer, fieldName, reader); - break; - case DOUBLE: - writeDouble(writer, fieldName, reader); - break; - case DECIMAL: - throw unsupportedError(logger, "Decimal type is currently not supported."); - case DATE: - writeDate(writer, fieldName, reader); - break; - case TIME: - writeTime(writer, fieldName, reader); - break; - case TIMESTAMP: - writeTimeStamp(writer, fieldName, reader); - break; - case INTERVAL: - throw unsupportedError(logger, "Interval type is currently not supported."); - case START_MAP: - writeToListOrMap((MapOrListWriterImpl) (reader.inMap() ? writer.map(fieldName) : writer.listoftmap(fieldName)), reader); - break; - case START_ARRAY: - writeToListOrMap((MapOrListWriterImpl) writer.list(fieldName), reader); - break; - default: - throw unsupportedError(logger, "Unsupported type: %s encountered during the query.", event); - } - } catch (IllegalStateException | IllegalArgumentException e) { - throw schemaChangeException(logger, e, "Possible schema change for field: '%s'", fieldName); - } - } - writer.end(); - } - - protected void writeTimeStamp(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writer.timeStamp(fieldName).writeTimeStamp(reader.getTimestampLong()); - } - - protected void writeTime(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writer.time(fieldName).writeTime(reader.getTimeInt()); - } - - protected void writeDate(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - long milliSecondsSinceEpoch = reader.getDateInt() * MILLISECONDS_IN_A_DAY; - writer.date(fieldName).writeDate(milliSecondsSinceEpoch); - } - - protected void writeDouble(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writer.float8(fieldName).writeFloat8(reader.getDouble()); - } - - protected void writeFloat(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writer.float4(fieldName).writeFloat4(reader.getFloat()); - } - - protected void writeLong(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writer.bigInt(fieldName).writeBigInt(reader.getLong()); - } - - protected void writeInt(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writer.integer(fieldName).writeInt(reader.getInt()); - } - - protected void writeShort(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writer.smallInt(fieldName).writeSmallInt(reader.getShort()); - } - - protected void writeByte(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writer.tinyInt(fieldName).writeTinyInt(reader.getByte()); - } - - protected void writeBoolean(MapOrListWriterImpl writer, String fieldName, DocumentReader reader) { - writer.bit(fieldName).writeBit(reader.getBoolean() ? 1 : 0); - } - - protected void writeBinary(MapOrListWriter writer, String fieldName, ByteBuffer buf) { - int bufLen = buf.remaining(); - buffer = buffer.reallocIfNeeded(bufLen); - buffer.setBytes(0, buf, buf.position(), bufLen); - writer.varBinary(fieldName).writeVarBinary(0, bufLen, buffer); - } - - protected void writeString(MapOrListWriter writer, String fieldName, String value) { - final byte[] strBytes = Bytes.toBytes(value); - buffer = buffer.reallocIfNeeded(strBytes.length); - buffer.setBytes(0, strBytes); - writer.varChar(fieldName).writeVarChar(0, strBytes.length, buffer); - } - - protected void writeBinary(MapWriter writer, String fieldName, ByteBuffer buf) { - int bufLen = buf.remaining(); - buffer = buffer.reallocIfNeeded(bufLen); - buffer.setBytes(0, buf, buf.position(), bufLen); - writer.varBinary(fieldName).writeVarBinary(0, bufLen, buffer); - } - - protected void writeString(MapWriter writer, String fieldName, String value) { - final byte[] strBytes = Bytes.toBytes(value); - buffer = buffer.reallocIfNeeded(strBytes.length); - buffer.setBytes(0, strBytes); - writer.varChar(fieldName).writeVarChar(0, strBytes.length, buffer); - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/ProjectionPassthroughVectorWriter.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/ProjectionPassthroughVectorWriter.java deleted file mode 100644 index 9d819e8a6fa..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/ProjectionPassthroughVectorWriter.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import static org.apache.drill.exec.store.mapr.PluginErrorHandler.dataReadError; - -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.Map.Entry; - -import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.vector.complex.impl.MapOrListWriterImpl; -import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; -import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapOrListWriter; -import org.ojai.DocumentConstants; -import org.ojai.DocumentReader.EventType; -import org.ojai.util.DocumentReaderWithProjection; -import org.ojai.util.FieldProjector; -import com.google.common.base.Preconditions; -import com.mapr.db.DBConstants; -import com.mapr.db.ojai.DBDocumentReaderBase; - -/** - * This implementation of DocumentReaderVectorWriter writes the encoded MapR-DB OJAI Document - * as binary data along with the fields required to execute Drill's operators. - */ -class ProjectionPassthroughVectorWriter extends DocumentReaderVectorWriter { - - private final boolean includeId; - private final FieldProjector projector; - - protected ProjectionPassthroughVectorWriter(final OjaiValueWriter valueWriter, - final FieldProjector projector, final boolean includeId) { - super(valueWriter); - this.includeId = includeId; - this.projector = Preconditions.checkNotNull(projector); - } - - @Override - protected void writeDBDocument(VectorContainerWriter vectorWriter, DBDocumentReaderBase reader) - throws SchemaChangeException { - if (reader.next() != EventType.START_MAP) { - throw dataReadError(logger, "The document did not start with START_MAP!"); - } - - MapOrListWriterImpl writer = new MapOrListWriterImpl(vectorWriter.rootAsMap()); - writer.start(); - MapOrListWriter documentMapWriter = writer.map(DBConstants.DOCUMENT_FIELD); - documentMapWriter.start(); - - // write _id field data - if (includeId) { - valueWriter.writeBinary(documentMapWriter, DocumentConstants.ID_KEY, reader.getIdData()); - } - - // write rest of the data buffers - Map dataMap = reader.getDataMap(); - for (Entry familyData : dataMap.entrySet()) { - valueWriter.writeBinary(documentMapWriter, String.valueOf(familyData.getKey()), familyData.getValue()); - } - documentMapWriter.end(); - - DocumentReaderWithProjection p = new DocumentReaderWithProjection(reader, projector); - valueWriter.writeToListOrMap(writer, p); - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/RestrictedJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/RestrictedJsonRecordReader.java deleted file mode 100644 index 4c6649a9ecc..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/RestrictedJsonRecordReader.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import com.mapr.db.Table; -import static org.apache.drill.exec.store.mapr.PluginErrorHandler.dataReadError; - -import java.nio.ByteBuffer; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import com.mapr.db.impl.BaseJsonTable; -import com.mapr.db.impl.MultiGet; - -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.common.exceptions.UserException; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.exec.ExecConstants; -import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.ops.FragmentContext; -import org.apache.drill.exec.ops.OperatorContext; -import org.apache.drill.exec.physical.impl.OutputMutator; -import org.apache.drill.exec.physical.impl.join.RowKeyJoin; -import org.apache.drill.exec.record.AbstractRecordBatch; -import org.apache.drill.exec.record.metadata.TupleMetadata; -import org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin; -import org.apache.drill.exec.store.mapr.db.MapRDBSubScanSpec; -import org.apache.drill.exec.store.mapr.db.RestrictedMapRDBSubScanSpec; - -import com.google.common.base.Stopwatch; -import com.mapr.db.impl.IdCodec; -import com.mapr.db.ojai.DBDocumentReaderBase; - -import org.ojai.Document; -import org.ojai.DocumentStream; -import org.ojai.FieldPath; - - -public class RestrictedJsonRecordReader extends MaprDBJsonRecordReader { - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RestrictedJsonRecordReader.class); - - private int batchSize; // batchSize for rowKey based document get - - private String [] projections = null; // multiGet projections - - public RestrictedJsonRecordReader(MapRDBSubScanSpec subScanSpec, - MapRDBFormatPlugin formatPlugin, - List projectedColumns, - FragmentContext context, - int maxRecordsToRead, - TupleMetadata schema) { - - super(subScanSpec, formatPlugin, projectedColumns, context, maxRecordsToRead, schema); - batchSize = (int)context.getOptions().getOption(ExecConstants.QUERY_ROWKEYJOIN_BATCHSIZE); - int idx = 0; - FieldPath[] scannedFields = this.getScannedFields(); - - // only populate projections for non-star query (for star, null is interpreted as all fields) - if (!this.isStarQuery() && scannedFields != null && scannedFields.length > 0) { - projections = new String[scannedFields.length]; - for (FieldPath path : scannedFields) { - projections[idx] = path.asPathString(); - ++idx; - } - } - } - - public void readToInitSchema() { - DBDocumentReaderBase reader = null; - vectorWriter.setPosition(0); - - try (DocumentStream dstream = table.find()) { - reader = (DBDocumentReaderBase) dstream.iterator().next().asReader(); - documentWriter.writeDBDocument(vectorWriter, reader); - } catch(UserException e) { - throw UserException.unsupportedError(e) - .addContext(String.format("Table: %s, document id: '%s'", - getTable().getPath(), - reader == null ? null : IdCodec.asString(reader.getId()))) - .build(logger); - } catch (SchemaChangeException e) { - if (getIgnoreSchemaChange()) { - logger.warn("{}. Dropping the row from result.", e.getMessage()); - logger.debug("Stack trace:", e); - } else { - throw dataReadError(logger, e); - } - } - finally { - vectorWriter.setPosition(0); - } - } - - @Override - public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException { - RestrictedMapRDBSubScanSpec rss = ((RestrictedMapRDBSubScanSpec) this.subScanSpec); - RowKeyJoin rjBatch = rss.getJoinForSubScan(); - if (rjBatch == null) { - throw new ExecutionSetupException("RowKeyJoin Batch is not setup for Restricted MapRDB Subscan"); - } - - AbstractRecordBatch.BatchState state = rjBatch.getBatchState(); - if ( state == AbstractRecordBatch.BatchState.BUILD_SCHEMA || - state == AbstractRecordBatch.BatchState.FIRST) { - super.setup(context, output); - } - return; - } - - @Override - public int next() { - Stopwatch watch = Stopwatch.createUnstarted(); - watch.start(); - RestrictedMapRDBSubScanSpec rss = ((RestrictedMapRDBSubScanSpec) this.subScanSpec); - - vectorWriter.allocate(); - vectorWriter.reset(); - - if (!rss.readyToGetRowKey()) { - // not ready to get rowkey, so we just load a record to initialize schema; only do this - // when we are in the build schema phase - if (rss.isBuildSchemaPhase()) { - readToInitSchema(); - } - return 0; - } - - Table table = super.formatPlugin.getJsonTableCache().getTable(subScanSpec.getTableName(), subScanSpec.getUserName()); - final MultiGet multiGet = new MultiGet((BaseJsonTable) table, condition, false, projections); - int recordCount = 0; - DBDocumentReaderBase reader = null; - - int maxRecordsForThisBatch = this.maxRecordsToRead > 0? - Math.min(rss.getMaxRowKeysToBeRead(), this.maxRecordsToRead) : - this.maxRecordsToRead == -1 ? rss.getMaxRowKeysToBeRead() : 0; - - Stopwatch timer = Stopwatch.createUnstarted(); - - while (recordCount < maxRecordsForThisBatch) { - ByteBuffer rowKeyIds[] = rss.getRowKeyIdsToRead(batchSize); - if (rowKeyIds == null) { - break; - } - try { - timer.start(); - final List docList = multiGet.doGet(rowKeyIds); - int index = 0; - long docsToRead = docList.size(); - // If limit pushdown then stop once we have `limit` rows from multiget i.e. maxRecordsForThisBatch - if (this.maxRecordsToRead != -1) { - docsToRead = Math.min(docsToRead, maxRecordsForThisBatch); - } - while (index < docsToRead) { - vectorWriter.setPosition(recordCount); - reader = (DBDocumentReaderBase) docList.get(index).asReader(); - documentWriter.writeDBDocument(vectorWriter, reader); - recordCount++; - index++; - } - timer.stop(); - } catch (UserException e) { - throw UserException.unsupportedError(e).addContext(String.format("Table: %s, document id: '%s'", - getTable().getPath(), reader == null ? null : IdCodec.asString(reader.getId()))).build(logger); - } catch (SchemaChangeException e) { - if (getIgnoreSchemaChange()) { - logger.warn("{}. Dropping the row from result.", e.getMessage()); - logger.debug("Stack trace:", e); - } else { - throw dataReadError(logger, e); - } - } - } - - vectorWriter.setValueCount(recordCount); - if (maxRecordsToRead > 0) { - if (maxRecordsToRead - recordCount >= 0) { - maxRecordsToRead -= recordCount; - } else { - maxRecordsToRead = 0; - } - } - - logger.debug("Took {} ms to get {} records, getrowkey {}", watch.elapsed(TimeUnit.MILLISECONDS), recordCount, timer.elapsed(TimeUnit.MILLISECONDS)); - return recordCount; - } - - @Override - public boolean hasNext() { - RestrictedMapRDBSubScanSpec rss = ((RestrictedMapRDBSubScanSpec) this.subScanSpec); - - RowKeyJoin rjBatch = rss.getJoinForSubScan(); - if (rjBatch == null) { - return false; - } - - boolean hasMore = false; - AbstractRecordBatch.BatchState state = rss.getJoinForSubScan().getBatchState(); - RowKeyJoin.RowKeyJoinState rkState = rss.getJoinForSubScan().getRowKeyJoinState(); - if ( state == AbstractRecordBatch.BatchState.BUILD_SCHEMA ) { - hasMore = true; - } else if ( state == AbstractRecordBatch.BatchState.FIRST) { - if (this.maxRecordsToRead > 0) { - rss.getJoinForSubScan().setBatchState(AbstractRecordBatch.BatchState.NOT_FIRST); - rss.getJoinForSubScan().setRowKeyJoinState(RowKeyJoin.RowKeyJoinState.PROCESSING); - hasMore = true; - } - } else if ( rkState == RowKeyJoin.RowKeyJoinState.INITIAL) { - if (this.maxRecordsToRead > 0) { - rss.getJoinForSubScan().setRowKeyJoinState(RowKeyJoin.RowKeyJoinState.PROCESSING); - hasMore = true; - } - } - - logger.debug("restricted reader hasMore = {}", hasMore); - - return hasMore; - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/RestrictedJsonTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/RestrictedJsonTableGroupScan.java deleted file mode 100644 index 790700638a7..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/RestrictedJsonTableGroupScan.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import java.util.List; -import java.util.NavigableMap; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import org.apache.drill.exec.record.metadata.TupleMetadata; -import org.apache.drill.exec.metastore.store.FileSystemMetadataProviderManager; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.exec.physical.base.GroupScan; -import org.apache.drill.exec.physical.base.PhysicalOperator; -import org.apache.drill.exec.physical.base.ScanStats; -import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty; -import org.apache.drill.exec.planner.index.MapRDBStatistics; -import org.apache.drill.exec.planner.cost.PluginCost; -import org.apache.drill.exec.planner.index.Statistics; -import org.apache.drill.exec.store.dfs.FileSystemPlugin; -import org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin; -import org.apache.drill.exec.store.mapr.db.MapRDBSubScan; -import org.apache.drill.exec.store.mapr.db.MapRDBSubScanSpec; -import org.apache.drill.exec.store.mapr.db.RestrictedMapRDBSubScan; -import org.apache.drill.exec.store.mapr.db.RestrictedMapRDBSubScanSpec; -import org.apache.drill.exec.store.mapr.db.TabletFragmentInfo; - -/** - * A RestrictedJsonTableGroupScan encapsulates (along with a subscan) the functionality - * for doing restricted (i.e skip) scan rather than sequential scan. The skipping is based - * on a supplied set of row keys (primary keys) from a join operator. - */ -@JsonTypeName("restricted-json-scan") -public class RestrictedJsonTableGroupScan extends JsonTableGroupScan { - - @JsonCreator - public RestrictedJsonTableGroupScan(@JsonProperty("userName") String userName, - @JsonProperty("storage") FileSystemPlugin storagePlugin, - @JsonProperty("format") MapRDBFormatPlugin formatPlugin, - @JsonProperty("scanSpec") JsonScanSpec scanSpec, /* scan spec of the original table */ - @JsonProperty("columns") List columns, - @JsonProperty("") MapRDBStatistics statistics, - @JsonProperty("schema") TupleMetadata schema) { - super(userName, storagePlugin, formatPlugin, scanSpec, columns, - statistics, FileSystemMetadataProviderManager.getMetadataProviderForSchema(schema)); - } - - // TODO: this method needs to be fully implemented - protected RestrictedMapRDBSubScanSpec getSubScanSpec(TabletFragmentInfo tfi) { - JsonScanSpec spec = scanSpec; - RestrictedMapRDBSubScanSpec subScanSpec = - new RestrictedMapRDBSubScanSpec( - spec.getTableName(), - getRegionsToScan().get(tfi), spec.getSerializedFilter(), getUserName()); - return subScanSpec; - } - - protected NavigableMap getRegionsToScan() { - return getRegionsToScan(formatPlugin.getRestrictedScanRangeSizeMB()); - } - - @Override - public MapRDBSubScan getSpecificScan(int minorFragmentId) { - assert minorFragmentId < endpointFragmentMapping.size() : String.format( - "Mappings length [%d] should be greater than minor fragment id [%d] but it isn't.", endpointFragmentMapping.size(), - minorFragmentId); - RestrictedMapRDBSubScan subscan = - new RestrictedMapRDBSubScan(getUserName(), formatPlugin, - getEndPointFragmentMapping(minorFragmentId), columns, maxRecordsToRead, TABLE_JSON, getSchema()); - - return subscan; - } - - private List getEndPointFragmentMapping(int minorFragmentId) { - List restrictedSubScanSpecList = Lists.newArrayList(); - List subScanSpecList = endpointFragmentMapping.get(minorFragmentId); - for (MapRDBSubScanSpec s : subScanSpecList) { - restrictedSubScanSpecList.add((RestrictedMapRDBSubScanSpec) s); - } - return restrictedSubScanSpecList; - } - - /** - * Private constructor, used for cloning. - * @param that The RestrictedJsonTableGroupScan to clone - */ - private RestrictedJsonTableGroupScan(RestrictedJsonTableGroupScan that) { - super(that); - } - - @Override - public GroupScan clone(JsonScanSpec scanSpec) { - RestrictedJsonTableGroupScan newScan = new RestrictedJsonTableGroupScan(this); - newScan.scanSpec = scanSpec; - newScan.resetRegionsToScan(); // resetting will force recalculation - return newScan; - } - - @Override - public GroupScan clone(List columns) { - RestrictedJsonTableGroupScan newScan = new RestrictedJsonTableGroupScan(this); - newScan.columns = columns; - return newScan; - } - - @Override - @JsonIgnore - public PhysicalOperator getNewWithChildren(List children) { - Preconditions.checkArgument(children.isEmpty()); - return new RestrictedJsonTableGroupScan(this); - } - - @Override - public ScanStats getScanStats() { - // TODO: ideally here we should use the rowcount from index scan, and multiply a factor of restricted scan - double rowCount; - PluginCost pluginCostModel = formatPlugin.getPluginCostModel(); - final int avgColumnSize = pluginCostModel.getAverageColumnSize(this); - int numColumns = (columns == null || columns.isEmpty()) ? STAR_COLS: columns.size(); - // Get the restricted group scan row count - same as the right side index rows - rowCount = computeRestrictedScanRowcount(); - // Get the average row size of the primary table - double avgRowSize = stats.getAvgRowSize(null, true); - if (avgRowSize == Statistics.AVG_ROWSIZE_UNKNOWN || avgRowSize == 0) { - avgRowSize = avgColumnSize * numColumns; - } - // restricted scan does random lookups and each row may belong to a different block, with the number - // of blocks upper bounded by the total num blocks in the primary table - double totalBlocksPrimary = Math.ceil((avgRowSize * fullTableRowCount)/pluginCostModel.getBlockSize(this)); - double numBlocks = Math.min(totalBlocksPrimary, rowCount); - double diskCost = numBlocks * pluginCostModel.getRandomBlockReadCost(this); - // For non-covering plans, the dominating cost would be of the join back. Reduce it using the factor - // for biasing towards non-covering plans. - diskCost *= stats.getRowKeyJoinBackIOFactor(); - logger.debug("RestrictedJsonGroupScan:{} rowCount:{}, avgRowSize:{}, blocks:{}, totalBlocks:{}, diskCost:{}", - System.identityHashCode(this), rowCount, avgRowSize, numBlocks, totalBlocksPrimary, diskCost); - return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, rowCount, 1, diskCost); - } - - private double computeRestrictedScanRowcount() { - double rowCount = Statistics.ROWCOUNT_UNKNOWN; - // The rowcount should be the same as the build side which was FORCED by putting it in forcedRowCountMap - if (forcedRowCountMap.get(null) != null) { - rowCount = forcedRowCountMap.get(null); - } - // If limit pushdown has occurred - factor it in the rowcount - if (rowCount == Statistics.ROWCOUNT_UNKNOWN || rowCount == 0) { - rowCount = (0.001f * fullTableRowCount); - } - if (this.maxRecordsToRead > 0) { - rowCount = Math.min(rowCount, this.maxRecordsToRead); - } - return rowCount; - } - - @Override - public boolean isRestrictedScan() { - return true; - } - - @Override - public String toString() { - return "RestrictedJsonTableGroupScan [ScanSpec=" + scanSpec + ", columns=" + columns - + ", rowcount=" + computeRestrictedScanRowcount() - + (maxRecordsToRead > 0 ? ", limit=" + maxRecordsToRead : "") - + (getMaxParallelizationWidth() > 0 ? ", maxwidth=" + getMaxParallelizationWidth() : "") + "]"; - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/RowCountVectorWriter.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/RowCountVectorWriter.java deleted file mode 100644 index 445bccb05bf..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/RowCountVectorWriter.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.json; - -import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; - -import com.mapr.db.ojai.DBDocumentReaderBase; - -/** - * This is an optimized implementation of DocumentReaderVectorWriter that writes the row count. - */ -class RowCountVectorWriter extends DocumentReaderVectorWriter { - - protected RowCountVectorWriter(final OjaiValueWriter valueWriter) { - super(valueWriter); - } - - @Override - protected void writeDBDocument(VectorContainerWriter vectorWriter, DBDocumentReaderBase reader) - throws SchemaChangeException { - vectorWriter.rootAsMap().bit("count").writeBit(1); - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/util/CommonFns.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/util/CommonFns.java deleted file mode 100644 index 0409de161e8..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/util/CommonFns.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.util; - -public class CommonFns { - - public static boolean isNullOrEmpty(final byte[] key) { - return key == null || key.length == 0; - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/util/FieldPathHelper.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/util/FieldPathHelper.java deleted file mode 100644 index 16a5b68aab1..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/util/FieldPathHelper.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.db.util; - -import com.google.common.collect.Queues; -import org.apache.drill.common.expression.PathSegment; -import org.apache.drill.common.expression.SchemaPath; -import org.ojai.FieldPath; -import org.ojai.FieldSegment; -import java.util.Deque; - -public class FieldPathHelper { - - /** - * Returns {@link FieldPath} equivalent of the specified {@link SchemaPath}. - * - * @param schemaPath {@link SchemaPath} instance that should be converted - * @return {@link FieldPath} equivalent of the specified {@link SchemaPath}. - */ - public static FieldPath schemaPathToFieldPath(SchemaPath schemaPath) { - Deque pathSegments = Queues.newArrayDeque(); - PathSegment pathSegment = schemaPath.getRootSegment(); - while (pathSegment != null) { - pathSegments.push(pathSegment); - pathSegment = pathSegment.getChild(); - } - - FieldSegment child = null; - while (!pathSegments.isEmpty()) { - pathSegment = pathSegments.pop(); - if (pathSegment.isNamed()) { - child = new FieldSegment.NameSegment(((PathSegment.NameSegment) pathSegment).getPath(), child, false); - } else { - child = new FieldSegment.IndexSegment(String.valueOf(((PathSegment.ArraySegment) pathSegment).getIndex()), child); - } - } - return new FieldPath((FieldSegment.NameSegment) child); - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatMatcher.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatMatcher.java deleted file mode 100644 index e4ee2542f53..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatMatcher.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.streams; - -import java.io.IOException; - -import org.apache.drill.exec.store.mapr.TableFormatMatcher; -import org.apache.drill.exec.store.mapr.TableFormatPlugin; - -import com.mapr.fs.MapRFileStatus; - -public class StreamsFormatMatcher extends TableFormatMatcher { - - public StreamsFormatMatcher(TableFormatPlugin plugin) { - super(plugin); - } - - @Override - protected boolean isSupportedTable(MapRFileStatus status) throws IOException { - return getFormatPlugin() - .getMaprFS() - .getTableProperties(status.getPath()) - .getAttr() - .getIsMarlinTable(); - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPlugin.java deleted file mode 100644 index 2ddf7529637..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPlugin.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.streams; - -import java.io.IOException; -import java.util.List; - -import org.apache.drill.common.exceptions.UserException; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.common.logical.StoragePluginConfig; -import org.apache.drill.exec.physical.base.AbstractGroupScan; -import org.apache.drill.exec.physical.base.AbstractWriter; -import org.apache.drill.exec.physical.base.PhysicalOperator; -import org.apache.drill.exec.planner.common.DrillStatsTable; -import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.store.dfs.FileSelection; -import org.apache.drill.exec.store.dfs.FormatMatcher; -import org.apache.drill.exec.store.mapr.TableFormatPlugin; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; - -public class StreamsFormatPlugin extends TableFormatPlugin { - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StreamsFormatPlugin.class); - private StreamsFormatMatcher matcher; - - public StreamsFormatPlugin(String name, DrillbitContext context, Configuration fsConf, - StoragePluginConfig storageConfig, StreamsFormatPluginConfig formatConfig) { - super(name, context, fsConf, storageConfig, formatConfig); - matcher = new StreamsFormatMatcher(this); - } - - @Override - public boolean supportsRead() { - return true; - } - - @Override - public boolean supportsWrite() { - return false; - } - - @Override - public boolean supportsAutoPartitioning() { - return false; - } - - @Override - public FormatMatcher getMatcher() { - return matcher; - } - - @Override - public AbstractWriter getWriter(PhysicalOperator child, String location, - List partitionColumns) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public AbstractGroupScan getGroupScan(String userName, FileSelection selection, List columns) { - List files = selection.getFiles(); - assert (files.size() == 1); - //TableProperties props = getMaprFS().getTableProperties(new Path(files.get(0))); - throw UserException.unsupportedError().message("MapR streams can not be querried at this time.").build(logger); - } - - @Override - public boolean supportsStatistics() { - return false; - } - - @Override - public DrillStatsTable.TableStatistics readStatistics(FileSystem fs, Path statsTablePath) { - throw new UnsupportedOperationException("unimplemented"); - } - - @Override - public void writeStatistics(DrillStatsTable.TableStatistics statistics, FileSystem fs, Path statsTablePath) { - throw new UnsupportedOperationException("unimplemented"); - } -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPluginConfig.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPluginConfig.java deleted file mode 100644 index e78f1e8b43b..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/streams/StreamsFormatPluginConfig.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mapr.streams; - -import org.apache.drill.exec.store.mapr.TableFormatPluginConfig; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonTypeName; - -@JsonTypeName("streams") @JsonInclude(Include.NON_DEFAULT) -public class StreamsFormatPluginConfig extends TableFormatPluginConfig { - - @Override - public int hashCode() { - return 47; - } - - @Override - public boolean equals(Object that) { - if (this == that) { - return true; - } else if (that == null || getClass() != that.getClass()) { - return false; - } - return impEquals(that); - } - - @Override - protected boolean impEquals(Object obj) { - return true; // TODO: compare custom properties once added - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/ConditionPlaceholder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/ConditionPlaceholder.java deleted file mode 100644 index 3bc36b3f96a..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/ConditionPlaceholder.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.udf.mapr.db; - -import org.apache.drill.exec.expr.DrillSimpleFunc; -import org.apache.drill.exec.expr.annotations.FunctionTemplate; -import org.apache.drill.exec.expr.annotations.Output; -import org.apache.drill.exec.expr.annotations.Param; -import org.apache.drill.exec.expr.holders.BigIntHolder; -import org.apache.drill.exec.expr.holders.BitHolder; -import org.apache.drill.exec.expr.holders.VarCharHolder; - -/** - * This is a placeholder for the ojai_condition() function. - * - * At this time, this function can only be used in predicates. The placeholder - * is here to prevent calcite from complaining; the function will get pushed down - * by the storage plug-in into DB. That process will go through JsonConditionBuilder.java, - * which will replace this function with the real OJAI equivalent to be pushed down. - * Therefore, there's no implementation here. - */ -@FunctionTemplate( - name="ojai_condition", - scope=FunctionTemplate.FunctionScope.SIMPLE, - nulls=FunctionTemplate.NullHandling.INTERNAL) -public class ConditionPlaceholder implements DrillSimpleFunc { - - @Param BigIntHolder /*FieldReader*/ field; - @Param(constant = true) VarCharHolder pattern; - - @Output BitHolder output; - - public void setup() { - } - - public void eval() { - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/DecodeFieldPath.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/DecodeFieldPath.java deleted file mode 100644 index a8c22380438..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/DecodeFieldPath.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.udf.mapr.db; - -import javax.inject.Inject; - -import org.apache.drill.exec.expr.DrillSimpleFunc; -import org.apache.drill.exec.expr.annotations.FunctionTemplate; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope; -import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling; -import org.apache.drill.exec.expr.annotations.Output; -import org.apache.drill.exec.expr.annotations.Param; -import org.apache.drill.exec.expr.holders.VarCharHolder; - -import io.netty.buffer.DrillBuf; - -@FunctionTemplate(name = "maprdb_decode_fieldpath", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL) -public class DecodeFieldPath implements DrillSimpleFunc { - @Param VarCharHolder input; - @Output VarCharHolder out; - - @Inject DrillBuf buffer; - - @Override - public void setup() { - } - - @Override - public void eval() { - String[] encodedPaths = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers. - toStringFromUTF8(input.start, input.end, input.buffer).split(","); - String[] decodedPaths = org.apache.drill.exec.util.EncodedSchemaPathSet.decode(encodedPaths); - java.util.Arrays.sort(decodedPaths); - - StringBuilder sb = new StringBuilder(); - for (String decodedPath : decodedPaths) { - sb.append(", ").append(org.ojai.FieldPath.parseFrom(decodedPath).asPathString()); - } - String outputString = "[" + sb.substring(2) + "]"; - final byte[] strBytes = outputString.getBytes(java.nio.charset.StandardCharsets.UTF_8); - buffer.setBytes(0, strBytes); - buffer.setIndex(0, strBytes.length); - - out.start = 0; - out.end = strBytes.length; - out.buffer = buffer; - } - -} - diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/MatchesPlaceholder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/MatchesPlaceholder.java deleted file mode 100644 index 38f61c33781..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/MatchesPlaceholder.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.udf.mapr.db; - -import org.apache.drill.exec.expr.DrillSimpleFunc; -import org.apache.drill.exec.expr.annotations.FunctionTemplate; -import org.apache.drill.exec.expr.annotations.Output; -import org.apache.drill.exec.expr.annotations.Param; -import org.apache.drill.exec.expr.holders.BigIntHolder; -import org.apache.drill.exec.expr.holders.BitHolder; -import org.apache.drill.exec.expr.holders.VarCharHolder; - -/** - * This is a placeholder for the matches() function. - * - * At this time, this function can only be used in predicates. The placeholder - * is here to prevent calcite from complaining; the function will get pushed down - * by the storage plug-in into DB. That process will go through JsonConditionBuilder.java, - * which will replace this function with the real OJAI equivalent to be pushed down. - * Therefore, there's no implementation here. - */ -@FunctionTemplate(name = "ojai_matches", - scope = FunctionTemplate.FunctionScope.SIMPLE, - nulls = FunctionTemplate.NullHandling.INTERNAL) -public class MatchesPlaceholder implements DrillSimpleFunc { - - @Param BigIntHolder /*FieldReader*/ field; - @Param(constant = true) VarCharHolder pattern; - - @Output BitHolder output; - - public void setup() { - } - - public void eval() { - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/NotMatchesPlaceholder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/NotMatchesPlaceholder.java deleted file mode 100644 index 248579f6653..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/NotMatchesPlaceholder.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.udf.mapr.db; - -import org.apache.drill.exec.expr.DrillSimpleFunc; -import org.apache.drill.exec.expr.annotations.FunctionTemplate; -import org.apache.drill.exec.expr.annotations.Output; -import org.apache.drill.exec.expr.annotations.Param; -import org.apache.drill.exec.expr.holders.BigIntHolder; -import org.apache.drill.exec.expr.holders.BitHolder; -import org.apache.drill.exec.expr.holders.VarCharHolder; - -/** - * This is a placeholder for the notMatches() function. - * - * At this time, this function can only be used in predicates. The placeholder - * is here to prevent calcite from complaining; the function will get pushed down - * by the storage plug-in into DB. That process will go through JsonConditionBuilder.java, - * which will replace this function with the real OJAI equivalent to be pushed down. - * Therefore, there's no implementation here. - */ -@FunctionTemplate(name = "ojai_notmatches", - scope = FunctionTemplate.FunctionScope.SIMPLE, - nulls = FunctionTemplate.NullHandling.INTERNAL) -public class NotMatchesPlaceholder implements DrillSimpleFunc { - - @Param BigIntHolder /*FieldReader*/ field; - @Param(constant = true) VarCharHolder pattern; - - @Output BitHolder output; - - public void setup() { - } - - public void eval() { - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/NotTypeOfPlaceholder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/NotTypeOfPlaceholder.java deleted file mode 100644 index 78abcc04265..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/NotTypeOfPlaceholder.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.udf.mapr.db; - -import org.apache.drill.exec.expr.DrillSimpleFunc; -import org.apache.drill.exec.expr.annotations.FunctionTemplate; -import org.apache.drill.exec.expr.annotations.Output; -import org.apache.drill.exec.expr.annotations.Param; -import org.apache.drill.exec.expr.holders.BigIntHolder; -import org.apache.drill.exec.expr.holders.BitHolder; -import org.apache.drill.exec.expr.holders.IntHolder; - -/** - * This is a placeholder for the nottypeof() function. - * - * At this time, this function can only be used in predicates. The placeholder - * is here to prevent calcite from complaining; the function will get pushed down - * by the storage plug-in into DB. That process will go through JsonConditionBuilder.java, - * which will replace this function with the real OJAI equivalent to be pushed down. - * Therefore, there's no implementation here. - */ -@FunctionTemplate(name = "ojai_nottypeof", - scope = FunctionTemplate.FunctionScope.SIMPLE, - nulls = FunctionTemplate.NullHandling.INTERNAL) -public class NotTypeOfPlaceholder implements DrillSimpleFunc { - - @Param BigIntHolder /*FieldReader*/ field; - @Param(constant = true) IntHolder typeCode; - - @Output BitHolder output; - - public void setup() { - } - - public void eval() { - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/SizeOfPlaceholder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/SizeOfPlaceholder.java deleted file mode 100644 index 8fcfee87403..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/SizeOfPlaceholder.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.udf.mapr.db; - -import org.apache.drill.exec.expr.DrillSimpleFunc; -import org.apache.drill.exec.expr.annotations.FunctionTemplate; -import org.apache.drill.exec.expr.annotations.Output; -import org.apache.drill.exec.expr.annotations.Param; -import org.apache.drill.exec.expr.holders.BigIntHolder; -import org.apache.drill.exec.expr.holders.BitHolder; -import org.apache.drill.exec.expr.holders.VarCharHolder; - -/** - * This is a placeholder for the sizeof() function. - * - * At this time, this function can only be used in predicates. The placeholder - * is here to prevent calcite from complaining; the function will get pushed down - * by the storage plug-in into DB. That process will go through JsonConditionBuilder.java, - * which will replace this function with the real OJAI equivalent to be pushed down. - * Therefore, there's no implementation here. - */ -@FunctionTemplate(name = "ojai_sizeof", - scope = FunctionTemplate.FunctionScope.SIMPLE, - nulls = FunctionTemplate.NullHandling.INTERNAL) -public class SizeOfPlaceholder implements DrillSimpleFunc { - - @Param BigIntHolder /*FieldReader*/ field; - @Param(constant = true) VarCharHolder relOp; - @Param(constant = true) BigIntHolder size; - - @Output BitHolder output; - - public void setup() { - } - - public void eval() { - } - -} diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/TypeOfPlaceholder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/TypeOfPlaceholder.java deleted file mode 100644 index 585404fdb9d..00000000000 --- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/udf/mapr/db/TypeOfPlaceholder.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.udf.mapr.db; - -import org.apache.drill.exec.expr.DrillSimpleFunc; -import org.apache.drill.exec.expr.annotations.FunctionTemplate; -import org.apache.drill.exec.expr.annotations.Output; -import org.apache.drill.exec.expr.annotations.Param; -import org.apache.drill.exec.expr.holders.BigIntHolder; -import org.apache.drill.exec.expr.holders.BitHolder; -import org.apache.drill.exec.expr.holders.IntHolder; - -/** - * This is a placeholder for the typeof() function. - * - * At this time, this function can only be used in predicates. The placeholder - * is here to prevent calcite from complaining; the function will get pushed down - * by the storage plug-in into DB. That process will go through JsonConditionBuilder.java, - * which will replace this function with the real OJAI equivalent to be pushed down. - * Therefore, there's no implementation here. - */ -@FunctionTemplate(name = "ojai_typeof", - scope = FunctionTemplate.FunctionScope.SIMPLE, - nulls = FunctionTemplate.NullHandling.INTERNAL) -public class TypeOfPlaceholder implements DrillSimpleFunc { - - @Param BigIntHolder /*FieldReader*/ field; - @Param(constant = true) IntHolder typeCode; - - @Output BitHolder output; - - public void setup() { - } - - public void eval() { - } - -} diff --git a/contrib/format-maprdb/src/main/resources/bootstrap-format-plugins.json b/contrib/format-maprdb/src/main/resources/bootstrap-format-plugins.json deleted file mode 100644 index a126709bd30..00000000000 --- a/contrib/format-maprdb/src/main/resources/bootstrap-format-plugins.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "storage":{ - "dfs": { - "type": "file", - "formats": { - "maprdb": { - "type": "maprdb" - } - } - }, - "s3": { - "type": "file", - "formats": { - "maprdb": { - "type": "maprdb" - } - } - } - } -} diff --git a/contrib/format-maprdb/src/main/resources/drill-module.conf b/contrib/format-maprdb/src/main/resources/drill-module.conf deleted file mode 100644 index 9dc958804a6..00000000000 --- a/contrib/format-maprdb/src/main/resources/drill-module.conf +++ /dev/null @@ -1,41 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This file tells Drill to consider this module when class path scanning. -# This file can also include any supplementary configuration information. -# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. - -format-maprdb: { - json: { - mediaType: SSD, - scanSizeMB: 128, - restrictedScanSizeMB: 4096, - useNumRegionsForDistribution: false, - tableCache: { - enabled: true, - size: 1000, - expireTimeInMinutes: 60 - }, - pluginCost: { - blockSize: 8192 - } - } -} - -# this picks up the UDFs -drill.classpath.scanning.packages += "org.apache.drill.exec.udf.mapr.db" diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java deleted file mode 100644 index aea03c9d949..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/MaprDBTestsSuite.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests; - -import java.io.InputStream; -import java.lang.management.ManagementFactory; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.store.StoragePluginRegistry; -import org.apache.drill.exec.store.dfs.FileSystemConfig; -import org.apache.drill.hbase.HBaseTestsSuite; -import org.apache.drill.test.BaseTest; -import org.apache.hadoop.conf.Configuration; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.runner.RunWith; -import org.junit.runners.Suite; -import org.junit.runners.Suite.SuiteClasses; - -import com.mapr.db.Admin; -import com.mapr.db.MapRDB; -import com.mapr.drill.maprdb.tests.binary.TestMapRDBFilterPushDown; -import com.mapr.drill.maprdb.tests.binary.TestMapRDBSimple; -import com.mapr.drill.maprdb.tests.json.TestScanRanges; -import com.mapr.drill.maprdb.tests.json.TestSimpleJson; - -@RunWith(Suite.class) -@SuiteClasses({ - TestMapRDBSimple.class, - TestMapRDBFilterPushDown.class, - TestSimpleJson.class, - TestScanRanges.class -}) -public class MaprDBTestsSuite extends BaseTest { - public static final int INDEX_FLUSH_TIMEOUT = 60000; - - private static final boolean IS_DEBUG = ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0; - - private static volatile AtomicInteger initCount = new AtomicInteger(0); - private static volatile Configuration conf; - - private static Admin admin; - - @BeforeClass - public static void setupTests() throws Exception { - if (initCount.get() == 0) { - synchronized (MaprDBTestsSuite.class) { - if (initCount.get() == 0) { - HBaseTestsSuite.configure(false /*manageHBaseCluster*/, true /*createTables*/); - HBaseTestsSuite.initCluster(); - - // Sleep to allow table data to be flushed to tables. - // Without this, the row count stats to return 0, - // causing the planner to reject optimized plans. - Thread.sleep(5000); - - admin = MapRDB.newAdmin(); - conf = HBaseTestsSuite.getConf(); - initCount.incrementAndGet(); // must increment while inside the synchronized block - return; - } - } - } - initCount.incrementAndGet(); - return; - } - - @AfterClass - public static void cleanupTests() throws Exception { - synchronized (MaprDBTestsSuite.class) { - if (initCount.decrementAndGet() == 0) { - HBaseTestsSuite.tearDownCluster(); - admin.close(); - } - } - } - - private static volatile boolean pluginsUpdated; - - public static Configuration createPluginAndGetConf(DrillbitContext ctx) throws Exception { - if (!pluginsUpdated) { - synchronized (MaprDBTestsSuite.class) { - if (!pluginsUpdated) { - StoragePluginRegistry pluginRegistry = ctx.getStorage(); - - String pluginConfStr = "{" + - " \"type\": \"file\"," + - " \"enabled\": true," + - " \"connection\": \"maprfs:///\"," + - " \"workspaces\": {" + - " \"default\": {" + - " \"location\": \"/tmp\"," + - " \"writable\": false," + - " \"defaultInputFormat\": \"maprdb\"" + - " }," + - " \"tmp\": {" + - " \"location\": \"/tmp\"," + - " \"writable\": true," + - " \"defaultInputFormat\": \"parquet\"" + - " }," + - " \"root\": {" + - " \"location\": \"/\"," + - " \"writable\": false," + - " \"defaultInputFormat\": \"maprdb\"" + - " }" + - " }," + - " \"formats\": {" + - " \"maprdb\": {" + - " \"type\": \"maprdb\"," + - " \"allTextMode\": false," + - " \"readAllNumbersAsDouble\": false," + - " \"enablePushdown\": true" + - " }," + - " \"parquet\": {" + - " \"type\": \"parquet\"" + - " }," + - " \"streams\": {" + - " \"type\": \"streams\"" + - " }" + - " }" + - "}"; - - FileSystemConfig pluginConfig = ctx.getLpPersistence().getMapper().readValue(pluginConfStr, FileSystemConfig.class); - // create the plugin with "hbase" name so that we can run HBase unit tests against them - pluginRegistry.put("hbase", pluginConfig); - } - } - } - return conf; - } - - public static boolean isDebug() { - return IS_DEBUG; - } - - public static Admin getAdmin() { - return admin; - } - - public static InputStream getJsonStream(String resourceName) { - return MaprDBTestsSuite.class.getResourceAsStream(resourceName); - } - -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBCFAsJSONString.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBCFAsJSONString.java deleted file mode 100644 index 4cd9f51eece..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBCFAsJSONString.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests.binary; - -import org.apache.drill.hbase.TestHBaseCFAsJSONString; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.experimental.categories.Category; - -import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; -import com.mapr.tests.annotations.ClusterTest; - -/** - * This class does not define any test method but includes all test methods - * defined in the parent class, all of which are tested against MapRDB instead - * of HBase. - */ -@Category(ClusterTest.class) -public class TestMapRDBCFAsJSONString extends TestHBaseCFAsJSONString { - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - MaprDBTestsSuite.setupTests(); - conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - MaprDBTestsSuite.cleanupTests(); - } -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java deleted file mode 100644 index c6d76a7d4a3..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBFilterPushDown.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests.binary; - -import org.apache.drill.hbase.TestHBaseFilterPushDown; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.experimental.categories.Category; - -import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; -import com.mapr.tests.annotations.ClusterTest; - -/** - * This class does not define any test method but includes all test methods - * defined in the parent class, all of which are tested against MapRDB instead - * of HBase. - */ -@Category(ClusterTest.class) -public class TestMapRDBFilterPushDown extends TestHBaseFilterPushDown { - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - MaprDBTestsSuite.setupTests(); - conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - MaprDBTestsSuite.cleanupTests(); - } - -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBProjectPushDown.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBProjectPushDown.java deleted file mode 100644 index b5d3909d137..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBProjectPushDown.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests.binary; - -import org.apache.drill.hbase.TestHBaseProjectPushDown; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.experimental.categories.Category; - -import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; -import com.mapr.tests.annotations.ClusterTest; - -/** - * This class does not define any test method but includes all test methods - * defined in the parent class, all of which are tested against MapRDB instead - * of HBase. - */ -@Category(ClusterTest.class) -public class TestMapRDBProjectPushDown extends TestHBaseProjectPushDown { - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - MaprDBTestsSuite.setupTests(); - conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - MaprDBTestsSuite.cleanupTests(); - } -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBQueries.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBQueries.java deleted file mode 100644 index 0dc30e722a9..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBQueries.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests.binary; - -import org.apache.drill.hbase.TestHBaseQueries; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.experimental.categories.Category; - -import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; -import com.mapr.tests.annotations.ClusterTest; - -/** - * This class does not define any test method but includes all test methods - * defined in the parent class, all of which are tested against MapRDB instead - * of HBase. - */ -@Category(ClusterTest.class) -public class TestMapRDBQueries extends TestHBaseQueries { - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - MaprDBTestsSuite.setupTests(); - conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - MaprDBTestsSuite.cleanupTests(); - } -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java deleted file mode 100644 index d4fad8e5784..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBSimple.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests.binary; - -import org.apache.drill.hbase.BaseHBaseTest; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; -import com.mapr.tests.annotations.ClusterTest; - -@Category(ClusterTest.class) -public class TestMapRDBSimple extends BaseHBaseTest { - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - MaprDBTestsSuite.setupTests(); - conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - MaprDBTestsSuite.cleanupTests(); - } - - @Test - public void testMe() throws Exception { - setColumnWidths(new int[] {8, 38, 38}); - final String sql = "SELECT\n" - + " *\n" - + "FROM\n" - + " hbase.`[TABLE_NAME]` tableName"; - runHBaseSQLVerifyCount(sql, 8); - } - -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/IndexHintPlanTest.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/IndexHintPlanTest.java deleted file mode 100644 index f7f0da258b4..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/IndexHintPlanTest.java +++ /dev/null @@ -1,165 +0,0 @@ -package com.mapr.drill.maprdb.tests.index; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import com.mapr.tests.annotations.ClusterTest; -import org.apache.drill.PlanTestBase; -import org.junit.experimental.categories.Category; -import org.junit.runners.MethodSorters; -import org.junit.FixMethodOrder; -import org.junit.Test; - -@FixMethodOrder(MethodSorters.NAME_ASCENDING) -@Category(ClusterTest.class) -public class IndexHintPlanTest extends IndexPlanTest { - - private static final String defaultHavingIndexPlan = "alter session reset `planner.enable_index_planning`"; - - @Test - // A simple testcase with index hint on a table which has only one index for a column t.id.ssn; - // This should pick i_ssn index for the query - public void testSimpleIndexHint() throws Exception { - String hintquery = "SELECT t.id.ssn as ssn FROM table(hbase.`index_test_primary`(type => 'maprdb', index => 'i_ssn')) as t " + - " where t.id.ssn = '100007423'"; - - String query = "SELECT t.id.ssn as ssn FROM hbase.`index_test_primary` as t where t.id.ssn = '100007423'"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(hintquery, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_ssn"}, - new String[]{"RowKeyJoin"} - ); - - // default plan picked by optimizer. - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_ssn"}, - new String[]{"RowKeyJoin"} - ); - testBuilder() - .sqlQuery(hintquery) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .go(); - - } - - - @Test - // A testcase where there are multiple index to pick from but only picks the index provided as hint. - // A valid index is provided as hint and it is useful during the index selection process, hence it will be selected. - public void testHintCaseWithMultipleIndexes_1() throws Exception { - - String hintquery = "SELECT t.`address`.`state` AS `state` FROM table(hbase.`index_test_primary`(type => 'maprdb', index => 'i_state_city')) as t " + - " where t.address.state = 'pc'"; - - String query = "SELECT t.`address`.`state` AS `state` FROM hbase.`index_test_primary` as t where t.address.state = 'pc'"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(hintquery, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_state_city"}, - new String[]{"RowKeyJoin"} - ); - - // default plan picked by optimizer - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=(i_state_city|i_state_age_phone)"}, - new String[]{"RowKeyJoin"} - ); - - } - - @Test - // A testcase where there are multiple index to pick from but only picks the index provided as hint. - // A valid index is provided as hint and it is useful during the index selection process, hence it will be selected. - // Difference between this testcase and the one before this is that index name is switched. This shows that index hint makes sure to select only one - // valid index specified as hint. - public void testHintCaseWithMultipleIndexes_2() throws Exception { - - String hintquery = "SELECT t.`address`.`state` AS `state` FROM table(hbase.`index_test_primary`(type => 'maprdb', index => 'i_state_age_phone')) as t " + - " where t.address.state = 'pc'"; - - String query = "SELECT t.`address`.`state` AS `state` FROM hbase.`index_test_primary` as t where t.address.state = 'pc'"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(hintquery, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_state_age_phone"}, - new String[]{"RowKeyJoin"} - ); - - // default plan picked by query optimizer. - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=(i_state_city|i_state_age_phone)"}, - new String[]{"RowKeyJoin"} - ); - - } - - // Negative cases - - @Test - // A testcase where there are multiple index to pick from but none of them equals to the index provided as hint (index hint is wrong). - // In this index is not at all present in the table hence it falls back to the case where the index itself is not given. - // Hence here one of the i_state_city or i_state_age_lic will be selected depending upon the cost. - public void testWithMultipleIndexesButNoIndexWithHint() throws Exception { - - String hintquery = "SELECT t.`address`.`state` AS `state` FROM table(hbase.`index_test_primary`(type => 'maprdb', index => 'i_state_and_city')) as t " + - " where t.address.state = 'pc'"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(hintquery, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=(i_state_city|i_state_age_phone)"}, - new String[]{"RowKeyJoin"} - ); - - } - - @Test - // A testcase where there are multiple index to pick from but none of them equals to the index provided as hint and the hint index is valid. - // Here the index name given is valid (i.e it is present in the table) but it is not useful. - // This case falls back to full table scan. - public void testWithMultipleIndexesButNoIndexWithValidHint() throws Exception { - - String hintquery = "SELECT t.`address`.`state` AS `state` FROM table(hbase.`index_test_primary`(type => 'maprdb', index => 'i_ssn')) as t " + - " where t.address.state = 'pc'"; - - String query = "SELECT t.`address`.`state` AS `state` FROM hbase.`index_test_primary` as t where t.address.state = 'pc'"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(hintquery, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary"}, - new String[]{"RowKeyJoin", "indexName="} - ); - - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=(i_state_city|i_state_age_phone)"}, - new String[]{"RowKeyJoin"} - ); - - } - - - @Test - // Covering index should be generated for a simple query instead of a RowKeyJoin. - public void testSimpleNoRowKeyJoin() throws Exception { - String query = "SELECT `reverseid` from table(hbase.`index_test_primary`(type => 'maprdb', index => 'hash_i_reverseid')) " + - "where `reverseid` = 1234"; - - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=hash_i_reverseid"}, - new String[]{"RowKeyJoin"} - ); - - } -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/IndexPlanTest.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/IndexPlanTest.java deleted file mode 100644 index bec5d3a4860..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/IndexPlanTest.java +++ /dev/null @@ -1,1995 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests.index; - -import com.mapr.db.Admin; -import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; -import com.mapr.drill.maprdb.tests.json.BaseJsonTest; -import com.mapr.tests.annotations.ClusterTest; -import org.apache.drill.PlanTestBase; -import org.joda.time.DateTime; -import org.joda.time.format.DateTimeFormat; -import org.apache.drill.common.config.DrillConfig; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.FixMethodOrder; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runners.MethodSorters; -import java.util.Properties; - - -@FixMethodOrder(MethodSorters.NAME_ASCENDING) -@Category(ClusterTest.class) -public class IndexPlanTest extends BaseJsonTest { - - final static String PRIMARY_TABLE_NAME = "/tmp/index_test_primary"; - - final static int PRIMARY_TABLE_SIZE = 10000; - private static final String sliceTargetSmall = "alter session set `planner.slice_target` = 1"; - private static final String sliceTargetDefault = "alter session reset `planner.slice_target`"; - private static final String noIndexPlan = "alter session set `planner.enable_index_planning` = false"; - private static final String defaultHavingIndexPlan = "alter session reset `planner.enable_index_planning`"; - private static final String disableHashAgg = "alter session set `planner.enable_hashagg` = false"; - private static final String enableHashAgg = "alter session set `planner.enable_hashagg` = true"; - private static final String lowNonCoveringSelectivityThreshold = "alter session set `planner.index.noncovering_selectivity_threshold` = 0.00001"; - private static final String defaultnonCoveringSelectivityThreshold = "alter session set `planner.index.noncovering_selectivity_threshold` = 0.025"; - private static final String incrnonCoveringSelectivityThreshold = "alter session set `planner.index.noncovering_selectivity_threshold` = 0.25"; - private static final String disableFTS = "alter session set `planner.disable_full_table_scan` = true"; - private static final String enableFTS = "alter session reset `planner.disable_full_table_scan`"; - private static final String preferIntersectPlans = "alter session set `planner.index.prefer_intersect_plans` = true"; - private static final String defaultIntersectPlans = "alter session reset `planner.index.prefer_intersect_plans`"; - private static final String lowRowKeyJoinBackIOFactor - = "alter session set `planner.index.rowkeyjoin_cost_factor` = 0.01"; - private static final String defaultRowKeyJoinBackIOFactor - = "alter session reset `planner.index.rowkeyjoin_cost_factor`"; - private static final String incrRowKeyJoinConvSelThreshold = "alter session set `planner.rowkeyjoin_conversion_selectivity_threshold` = 1.0"; - private static final String defaultRowKeyConvSelThreshold = "alter session reset `planner.rowkeyjoin_conversion_selectivity_threshold`"; - private static final String forceRowKeyJoinConversionUsingHashJoin = "alter session set `planner.rowkeyjoin_conversion_using_hashjoin` = true"; - private static final String defaultRowKeyJoinConversionUsingHashJoin = "alter session reset `planner.rowkeyjoin_conversion_using_hashjoin`"; - /** - * A sample row of this 10K table: - ------------------+-----------------------------+--------+ - | 1012 | {"city":"pfrrs","state":"pc"} | {"email":"KfFzKUZwNk@gmail.com","phone":"6500005471"} | - {"ssn":"100007423"} | {"fname":"KfFzK","lname":"UZwNk"} | {"age":53.0,"income":45.0} | 1012 | - * - * This test suite generate random content to fill all the rows, since the random function always start from - * the same seed for different runs, when the row count is not changed, the data in table will always be the same, - * thus the query result could be predicted and verified. - */ - - @BeforeClass - public static void setupTableIndexes() throws Exception { - - Properties overrideProps = new Properties(); - overrideProps.setProperty("format-maprdb.json.useNumRegionsForDistribution", "true"); - updateTestCluster(1, DrillConfig.create(overrideProps)); - - MaprDBTestsSuite.setupTests(); - MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); - - test(incrnonCoveringSelectivityThreshold); - - System.out.print("setupTableIndexes begins"); - Admin admin = MaprDBTestsSuite.getAdmin(); - if (admin != null) { - if (admin.tableExists(PRIMARY_TABLE_NAME)) { - admin.deleteTable(PRIMARY_TABLE_NAME); - } - } - - LargeTableGen gen = new LargeTableGen(MaprDBTestsSuite.getAdmin()); - /** - * indexDef is an array of string, LargeTableGen.generateTableWithIndex will take it as parameter to generate indexes - * for primary table. - * indexDef[3*i] defines i-th index's indexName, NOTE: IF the name begins with "hash", it is a hash index - * indexDef[3*i+1] indexed field, - * and indexDef[3*i+2] defines i-th index's non-indexed fields - */ - final String[] indexDef = //null; - {"i_ssn", "id.ssn", "contact.phone", - "i_state_city", "address.state,address.city", "name.fname,name.lname",//mainly for composite key test - "i_age", "personal.age", "", - "i_age_desc", "personal.age:desc", "", - "i_income", "personal.income", "", - "i_lic", "driverlicense", "reverseid", - "i_state_city_dl", "address.state,address.city", "driverlicense", - "i_cast_int_ssn", "$CAST(id.ssn@INT)", "contact.phone", - "i_cast_vchar_lic", "$CAST(driverlicense@STRING)","contact.email", - "i_state_age_phone", "address.state,personal.age,contact.phone", "name.fname", - "i_cast_age_income_phone", "$CAST(personal.age@INT),$CAST(personal.income@INT),contact.phone", "name.fname", - "i_age_with_fname", "personal.age", "name.fname", - "i_rowid_cast_date_birthdate", "rowid", "$CAST(personal.birthdate@DATE)", - "hash_i_reverseid", "reverseid", "", - "hash_i_cast_timestamp_firstlogin", "$CAST(activity.irs.firstlogin@TIMESTAMP)", "id.ssn" - }; - gen.generateTableWithIndex(PRIMARY_TABLE_NAME, PRIMARY_TABLE_SIZE, indexDef); - } - - @AfterClass - public static void cleanupTableIndexes() throws Exception { - Admin admin = MaprDBTestsSuite.getAdmin(); - if (admin != null) { - if (admin.tableExists(PRIMARY_TABLE_NAME)) { - // admin.deleteTable(PRIMARY_TABLE_NAME); - } - } - test(defaultnonCoveringSelectivityThreshold); - } - - @Test - public void CTASTestTable() throws Exception { - String ctasQuery = "CREATE TABLE hbase.tmp.`backup_index_test_primary` " + - "AS SELECT * FROM hbase.`index_test_primary` as t "; - test(ctasQuery); - test("DROP TABLE IF EXISTS hbase.tmp.`backup_index_test_primary`"); - } - - @Test - public void CoveringPlanWithNonIndexedField() throws Exception { - - String query = "SELECT t.`contact`.`phone` AS `phone` FROM hbase.`index_test_primary` as t " + - " where t.id.ssn = '100007423'"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_ssn"}, - new String[]{"RowKeyJoin"} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("phone").baselineValues("6500005471") - .go(); - - } - - @Test - public void CoveringPlanWithOnlyIndexedField() throws Exception { - String query = "SELECT t.`id`.`ssn` AS `ssn` FROM hbase.`index_test_primary` as t " + - " where t.id.ssn = '100007423'"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_ssn"}, - new String[]{"RowKeyJoin"} - ); - - testBuilder() - .optionSettingQueriesForTestQuery(defaultHavingIndexPlan) - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .go(); - - } - - @Test - public void NoIndexPlanForNonIndexField() throws Exception { - - String query = "SELECT t.`id`.`ssn` AS `ssn` FROM hbase.`index_test_primary` as t " + - " where t.contact.phone = '6500005471'"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary"}, - new String[]{"RowKeyJoin", "indexName="} - ); - - testBuilder() - .sqlQuery(query) - .unOrdered() - .baselineColumns("ssn").baselineValues("100007423") - .baselineColumns("ssn").baselineValues("100007632") - .go(); - - } - - @Test - public void NonCoveringPlan() throws Exception { - - String query = "SELECT t.`name`.`fname` AS `fname` FROM hbase.`index_test_primary` as t " + - " where t.id.ssn = '100007423'"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"RowKeyJoin", ".*RestrictedJsonTableGroupScan.*tableName=.*index_test_primary,", - ".*JsonTableGroupScan.*tableName=.*index_test_primary,.*indexName=i_ssn"}, - new String[]{} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("fname").baselineValues("KfFzK") - .go(); - - } - - @Test - public void RangeConditionIndexPlan() throws Exception { - String query = "SELECT t.`name`.`lname` AS `lname` FROM hbase.`index_test_primary` as t " + - " where t.personal.age > 52 AND t.name.fname='KfFzK'"; - try { - test(defaultHavingIndexPlan + ";" + lowRowKeyJoinBackIOFactor + ";"); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"RowKeyJoin", ".*RestrictedJsonTableGroupScan.*tableName=.*index_test_primary,", - ".*JsonTableGroupScan.*tableName=.*index_test_primary,.*indexName=(i_age|i_age_with_fname)"}, - new String[]{} - ); - testBuilder() - .optionSettingQueriesForTestQuery(defaultHavingIndexPlan) - .optionSettingQueriesForTestQuery(lowRowKeyJoinBackIOFactor) - .optionSettingQueriesForBaseline(noIndexPlan) - .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(query) - .build() - .run(); - - testBuilder() - .optionSettingQueriesForTestQuery(sliceTargetSmall) - .optionSettingQueriesForBaseline(sliceTargetDefault) - .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(query) - .build() - .run(); - } finally { - test(defaultRowKeyJoinBackIOFactor); - } - } - - @Test - public void CoveringWithSimpleFieldsOnly() throws Exception { - - String query = "SELECT t._id AS `tid` FROM hbase.`index_test_primary` as t " + - " where t.driverlicense = 100007423"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"JsonTableGroupScan.*tableName=.*index_test_primary,.*indexName=i_lic"}, - new String[]{"RowKeyJoin"} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("tid").baselineValues("1012") - .go(); - - } - - @Test - public void NonCoveringWithSimpleFieldsOnly() throws Exception { - - String query = "SELECT t.rowid AS `rowid` FROM hbase.`index_test_primary` as t " + - " where t.driverlicense = 100007423"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"RowKeyJoin(.*[\n\r])+.*" + - "RestrictedJsonTableGroupScan.*tableName=.*index_test_primary(.*[\n\r])+.*" + - "JsonTableGroupScan.*tableName=.*index_test_primary,.*indexName=i_lic"}, - new String[]{} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("rowid").baselineValues("1012") - .go(); - - } - - @Test - public void NonCoveringWithExtraConditonOnPrimary() throws Exception { - - String query = "SELECT t.`name`.`fname` AS `fname` FROM hbase.`index_test_primary` as t " + - " where t.personal.age = 53 AND t.name.lname='UZwNk'"; - try { - test(defaultHavingIndexPlan + ";" + lowRowKeyJoinBackIOFactor + ";"); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"RowKeyJoin", ".*RestrictedJsonTableGroupScan", - ".*JsonTableGroupScan.*indexName=i_age",}, - new String[]{} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("fname").baselineValues("KfFzK") - .go(); - } finally { - test(defaultRowKeyJoinBackIOFactor); - } - - } - - @Test - public void Intersect2indexesPlan() throws Exception { - - String query = "SELECT t.`name`.`lname` AS `lname` FROM hbase.`index_test_primary` as t " + - " where t.personal.age = 53 AND t.personal.income=45"; - try { - test(defaultHavingIndexPlan); - test(preferIntersectPlans + ";" + disableFTS); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*HashJoin(.*[\n\r])+.*JsonTableGroupScan.*indexName=(i_age|i_income)(.*[\n\r])+.*JsonTableGroupScan.*indexName=(i_age|i_income)"}, - new String[]{} - ); - - testBuilder() - .sqlQuery(query) - .unOrdered() - .baselineColumns("lname").baselineValues("UZwNk") - .baselineColumns("lname").baselineValues("foNwtze") - .baselineColumns("lname").baselineValues("qGZVfY") - .go(); - testBuilder() - .optionSettingQueriesForTestQuery(sliceTargetSmall) - .optionSettingQueriesForBaseline(sliceTargetDefault) - .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(query) - .build() - .run(); - } finally { - test(defaultIntersectPlans + ";" + enableFTS); - } - - } - - @Test - public void CompositeIndexNonCoveringPlan() throws Exception { - - String query = "SELECT t.`id`.`ssn` AS `ssn` FROM hbase.`index_test_primary` as t " + - " where t.address.state = 'pc' AND t.address.city='pfrrs'"; - try { - test(defaultHavingIndexPlan + ";" + lowRowKeyJoinBackIOFactor + ";"); - - //either i_state_city or i_state_age_phone will be picked depends on cost model, both is fine for testing composite index nonCovering plan - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*JsonTableGroupScan.*indexName=i_state_"}, - new String[]{} - ); - - testBuilder() - .sqlQuery(query) - .unOrdered() - .baselineColumns("ssn").baselineValues("100007423") - .baselineColumns("ssn").baselineValues("100008861") - .go(); - - testBuilder() - .optionSettingQueriesForTestQuery(sliceTargetSmall) - .optionSettingQueriesForBaseline(sliceTargetDefault) - .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(query) - .build() - .run(); - } finally { - test(defaultRowKeyJoinBackIOFactor); - } - - } - - @Test//filter cover indexed, included and not in index at all filter - public void CompositeIndexNonCoveringFilterWithAllFieldsPlan() throws Exception { - - String query = "SELECT t.`id`.`ssn` AS `ssn` FROM hbase.`index_test_primary` as t " + - " where t.address.state = 'pc' AND t.address.city='pfrrs' AND t.driverlicense IN (100007423, 100007424)"; - test(defaultHavingIndexPlan+";"+lowRowKeyJoinBackIOFactor+";"); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan.*condition=.*state.*city.*driverlicense.*or.*driverlicense.*(.*[\n\r])+.*JsonTableGroupScan.*indexName="}, - new String[]{} - ); - - testBuilder() - .sqlQuery(query) - .unOrdered() - .baselineColumns("ssn").baselineValues("100007423") - .go(); - - testBuilder() - .optionSettingQueriesForTestQuery(sliceTargetSmall) - .optionSettingQueriesForBaseline(sliceTargetDefault) - .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(query) - .build() - .run(); - - } - - @Test - public void CompositeIndexCoveringPlan() throws Exception { - - String query = "SELECT t.`address`.`city` AS `city` FROM hbase.`index_test_primary` as t " + - " where t.address.state = 'pc' AND t.address.city='pfrrs'"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*indexName=i_state_city"}, - new String[]{"RowKeyJoin", "Filter"} - ); - - testBuilder() - .sqlQuery(query) - .unOrdered() - .baselineColumns("city").baselineValues("pfrrs") - .baselineColumns("city").baselineValues("pfrrs") - .go(); - - testBuilder() - .optionSettingQueriesForTestQuery(sliceTargetSmall) - .optionSettingQueriesForBaseline(sliceTargetDefault) - .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(query) - .build() - .run(); - - } - - @Test - public void TestNonCoveringRangePartition_1() throws Exception { - - String query = "SELECT t.`name`.`lname` AS `lname` FROM hbase.`index_test_primary` as t " + - " where t.personal.age = 53"; - String[] expectedPlan = new String[] {"RowKeyJoin(.*[\n\r])+.*" + - "RestrictedJsonTableGroupScan.*tableName=.*index_test_primary(.*[\n\r])+.*" + - "RangePartitionExchange(.*[\n\r])+.*" + - "JsonTableGroupScan.*tableName=.*index_test_primary,.*indexName=(i_age|i_age_with_fname)"}; - test(defaultHavingIndexPlan+";"+sliceTargetSmall+";"); - PlanTestBase.testPlanMatchingPatterns(query, - expectedPlan, new String[]{}); - - try { - testBuilder() - .optionSettingQueriesForTestQuery(defaultHavingIndexPlan) - .optionSettingQueriesForBaseline(noIndexPlan) - .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(query) - .build() - .run(); - } finally { - test(defaultHavingIndexPlan); - test(sliceTargetDefault); - } - - } - - @Test - public void TestCastVarCharCoveringPlan() throws Exception { - // length 255 is to exact match the casted indexed field's length - String query = "SELECT t._id as tid, cast(t.driverlicense as varchar(255)) as driverlicense FROM hbase.`index_test_primary` as t " + - " where cast(t.driverlicense as varchar(255))='100007423'"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_cast_vchar_lic"}, - new String[]{"RowKeyJoin"} - ); - - testBuilder() - .optionSettingQueriesForTestQuery(defaultHavingIndexPlan) - .sqlQuery(query) - .ordered() - .baselineColumns("tid", "driverlicense").baselineValues("1012", "100007423") - .go(); - - } - - @Test - public void TestCastINTCoveringPlan() throws Exception { - String query = "SELECT t._id as tid, CAST(t.id.ssn as INT) as ssn, t.contact.phone AS `phone` FROM hbase.`index_test_primary` as t " + - " where CAST(t.id.ssn as INT) = 100007423"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_cast_int_ssn"}, - new String[]{"RowKeyJoin"} - ); - - testBuilder() - .optionSettingQueriesForTestQuery(defaultHavingIndexPlan) - .sqlQuery(query) - .ordered() - .baselineColumns("tid", "ssn", "phone").baselineValues("1012", 100007423, "6500005471") - .go(); - - } - - @Test - public void TestCastNonCoveringPlan() throws Exception { - String query = "SELECT t.id.ssn AS `ssn` FROM hbase.`index_test_primary` as t " + - " where CAST(t.id.ssn as INT) = 100007423"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*JsonTableGroupScan.*indexName=i_cast_int_ssn"}, - new String[]{} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .go(); - - } - - @Test - public void TestCastVarchar_ConvertToRangePlan() throws Exception { - String query = "SELECT t.id.ssn AS `ssn` FROM hbase.`index_test_primary` as t " + - " where CAST(driverlicense as VARCHAR(10)) = '100007423'"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*JsonTableGroupScan.*MATCHES \"\\^.*100007423.*E.*\\$\".*indexName=i_cast_vchar_lic"}, - new String[]{} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .go(); - - } - - @Test // cast expression in filter is not indexed, but the same field casted to different type was indexed (CAST id.ssn as INT) - public void TestCastNoIndexPlan() throws Exception { - String query = "select t.id.ssn from hbase.`index_test_primary` t where cast(t.id.ssn as varchar(10)) = '100007423'"; - - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{}, - new String[]{"indexName"} - ); - - } - - @Test - public void TestLongerCastVarCharNoIndex() throws Exception { - // length 256 is to exact match the casted indexed field's length - String query = "SELECT t._id as tid, cast(t.driverlicense as varchar(500)) as driverlicense FROM hbase.`index_test_primary` as t " + - " where cast(t.driverlicense as varchar(500))='100007423'"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {}, - new String[]{"RowKeyJoin", "indexName="} - ); - - } - - @Test - public void TestCoveringPlanSortRemoved() throws Exception { - String query = "SELECT t.`contact`.`phone` as phone FROM hbase.`index_test_primary` as t " + - " where t.id.ssn <'100000003' order by t.id.ssn"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_ssn"}, - new String[]{"Sort"} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("phone").baselineValues("6500008069") - .baselineColumns("phone").baselineValues("6500001411") - .baselineColumns("phone").baselineValues("6500001595") - .go(); - } - - @Test - public void TestCoveringPlanSortNotRemoved() throws Exception { - String query = "SELECT t.`contact`.`phone` as phone FROM hbase.`index_test_primary` as t " + - " where t.id.ssn <'100000003' order by t.contact.phone"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"Sort", ".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_ssn"}, - new String[]{"RowkeyJoin"} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("phone").baselineValues("6500001411") - .baselineColumns("phone").baselineValues("6500001595") - .baselineColumns("phone").baselineValues("6500008069") - .go(); - } - - @Test - public void TestCoveringPlanSortRemovedWithSimpleFields() throws Exception { - String query = "SELECT t.driverlicense as l FROM hbase.`index_test_primary` as t " + - " where t.driverlicense < 100000003 order by t.driverlicense"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_lic"}, - new String[]{"Sort"} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("l").baselineValues(100000000l) - .baselineColumns("l").baselineValues(100000001l) - .baselineColumns("l").baselineValues(100000002l) - .go(); - } - - @Test - public void TestNonCoveringPlanSortRemoved() throws Exception { - String query = "SELECT t.contact.phone as phone FROM hbase.`index_test_primary` as t " + - " where t.driverlicense < 100000003 order by t.driverlicense"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*JsonTableGroupScan.*indexName=i_lic"}, - new String[]{"Sort"} - ); - String query2 = "SELECT t.name.fname as fname FROM hbase.`index_test_primary` as t " + - " where t.id.ssn < '100000003' order by t.id.ssn"; - PlanTestBase.testPlanMatchingPatterns(query2, - new String[] {"RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*JsonTableGroupScan.*indexName="}, - new String[]{"Sort"} - ); - - // simple field, driverlicense - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("phone").baselineValues("6500008069") - .baselineColumns("phone").baselineValues("6500001411") - .baselineColumns("phone").baselineValues("6500001595") - .go(); - - // query on field of item expression(having capProject), non-simple field t.id.ssn - testBuilder() - .sqlQuery(query2) - .ordered() - .baselineColumns("fname").baselineValues("VcFahj") - .baselineColumns("fname").baselineValues("WbKVK") - .baselineColumns("fname").baselineValues("vSAEsyFN") - .go(); - - test(sliceTargetSmall); - try { - PlanTestBase.testPlanMatchingPatterns(query2, - new String[]{"SingleMergeExchange(.*[\n\r])+.*" - + "RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*JsonTableGroupScan.*indexName=i_ssn"}, - new String[]{"Sort"} - ); - } finally { - test(sliceTargetDefault); - } - } - - // test cases are from TestNonCoveringPlanSortRemoved. Sort was removed when force_sort_noncovering was default(false) - @Test - public void TestNonCoveringPlanWithNoRemoveSortOption() throws Exception { - try { - test("alter session set `planner.index.force_sort_noncovering`=true"); - test(defaultHavingIndexPlan); - - String query = "SELECT t.contact.phone as phone FROM hbase.`index_test_primary` as t " + - " where t.driverlicense < 100000003 order by t.driverlicense"; - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"Sort", "RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*JsonTableGroupScan.*indexName=i_lic"}, - new String[]{} - ); - - String query2 = "SELECT t.name.fname as fname FROM hbase.`index_test_primary` as t " + - " where t.id.ssn < '100000003' order by t.id.ssn"; - PlanTestBase.testPlanMatchingPatterns(query2, - new String[]{"Sort", "RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*JsonTableGroupScan.*indexName="}, - new String[]{} - ); - - // simple field, driverlicense - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("phone").baselineValues("6500008069") - .baselineColumns("phone").baselineValues("6500001411") - .baselineColumns("phone").baselineValues("6500001595") - .go(); - - // query on field of item expression(having capProject), non-simple field t.id.ssn - testBuilder() - .sqlQuery(query2) - .ordered() - .baselineColumns("fname").baselineValues("VcFahj") - .baselineColumns("fname").baselineValues("WbKVK") - .baselineColumns("fname").baselineValues("vSAEsyFN") - .go(); - - test(sliceTargetSmall); - try { - PlanTestBase.testPlanMatchingPatterns(query2, - new String[]{"Sort", "SingleMergeExchange(.*[\n\r])+.*" - + "RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*JsonTableGroupScan.*indexName=i_ssn"}, - new String[]{} - ); - } finally { - test(sliceTargetDefault); - } - } - finally { - test("alter session reset `planner.index.force_sort_noncovering`"); - } - } - - @Test // 2 table join, each table has local predicate on top-level column - public void TestCoveringPlanJoin_1() throws Exception { - String query = "SELECT count(*) as cnt FROM hbase.`index_test_primary` as t1 " + - " inner join hbase.`index_test_primary` as t2 on t1.driverlicense = t2.driverlicense " + - " where t1.driverlicense < 100000003 and t2.driverlicense < 100000003"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=", - ".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName="}, - new String[]{} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("cnt").baselineValues(3L) - .go(); - } - - @Test // 2 table join, each table has local predicate on nested column - public void TestCoveringPlanJoin_2() throws Exception { - String query = "SELECT count(*) as cnt FROM hbase.`index_test_primary` as t1 " + - " inner join hbase.`index_test_primary` as t2 on t1.contact.phone = t2.contact.phone " + - " where t1.id.ssn < '100000003' and t2.id.ssn < '100000003' "; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=", - ".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName="}, - new String[]{} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("cnt").baselineValues(3L) - .go(); - } - - @Test // leading prefix of index has Equality conditions and ORDER BY last column; Sort SHOULD be dropped - public void TestCoveringPlanSortPrefix_1() throws Exception { - String query = "SELECT t.contact.phone FROM hbase.`index_test_primary` as t " + - " where t.address.state = 'wo' and t.personal.age = 35 and t.contact.phone < '6500003000' order by t.contact.phone"; - test(defaultHavingIndexPlan); - - // we should glue to index i_state_age_phone to make sure we are testing the targeted prefix construction code path - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_state_age_phone"}, - new String[]{"Sort"} - ); - - // compare the results of index plan with the no-index plan - testBuilder() - .optionSettingQueriesForTestQuery(defaultHavingIndexPlan) - .optionSettingQueriesForBaseline(noIndexPlan) - .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(query) - .build() - .run(); - } - - @Test // leading prefix of index has Non-Equality conditions and ORDER BY last column; Sort SHOULD NOT be dropped - public void TestCoveringPlanSortPrefix_2() throws Exception { - String query = "SELECT t.contact.phone FROM hbase.`index_test_primary` as t " + - " where t.address.state = 'wo' and t.personal.age < 35 and t.contact.phone < '6500003000' order by t.contact.phone"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"Sort", ".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_state_age_phone"}, - new String[]{} - ); - - // compare the results of index plan with the no-index plan - testBuilder() - .optionSettingQueriesForTestQuery(defaultHavingIndexPlan) - .optionSettingQueriesForBaseline(noIndexPlan) - .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(query) - .build() - .run(); - } - - @Test // ORDER BY last two columns not in the indexed order; Sort SHOULD NOT be dropped - public void TestCoveringPlanSortPrefix_3() throws Exception { - String query = "SELECT CAST(t.personal.age as VARCHAR) as age, t.contact.phone FROM hbase.`index_test_primary` as t " + - " where t.address.state = 'wo' and t.personal.age < 35 and t.contact.phone < '6500003000' order by t.contact.phone, t.personal.age"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"Sort", ".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_state_age_phone"}, - new String[]{} - ); - - // compare the results of index plan with the no-index plan - testBuilder() - .optionSettingQueriesForTestQuery(defaultHavingIndexPlan) - .optionSettingQueriesForBaseline(noIndexPlan) - .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(query) - .build() - .run(); - } - - @Test // last two index fields in non-Equality conditions, ORDER BY last two fields; Sort SHOULD be dropped - public void TestCoveringPlanSortPrefix_4() throws Exception { - String query = "SELECT t._id as tid, t.contact.phone, CAST(t.personal.age as VARCHAR) as age FROM hbase.`index_test_primary` as t " + - " where t.address.state = 'wo' and t.personal.age < 35 and t.contact.phone < '6500003000' order by t.personal.age, t.contact.phone"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_state_age_phone"}, - new String[]{"Sort"} - ); - - // compare the results of index plan with the no-index plan - testBuilder() - .optionSettingQueriesForTestQuery(defaultHavingIndexPlan) - .optionSettingQueriesForBaseline(noIndexPlan) - .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(query) - .build() - .run(); - } - - @Test // index field in two or more equality conditions, it is not leading prefix, Sort SHOULD NOT be dropped - public void TestCoveringPlanSortPrefix_5() throws Exception { - String query = "SELECT t._id as tid, t.contact.phone, CAST(t.personal.age as VARCHAR) as age FROM hbase.`index_test_primary` as t " + - " where t.address.state = 'wo' and t.personal.age IN (31, 32, 33, 34) and t.contact.phone < '6500003000' order by t.contact.phone"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"Sort", ".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_state_age_phone"}, - new String[]{} - ); - - // compare the results of index plan with the no-index plan - testBuilder() - .optionSettingQueriesForTestQuery(defaultHavingIndexPlan) - .optionSettingQueriesForBaseline(noIndexPlan) - .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(query) - .build() - .run(); - } - - @Test // last two index fields in non-Equality conditions, ORDER BY last two fields NULLS FIRST; Sort SHOULD NOT be dropped - public void TestCoveringPlanSortPrefix_6() throws Exception { - String query = "SELECT t._id as tid, t.contact.phone, CAST(t.personal.age as VARCHAR) as age FROM hbase.`index_test_primary` as t " + - " where t.address.state = 'wo' and t.personal.age < 35 and t.contact.phone < '6500003000' order by t.personal.age, t.contact.phone NULLS FIRST"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"Sort", ".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_state_age_phone"}, - new String[]{} - ); - - // compare the results of index plan with the no-index plan - testBuilder() - .optionSettingQueriesForTestQuery(defaultHavingIndexPlan) - .optionSettingQueriesForBaseline(noIndexPlan) - .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(query) - .build() - .run(); - } - - @Test // last two index fields in non-Equality conditions, ORDER BY last two fields NULLS LAST; Sort SHOULD be dropped - public void TestCoveringPlanSortPrefix_7() throws Exception { - String query = "SELECT t._id as tid, t.contact.phone, CAST(t.personal.age as VARCHAR) as age FROM hbase.`index_test_primary` as t " + - " where t.address.state = 'wo' and t.personal.age < 35 and t.contact.phone < '6500003000' order by t.personal.age, t.contact.phone NULLS LAST"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_state_age_phone"}, - new String[]{"Sort"} - ); - - // compare the results of index plan with the no-index plan - testBuilder() - .optionSettingQueriesForTestQuery(defaultHavingIndexPlan) - .optionSettingQueriesForBaseline(noIndexPlan) - .unOrdered() - .sqlQuery(query) - .sqlBaselineQuery(query) - .build() - .run(); - } - - @Test - public void orderByCastCoveringPlan() throws Exception { - String query = "SELECT t.contact.phone as phone FROM hbase.`index_test_primary` as t " + - " where CAST(t.id.ssn as INT) < 100000003 order by CAST(t.id.ssn as INT)"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName="}, - new String[]{"Sort"} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("phone").baselineValues("6500008069") - .baselineColumns("phone").baselineValues("6500001411") - .baselineColumns("phone").baselineValues("6500001595") - .go(); - } - - @Test // non-covering plan. sort by the only indexed field, sort SHOULD be removed - public void orderByNonCoveringPlan() throws Exception { - String query = "SELECT t.name.lname as lname FROM hbase.`index_test_primary` as t " + - " where t.id.ssn < '100000003' order by t.id.ssn"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*JsonTableGroupScan.*indexName="}, - new String[]{"Sort"} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("lname").baselineValues("iuMG") - .baselineColumns("lname").baselineValues("KpFq") - .baselineColumns("lname").baselineValues("bkkAvz") - .go(); - } - - @Test // non-covering plan. order by cast indexed field, sort SHOULD be removed - public void orderByCastNonCoveringPlan() throws Exception { - String query = "SELECT t.name.lname as lname FROM hbase.`index_test_primary` as t " + - " where CAST(t.id.ssn as INT) < 100000003 order by CAST(t.id.ssn as INT)"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*JsonTableGroupScan.*indexName="}, - new String[]{"Sort"} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("lname").baselineValues("iuMG") - .baselineColumns("lname").baselineValues("KpFq") - .baselineColumns("lname").baselineValues("bkkAvz") - .go(); - } - - - @Ignore // in statsCache, condition state+city has rowcount 1250, but state only has 1000. so it is picking i_state_age_phone - @Test // non-covering, order by non leading field, and leading fields are not in equality condition, Sort SHOULD NOT be removed - public void NonCoveringPlan_SortPrefix_1() throws Exception { - - String query = "SELECT t.`id`.`ssn` AS `ssn` FROM hbase.`index_test_primary` as t " + - " where t.address.state > 'pc' AND t.address.city>'pfrrr' AND t.address.city<'pfrrt' order by t.adddress.city"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"Sort", - "RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*JsonTableGroupScan.*indexName=i_state_city"}, - new String[]{} - ); - - } - - @Test // non-covering, order by non leading field, and leading fields are in equality condition, Sort SHOULD be removed - public void NonCoveringPlan_SortPrefix_2() throws Exception { - - String query = "SELECT t.`id`.`ssn` AS `ssn` FROM hbase.`index_test_primary` as t " + - " where t.address.state = 'pc' AND t.address.city>'pfrrr' AND t.address.city<'pfrrt' order by t.address.city"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] { - "RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*JsonTableGroupScan.*indexName=i_state_city"}, - new String[]{"Sort"} - ); - - } - - @Ignore ("Should be modified to get an index plan; not very useful since most covering plan filters get pushed") - @Test // Correct projection and results when filter on non-indexed column in covering plan. - public void nonIndexedColumnFilterCoveringPlan() throws Exception { - String query = "SELECT t.name.fname as fname FROM hbase.`index_test_primary` as t " + - " where t.personal.age > 68 and t.name.fname IN ('CnGobfR', 'THOHP')"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {".*Filter.*CnGobfR.*THOHP.*", - ".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName="}, - new String[] {".*Filter.*ITEM*CnGobfR.*THOHP.*"}); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("fname").baselineValues("CnGobfR") - .baselineColumns("fname").baselineValues("THOHP") - .baselineColumns("fname").baselineValues("CnGobfR") - .go(); - } - - @Test - @Ignore - public void orderByLimitNonCoveringPlan() throws Exception { - String query = "SELECT t.name.lname as lname FROM hbase.`index_test_primary` as t " + - " where t.id.ssn < '100000003' order by t.id.ssn limit 2"; - try { - test(defaultHavingIndexPlan); - test(sliceTargetSmall); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"Limit(.*[\n\r])+.*SingleMergeExchange(.*[\n\r])+.*Limit(.*[\n\r])+.*indexName="}, - new String[]{"Sort"} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("lname").baselineValues("iuMG") - .baselineColumns("lname").baselineValues("KpFq") - .go(); - } finally { - test(sliceTargetDefault); - } - } - - @Test - public void orderByLimitCoveringPlan() throws Exception { - String query = "SELECT t.contact.phone as phone FROM hbase.`index_test_primary` as t " + - " where t.id.ssn < '100000003' order by t.id.ssn limit 2"; - test(defaultHavingIndexPlan); - - //when index table has only one tablet, the SingleMergeExchange in the middle of two Limits will be removed. - //The lower limit gets pushed into the scan - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"Limit(.*[\n\r])+.*indexName=.*limit=2"}, - new String[]{"Sort"} - ); - - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("phone").baselineValues("6500008069") - .baselineColumns("phone").baselineValues("6500001411") - .go(); - } - - @Test - public void pickAnyIndexWithFTSDisabledPlan() throws Exception { - String lowCoveringSel = "alter session set `planner.index.covering_selectivity_threshold` = 0.025"; - String defaultCoveringSel = "alter session reset `planner.index.covering_selectivity_threshold`"; - String query = "SELECT t.`contact`.`phone` AS `phone` FROM hbase.`index_test_primary` as t " + - " where t.id.ssn = '100007423'"; - try { - test(defaultHavingIndexPlan + ";" + lowCoveringSel + ";"); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{".*JsonTableGroupScan.*tableName=.*index_test_primary"}, - new String[]{".*indexName=i_ssn"} - ); - // Must not throw CANNOTPLANEXCEPTION - test(defaultHavingIndexPlan + ";" + lowCoveringSel + ";" + disableFTS + ";"); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_ssn"}, - new String[]{"RowKeyJoin"} - ); - } finally { - test(defaultCoveringSel+";"+enableFTS+";"); - } - } - - @Test - public void testCaseSensitive() throws Exception { - String query = "SELECT t.contact.phone as phone FROM hbase.`index_test_primary` as t " + - " where t.id.SSN = '100000003' "; - test(defaultHavingIndexPlan); - - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {""}, - new String[]{"indexName"} - ); - - } - - @Test - public void testCaseSensitiveIncludedField() throws Exception { - - String query = "SELECT t.`CONTACT`.`phone` AS `phone` FROM hbase.`index_test_primary` as t " + - " where t.id.ssn = '100007423'"; - test(defaultHavingIndexPlan); - - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"RowKeyJoin", - ".*JsonTableGroupScan.*tableName=.*index_test_primary.*indexName=i_ssn"}, - new String[]{} - ); - } - - - @Test - public void testHashIndexNoRemovingSort() throws Exception { - String query = "SELECT t.`contact`.`phone` as phone FROM hbase.`index_test_primary` as t " + - " where t.reverseid <'10' order by t.reverseid"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"Sort", "indexName=hash_i_reverseid", "RowKeyJoin"}, - new String[]{} - ); - } - - @Ignore - @Test - public void testCastTimestampPlan() throws Exception { - String query = "SELECT t.id.ssn as ssn FROM hbase.`index_test_primary` as t " + - " where cast(t.activity.irs.firstlogin as timestamp)=to_timestamp('2013-02-04 22:34:38.0', 'YYYY-MM-dd HH:mm:ss.S')"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"indexName=hash_i_cast_timestamp_firstlogin"}, - new String[]{"RowKeyJoin"} - ); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .go(); - - } - - @Test - public void testNotConditionNoIndexPlan() throws Exception { - String query = "SELECT t.`id`.`ssn` AS `ssn` FROM hbase.`index_test_primary` as t " + - " where NOT t.id.ssn = '100007423'"; - - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {}, - new String[]{"indexName="} - ); - - - String notInQuery = "SELECT t.`id`.`ssn` AS `ssn` FROM hbase.`index_test_primary` as t " + - " where t.id.ssn NOT IN ('100007423', '100007424')"; - PlanTestBase.testPlanMatchingPatterns(notInQuery, - new String[] {}, - new String[]{"indexName="} - ); - - String notLikeQuery = "SELECT t.`id`.`ssn` AS `ssn` FROM hbase.`index_test_primary` as t " + - " where t.id.ssn NOT LIKE '100007423'"; - PlanTestBase.testPlanMatchingPatterns(notLikeQuery, - new String[] {}, - new String[]{"indexName="} - ); - - } - - @Test - public void testNoFilterOrderByCoveringPlan() throws Exception { - String query = "SELECT t.`id`.`ssn` AS `ssn`, t.contact.phone as phone FROM hbase.`index_test_primary` as t " + - "order by t.id.ssn limit 2"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"indexName=i_ssn"}, - new String[]{"Sort", "TopN", "RowKeyJoin"} - ); - testBuilder() - .ordered() - .sqlQuery(query) - .baselineColumns("ssn", "phone").baselineValues("100000000", "6500008069") - .baselineColumns("ssn", "phone").baselineValues("100000001", "6500001411") - .build() - .run(); - } - - @Test - public void testNoFilterAndLimitOrderByCoveringPlan() throws Exception { - String query = "SELECT t.`id`.`ssn` AS `ssn`, t.contact.phone as phone FROM hbase.`index_test_primary` as t " + - "order by t.id.ssn"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"Sort"}, - new String[]{"indexName=*", "RowKeyJoin", "TopN"} - ); - } - - @Test - public void testNoFilterOrderByCast() throws Exception { - String query = "SELECT CAST(t.id.ssn as INT) AS `ssn`, t.contact.phone as phone FROM hbase.`index_test_primary` as t " + - "order by CAST(t.id.ssn as INT) limit 2"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"indexName=i_cast_int_ssn"}, - new String[]{"TopN", "Sort", "RowKeyJoin"} - ); - testBuilder() - .ordered() - .sqlQuery(query) - .baselineColumns("ssn", "phone").baselineValues(100000000, "6500008069") - .baselineColumns("ssn", "phone").baselineValues(100000001, "6500001411") - .build() - .run(); - } - - @Test - public void testNoFilterAndLimitOrderByCast() throws Exception { - String query = "SELECT CAST(t.id.ssn as INT) AS `ssn`, t.contact.phone as phone FROM hbase.`index_test_primary` as t " + - "order by CAST(t.id.ssn as INT)"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] { "Sort"}, - new String[]{"indexName=*","TopN", "RowKeyJoin"} - ); - } - - @Test - public void testNoFilterOrderByHashIndex() throws Exception { - String query = "SELECT cast(t.activity.irs.firstlogin as timestamp) AS `firstlogin`, t.id.ssn as ssn FROM hbase.`index_test_primary` as t " + - "order by cast(t.activity.irs.firstlogin as timestamp), t.id.ssn limit 2"; - test(defaultHavingIndexPlan); - // no collation for hash index so Sort or TopN must have been preserved - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"(Sort|TopN)"}, - new String[]{"indexName="} - ); - DateTime date = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss") - .parseDateTime("2010-01-21 00:12:24"); - - testBuilder() - .ordered() - .sqlQuery(query) - .baselineColumns("firstlogin", "ssn").baselineValues(date, "100005592") - .baselineColumns("firstlogin", "ssn").baselineValues(date, "100005844") - .build() - .run(); - } - - @Test - public void testNoFilterOrderBySimpleField() throws Exception { - String query = "SELECT t.reverseid as rid, t.driverlicense as lic FROM hbase.`index_test_primary` as t " + - "order by t.driverlicense limit 2"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"indexName=i_lic"}, - new String[]{"Sort", "TopN"} - ); - testBuilder() - .ordered() - .sqlQuery(query) - .baselineColumns("rid", "lic").baselineValues("4539", 100000000L) - .baselineColumns("rid", "lic").baselineValues("943", 100000001L) - .build() - .run(); - } - - @Test // negative case for no filter plan - public void testNoFilterOrderByNoIndexMatch() throws Exception { - String query = "SELECT t.`id`.`ssn` AS `ssn`, t.contact.phone as phone FROM hbase.`index_test_primary` as t " + - "order by t.name.fname limit 2"; - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"(Sort|TopN)"}, - new String[]{"indexName="} - ); - } - -// This test case encounters an error : -// "Error: SYSTEM ERROR: IllegalStateException: Batch data read operation (iterator()) attempted when last -// next() call on batch [#16, ScanBatch] returned NONE (not OK or OK_NEW_SCHEMA)." -// TODO: fix the root cause of the above error then enable the test - @Test - @Ignore - public void IntersectPlanWithOneSideNoRows() throws Exception { - try { - String query = "SELECT t.`name`.`lname` AS `lname` FROM hbase.`index_test_primary` as t " + - " where t.personal.age = 53 AND t.personal.income=111145"; - test(defaultHavingIndexPlan); - test(preferIntersectPlans + ";" + disableFTS); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*HashJoin(.*[\n\r])+.*JsonTableGroupScan.*indexName=(i_age|i_income)(.*[\n\r])+.*JsonTableGroupScan.*indexName=(i_age|i_income)"}, - new String[]{} - ); - - testNoResult(query); - - } finally { - test(defaultIntersectPlans + ";" + enableFTS); - } - } - - @Test - public void testTrailingFieldIndexCovering() throws Exception { - String query = "SELECT t.`name`.`fname` AS `fname` FROM hbase.`index_test_primary` as t " + - " where cast(t.personal.age as INT)=53 AND t.contact.phone='6500005471' "; - - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"indexName=i_cast_age_income_phone"}, - new String[]{"RowKeyJoin"} - ); - - testBuilder() - .ordered() - .sqlQuery(query) - .baselineColumns("fname").baselineValues("KfFzK") - .build() - .run(); - } - - @Test - public void testIncludedFieldCovering() throws Exception { - String query = "SELECT t.`contact`.`phone` AS `phone` FROM hbase.`index_test_primary` as t " + - " where cast(t.personal.age as INT)=53 AND t.name.fname='KfFzK' "; - - test(defaultHavingIndexPlan); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"indexName=i_cast_age_income_phone"}, - new String[]{"RowKeyJoin"} - ); - - testBuilder() - .ordered() - .sqlQuery(query) - .baselineColumns("phone").baselineValues("6500005471") - .build() - .run(); - } - - @Test - public void testWithFilterGroupBy() throws Exception { - String query = " select t1.driverlicense from hbase.`index_test_primary` t1" + - " where t1.driverlicense > 100000001 group by t1.driverlicense limit 2"; - try { - test(defaultHavingIndexPlan); - test(disableHashAgg); - // no collation for hash index so Sort or TopN must have been preserved - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"indexName=i_lic", "StreamAgg"}, - new String[]{"(Sort|TopN)"} - ); - - testBuilder() - .ordered() - .sqlQuery(query) - .optionSettingQueriesForTestQuery(disableHashAgg) - .baselineColumns("driverlicense").baselineValues(100000002L) - .baselineColumns("driverlicense").baselineValues(100000003L) - .build() - .run(); - } finally { - test(enableHashAgg); - } - } - - @Test - public void testNoFilterOrderByDesc() throws Exception { - String query = " select t1.driverlicense from hbase.`index_test_primary` t1" + - " order by t1.driverlicense desc limit 2"; - test(defaultHavingIndexPlan); - // no collation for hash index so Sort or TopN must have been preserved - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"(Sort|TopN)"}, - new String[]{"indexName="} - ); - - testBuilder() - .unOrdered() - .sqlQuery(query) - .baselineColumns("driverlicense").baselineValues(100009999L) - .baselineColumns("driverlicense").baselineValues(100009998L) - .build() - .run(); - } - - @Test - public void testNoFilterGroupBy() throws Exception { - String query = " select t1.driverlicense from hbase.`index_test_primary` t1" + - " group by t1.driverlicense limit 2"; - try { - test(defaultHavingIndexPlan); - test(disableHashAgg); - // no collation for hash index so Sort or TopN must have been preserved - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"indexName=i_lic", "StreamAgg"}, - new String[]{"(Sort|TopN)"} - ); - - testBuilder() - .ordered() - .sqlQuery(query) - .optionSettingQueriesForTestQuery(disableHashAgg) - .baselineColumns("driverlicense").baselineValues(100000000L) - .baselineColumns("driverlicense").baselineValues(100000001L) - .build() - .run(); - } finally { - test(enableHashAgg); - } - } - - @Test - public void testNoFilterGroupByCoveringPlan() throws Exception { - String query = "SELECT t.`id`.`ssn` AS `ssn`, max(t.contact.phone) as phone FROM hbase.`index_test_primary` as t " + - "group by t.id.ssn limit 2"; - try { - test(defaultHavingIndexPlan); - test(disableHashAgg); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"indexName=i_ssn", "StreamAgg"}, - new String[]{"Sort", "TopN", "RowKeyJoin"} - ); - testBuilder() - .ordered() - .sqlQuery(query) - .optionSettingQueriesForTestQuery(disableHashAgg) - .baselineColumns("ssn", "phone").baselineValues("100000000", "6500008069") - .baselineColumns("ssn", "phone").baselineValues("100000001", "6500001411") - .build() - .run(); - } finally { - test(enableHashAgg); - } - } - - @Test - public void testNoFilterGroupByCast() throws Exception { - String query = "SELECT CAST(t.id.ssn as INT) AS `ssn`, max(t.contact.phone) as phone FROM hbase.`index_test_primary` as t " + - "group by CAST(t.id.ssn as INT) limit 2"; - try { - test(defaultHavingIndexPlan); - test(disableHashAgg); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"indexName=i_cast_int_ssn", "StreamAgg"}, - new String[]{"TopN", "Sort", "RowKeyJoin"} - ); - testBuilder() - .ordered() - .sqlQuery(query) - .optionSettingQueriesForTestQuery(disableHashAgg) - .baselineColumns("ssn", "phone").baselineValues(100000000, "6500008069") - .baselineColumns("ssn", "phone").baselineValues(100000001, "6500001411") - .build() - .run(); - } finally { - test(enableHashAgg); - } - } - - @Test - public void testNoFilterGroupByHashIndex() throws Exception { - String query = "SELECT cast(t.activity.irs.firstlogin as timestamp) AS `firstlogin`, max(t.id.ssn) as ssn FROM hbase.`index_test_primary` as t " + - "group by cast(t.activity.irs.firstlogin as timestamp) limit 2"; - try { - test(defaultHavingIndexPlan); - test(disableHashAgg); - // no collation for hash index so Sort or TopN must have been preserved - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"(Sort|TopN)", "StreamAgg"}, - new String[]{"indexName="} - ); - DateTime date1 = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss") - .parseDateTime("2010-01-21 00:12:24"); - - DateTime date2 = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss") - .parseDateTime("2010-01-21 00:24:48"); - testBuilder() - .unOrdered() - .sqlQuery(query) - .optionSettingQueriesForTestQuery(disableHashAgg) - .baselineColumns("firstlogin", "ssn").baselineValues(date1, "100006852") - .baselineColumns("firstlogin", "ssn").baselineValues(date2, "100003660") - .build() - .run(); - } finally { - test(enableHashAgg); - } - } - - @Test - public void testNoFilterGroupBySimpleField() throws Exception { - String query = "SELECT max(t.reverseid) as rid, t.driverlicense as lic FROM hbase.`index_test_primary` as t " + - "group by t.driverlicense limit 2"; - try { - test(defaultHavingIndexPlan); - test(disableHashAgg); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"indexName=i_lic", "StreamAgg"}, - new String[]{"Sort", "TopN"} - ); - testBuilder() - .ordered() - .sqlQuery(query) - .optionSettingQueriesForTestQuery(disableHashAgg) - .baselineColumns("rid", "lic").baselineValues("4539", 100000000L) - .baselineColumns("rid", "lic").baselineValues("943", 100000001L) - .build() - .run(); - } finally { - test(enableHashAgg); - } - } - - @Test // negative case for no filter plan - public void testNoFilterGroupByNoIndexMatch() throws Exception { - String query = "SELECT max(t.`id`.`ssn`) AS `ssn`, max(t.contact.phone) as phone FROM hbase.`index_test_primary` as t " + - "group by t.name.fname limit 2"; - try { - test(defaultHavingIndexPlan); - test(disableHashAgg); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"(Sort|TopN)", "StreamAgg"}, - new String[]{"indexName="} - ); - } finally { - test(enableHashAgg); - } - } - - @Test - public void testNoFilterGroupBySimpleFieldParallel() throws Exception { - String query = "SELECT max(t.reverseid) as rid, t.driverlicense as lic FROM hbase.`index_test_primary` as t " + - "group by t.driverlicense order by t.driverlicense limit 2"; - try { - test(defaultHavingIndexPlan); - test(disableHashAgg); - test(sliceTargetSmall); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"indexName=i_lic", "StreamAgg", "HashToMergeExchange"}, - new String[]{"Sort", "TopN"} - ); - testBuilder() - .unOrdered() - .sqlQuery(query) - .optionSettingQueriesForTestQuery(defaultHavingIndexPlan) - .optionSettingQueriesForTestQuery(disableHashAgg) - .optionSettingQueriesForTestQuery(sliceTargetSmall) - .baselineColumns("rid", "lic").baselineValues("4539", 100000000L) - .baselineColumns("rid", "lic").baselineValues("943", 100000001L) - .build() - .run(); - } finally { - test(enableHashAgg); - test(sliceTargetDefault); - } - } - - @Test - public void testLimitPushdownCoveringPlan() throws Exception { - String query = "SELECT t.`name`.`fname` AS `fname` FROM hbase.`index_test_primary` as t " + - " where t.personal.age = 53 limit 3"; - try { - test(defaultHavingIndexPlan + ";" + disableFTS + ";"); - PlanTestBase.testPlanWithAttributesMatchingPatterns(query, - new String[]{".*JsonTableGroupScan.*indexName=i_age_with_fname.*rowcount = 3.0"}, - new String[]{} - ); - } finally { - test(enableFTS); - } - } - - @Test - public void testLimitPushdownOrderByCoveringPlan() throws Exception { - String query = "SELECT t.`name`.`fname` AS `fname` FROM hbase.`index_test_primary` as t " + - " where t.personal.age = 53 order by t.personal.age limit 3"; - try { - test(defaultHavingIndexPlan + ";" + disableFTS + ";"); - PlanTestBase.testPlanWithAttributesMatchingPatterns(query, - new String[]{".*JsonTableGroupScan.*indexName=i_age_with_fname.*rowcount = 3.0"}, - new String[]{} - ); - } finally { - test(enableFTS); - } - } - - @Test - public void testLimitPushdownNonCoveringPlan() throws Exception { - String query = "SELECT t.`name`.`lname` AS `lname` FROM hbase.`index_test_primary` as t " + - " where t.personal.age = 53 limit 7"; - try { - test(defaultHavingIndexPlan+";"+disableFTS+";"); - PlanTestBase.testPlanWithAttributesMatchingPatterns(query, - new String[]{"RowKeyJoin", ".*RestrictedJsonTableGroupScan.*tableName=.*index_test_primary.*rowcount = 7.0"}, - new String[]{} - ); - } finally { - test(enableFTS); - } - } - - @Test - public void testLimitPushdownOrderByNonCoveringPlan() throws Exception { - // Limit pushdown should NOT happen past rowkey join when ordering is required - String query = "SELECT t.`name`.`lname` AS `lname` FROM hbase.`index_test_primary` as t " + - " where t.personal.age = 53 order by t.personal.age limit 7"; - try { - test(defaultHavingIndexPlan + ";" + disableFTS + ";" + sliceTargetSmall + ";"); - PlanTestBase.testPlanWithAttributesMatchingPatterns(query, - new String[]{"RowKeyJoin", ".*RestrictedJsonTableGroupScan.*"}, - new String[]{".*tableName=.*index_test_primary.*rowcount = 7.*"} - ); - } finally { - test(enableFTS); - } - } - - @Test - public void testLimit0Pushdown() throws Exception { - // Limit pushdown should NOT happen past project with CONVERT_FROMJSON - String query = "select convert_from(convert_to(t.`name`.`lname`, 'JSON'), 'JSON') " + - "from hbase.`index_test_primary` as t limit 0"; - try { - test(defaultHavingIndexPlan + ";"); - PlanTestBase.testPlanWithAttributesMatchingPatterns(query, - new String[]{"Limit(.*[\n\r])+.*Project.*CONVERT_FROMJSON(.*[\n\r])+.*Scan"}, - new String[]{} - ); - } finally { - } - } - - @Test - public void testRemovalOfReduntantHashToMergeExchange() throws Exception { - String query = "SELECT t.driverlicense as lic FROM hbase.`index_test_primary` as t " + - "order by t.driverlicense limit 2"; - try { - test(defaultHavingIndexPlan); - test(sliceTargetSmall); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"indexName=i_lic"}, - new String[]{"HashToMergeExchange", "Sort", "TopN"}); - testBuilder() - .ordered() - .sqlQuery(query) - .baselineColumns("lic").baselineValues(100000000L) - .baselineColumns("lic").baselineValues(100000001L) - .build() - .run(); - } finally { - test(sliceTargetDefault); - } - } - - @Test - public void testMultiPhaseAgg() throws Exception { - String query = "select count(t.reverseid) from hbase.`index_test_primary` as t " + - "group by t.driverlicense order by t.driverlicense"; - try { - test(defaultHavingIndexPlan); - test(sliceTargetSmall); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{"indexName=i_lic", "HashToMergeExchange", "StreamAgg", "StreamAgg"}, - new String[]{"Sort", "TopN"}); - } finally { - test(sliceTargetDefault); - } - } - - @Test - public void testHangForSimpleDistinct() throws Exception { - String query = "select distinct t.driverlicense from hbase.`index_test_primary` as t order by t.driverlicense limit 1"; - - try { - test(sliceTargetSmall); - testBuilder() - .ordered() - .sqlQuery(query) - .baselineColumns("driverlicense").baselineValues(100000000L) - .build() - .run(); - } finally { - test(sliceTargetDefault); - } - } - - @Test - public void testRowkeyJoinPushdown_1() throws Exception { - // _id IN (select col ...) - String query = "select t1.id.ssn as ssn from hbase.`index_test_primary` t1 where _id in (select t2._id " + - " from hbase.`index_test_primary` t2 where t2.address.city = 'pfrrs' and t2.address.state = 'pc')"; - try { - test(incrRowKeyJoinConvSelThreshold + ";" + lowNonCoveringSelectivityThreshold + ";"); - PlanTestBase.testPlanMatchingPatterns(query, new String[] {"RowKeyJoin"}, new String[] {}); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .baselineColumns("ssn").baselineValues("100008861") - .go(); - } finally { - test(defaultRowKeyConvSelThreshold + ";" + defaultnonCoveringSelectivityThreshold + ";"); - } - } - - @Test - public void testRowkeyJoinPushdown_2() throws Exception { - // _id = col - String query = "select t1.id.ssn as ssn from hbase.`index_test_primary` t1, hbase.`index_test_primary` t2 " + - " where t1._id = t2._id and t2.address.city = 'pfrrs' and t2.address.state = 'pc'"; - try { - test(incrRowKeyJoinConvSelThreshold + ";" + lowNonCoveringSelectivityThreshold + ";"); - PlanTestBase.testPlanMatchingPatterns(query, new String[] {"RowKeyJoin"}, new String[] {}); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .baselineColumns("ssn").baselineValues("100008861") - .go(); - } finally { - test(defaultRowKeyConvSelThreshold + ";" + defaultnonCoveringSelectivityThreshold + ";"); - } - } - - @Test - public void testRowkeyJoinPushdown_3() throws Exception { - // filters on both sides of the join - String query = "select t1.id.ssn as ssn from hbase.`index_test_primary` t1, hbase.`index_test_primary` t2 " + - " where t1._id = t2._id and t1.address.city = 'pfrrs' and t2.address.city = 'pfrrs'"; - try { - test(incrRowKeyJoinConvSelThreshold + ";" + lowNonCoveringSelectivityThreshold + ";"); - PlanTestBase.testPlanMatchingPatterns(query, new String[] {"RowKeyJoin"}, new String[] {}); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .baselineColumns("ssn").baselineValues("100008861") - .go(); - } finally { - test(defaultRowKeyConvSelThreshold + ";" + defaultnonCoveringSelectivityThreshold + ";"); - } - } - - @Test - public void testRowkeyJoinPushdown_4() throws Exception { - // _id = cast(col as int) works since the rowids are internally cast to string! - String query = "select t1.id.ssn as ssn from hbase.`index_test_primary` t1, hbase.`index_test_primary` t2 " + - " where t1._id = cast(t2.rowid as int) and t2.address.city = 'pfrrs'"; - try { - test(incrRowKeyJoinConvSelThreshold + ";" + lowNonCoveringSelectivityThreshold + ";"); - PlanTestBase.testPlanMatchingPatterns(query, new String[] {"RowKeyJoin"}, new String[] {}); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .baselineColumns("ssn").baselineValues("100008861") - .go(); - } finally { - test(defaultRowKeyConvSelThreshold + ";" + defaultnonCoveringSelectivityThreshold + ";"); - } - } - - @Test - public void testRowkeyJoinPushdown_5() throws Exception { - // _id = cast(cast(col as int) as varchar(10) - String query = "select t1.id.ssn as ssn from hbase.`index_test_primary` t1, hbase.`index_test_primary` t2 " + - " where t1._id = cast(cast(t2.rowid as int) as varchar(10)) and t2.address.city = 'pfrrs'"; - try { - test(incrRowKeyJoinConvSelThreshold + ";" + lowNonCoveringSelectivityThreshold + ";"); - PlanTestBase.testPlanMatchingPatterns(query, new String[] {"RowKeyJoin"}, new String[] {}); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .baselineColumns("ssn").baselineValues("100008861") - .go(); - } finally { - test(defaultRowKeyConvSelThreshold + ";" + defaultnonCoveringSelectivityThreshold + ";"); - } - } - - @Test - public void testRowkeyJoinPushdown_6() throws Exception { - // _id IN (select cast(cast(col as int) as varchar(10) ... JOIN ...) - String query = "select t1.id.ssn as ssn from hbase.`index_test_primary` t1 where _id in " + - "(select cast(cast(t2.rowid as int) as varchar(10)) from hbase.`index_test_primary` t2, hbase.`index_test_primary` t3 " + - "where t2.address.city = t3.address.city and t2.name.fname = 'ubar')"; - try { - test(incrRowKeyJoinConvSelThreshold + ";" + lowNonCoveringSelectivityThreshold + ";"); - PlanTestBase.testPlanMatchingPatterns(query, new String[] {"RowKeyJoin"}, new String[] {}); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100001382") - .go(); - } finally { - test(defaultRowKeyConvSelThreshold + ";" + defaultnonCoveringSelectivityThreshold + ";"); - } - } - - @Test - public void testRowkeyJoinPushdown_7() throws Exception { - // with non-covering index - String query = "select t1.id.ssn as ssn from hbase.`index_test_primary` t1, hbase.`index_test_primary` t2 " + - "where t1._id = t2.rowid and t2.address.city = 'pfrrs'"; - try { - test(incrRowKeyJoinConvSelThreshold + ";" + incrnonCoveringSelectivityThreshold + ";"); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"RowKeyJoin", "RestrictedJsonTableGroupScan", "RowKeyJoin", "Scan.*condition=\\(address.city = \"pfrrs\"\\)"}, - new String[] {}); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .baselineColumns("ssn").baselineValues("100008861") - .go(); - } finally { - test(defaultRowKeyConvSelThreshold + ";" + defaultnonCoveringSelectivityThreshold + ";"); - } - } - - @Test - public void testRowkeyJoinPushdown_8() throws Exception { - // with covering index - String query = "select t1.id.ssn as ssn from hbase.`index_test_primary` t1, hbase.`index_test_primary` t2 " + - "where t1._id = t2.rowid and t2.rowid = '1012'"; - try { - test(incrRowKeyJoinConvSelThreshold); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"RowKeyJoin", "indexName=i_rowid_cast_date_birthdate"}, - new String[] {}); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .go(); - } finally { - test(defaultRowKeyConvSelThreshold); - } - } - - @Test - public void testRowkeyJoinPushdown_9() throws Exception { - // Negative test - rowkey join should not be present - String query = "select t1.id.ssn as ssn from hbase.`index_test_primary` t1 where cast(_id as varchar(10)) in " + - "(select t2._id from hbase.`index_test_primary` t2 where t2.address.city = 'pfrrs')"; - try { - test(incrRowKeyJoinConvSelThreshold + ";" + lowNonCoveringSelectivityThreshold + ";"); - PlanTestBase.testPlanMatchingPatterns(query, new String[] {}, new String[] {"RowKeyJoin"}); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .baselineColumns("ssn").baselineValues("100008861") - .go(); - } finally { - test(defaultRowKeyConvSelThreshold + ";" + defaultnonCoveringSelectivityThreshold + ";"); - } - } - - @Test - public void testRowkeyJoinPushdown_10() throws Exception { - // Negative test - rowkey join should not be present - String query = "select t1.id.ssn as ssn from hbase.`index_test_primary` t1, hbase.`index_test_primary` t2 " + - " where cast(t1._id as varchar(10)) = cast(t2._id as varchar(10)) and t2.address.city = 'pfrrs'"; - try { - test(incrRowKeyJoinConvSelThreshold + ";" + lowNonCoveringSelectivityThreshold + ";"); - PlanTestBase.testPlanMatchingPatterns(query, new String[] {}, new String[] {"RowKeyJoin"}); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .baselineColumns("ssn").baselineValues("100008861") - .go(); - } finally { - test(defaultRowKeyConvSelThreshold + ";" + defaultnonCoveringSelectivityThreshold + ";"); - } - } - - @Test - public void testRowkeyJoinPushdown_11() throws Exception { - // Negative test - rowkey join should not be present - String query = "select t1.id.ssn as ssn from hbase.`index_test_primary` t1 where cast(_id as varchar(10)) in " + - "(select t2._id from hbase.`index_test_primary` t2, hbase.`index_test_primary` t3 where t2.address.city = t3.address.city " + - "and t2.address.city = 'pfrrs')"; - try { - test(incrRowKeyJoinConvSelThreshold + ";" + lowNonCoveringSelectivityThreshold + ";"); - PlanTestBase.testPlanMatchingPatterns(query, new String[] {}, new String[] {"RowKeyJoin"}); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .baselineColumns("ssn").baselineValues("100008861") - .go(); - } finally { - test(defaultRowKeyConvSelThreshold + ";" + defaultnonCoveringSelectivityThreshold + ";"); - } - } - - @Ignore - @Test - public void testRowkeyJoinPushdown_12() throws Exception { - // JOIN _id IN (select cast(cast(col as int) as varchar(10) ... JOIN ...) - rowkey join appears in intermediate join order - String query = "select t1.id.ssn as ssn from hbase.`index_test_primary` t1, hbase.`index_test_primary` t4 " + - "where t1.address.city = t4.address.city and t1._id in (select cast(cast(t2.rowid as int) as varchar(10)) " + - "from hbase.`index_test_primary` t2, hbase.`index_test_primary` t3 where t2.address.city = t3.address.city " + - "and t2.address.state = 'pc') and t4.address.state = 'pc'"; - try { - test(incrRowKeyJoinConvSelThreshold + ";" + lowNonCoveringSelectivityThreshold + ";"); - PlanTestBase.testPlanMatchingPatterns(query, - new String[] {"HashJoin(.*[\n\r])+.*Scan.*indexName=i_state_city_dl(.*[\n\r])+.*RowKeyJoin(.*[\n\r])+.*RestrictedJsonTableGroupScan(.*[\n\r])+.*HashAgg\\(group=\\[\\{0\\}\\]\\)(.*[\n\r])+.*HashJoin"}, - new String[] {}); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .baselineColumns("ssn").baselineValues("100007423") - .go(); - } finally { - test(defaultRowKeyConvSelThreshold + ";" + defaultnonCoveringSelectivityThreshold + ";"); - } - } - - @Ignore - @Test - public void testRowkeyJoinPushdown_13() throws Exception { - // Check option planner.rowkeyjoin_conversion_using_hashjoin works as expected! - String query = "select t1.id.ssn as ssn from hbase.`index_test_primary` t1 where _id in (select t2._id " + - " from hbase.`index_test_primary` t2 where t2.address.city = 'pfrrs')"; - try { - test(incrRowKeyJoinConvSelThreshold + ";" + lowNonCoveringSelectivityThreshold + ";"); - PlanTestBase.testPlanMatchingPatterns(query, new String[]{"RowKeyJoin"}, new String[]{}); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .baselineColumns("ssn").baselineValues("100008861") - .go(); - test(incrRowKeyJoinConvSelThreshold + ";" + lowNonCoveringSelectivityThreshold + ";" + - forceRowKeyJoinConversionUsingHashJoin + ";"); - PlanTestBase.testPlanMatchingPatterns(query, new String[]{"HashJoin"}, new String[]{"RowKeyJoin"}); - testBuilder() - .sqlQuery(query) - .ordered() - .baselineColumns("ssn").baselineValues("100007423") - .baselineColumns("ssn").baselineValues("100008861") - .go(); - } finally { - test(defaultRowKeyConvSelThreshold + ";" + defaultnonCoveringSelectivityThreshold + ";" + - defaultRowKeyJoinConversionUsingHashJoin); - } - } - - public void TestIndexScanWithDescOrderByNullsFirst() throws Exception { - - String query = "select t.personal.age from hbase.`index_test_primary` t order by t.personal.age desc nulls first limit 1"; - try { - test(defaultHavingIndexPlan + ";" + lowRowKeyJoinBackIOFactor + ";"); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{".*JsonTableGroupScan.*indexName=i_age_desc.*"}, - new String[]{} - ); - } finally { - test(defaultRowKeyJoinBackIOFactor); - } - return; - } - - @Test - public void TestIndexScanWithDescOrderByNullsLast() throws Exception { - - String query = "select t.personal.age from hbase.`index_test_primary` t order by t.personal.age desc nulls last limit 1"; - try { - test(defaultHavingIndexPlan + ";" + lowRowKeyJoinBackIOFactor + ";"); - PlanTestBase.testPlanMatchingPatterns(query, - new String[]{}, - new String[]{".*indexName=i_age_desc.*"} - ); - } finally { - test(defaultRowKeyJoinBackIOFactor); - } - return; - } -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/LargeTableGen.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/LargeTableGen.java deleted file mode 100644 index 1b64d8e4a67..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/LargeTableGen.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests.index; - -import static com.mapr.drill.maprdb.tests.MaprDBTestsSuite.INDEX_FLUSH_TIMEOUT; - -import java.io.InputStream; -import java.io.StringBufferInputStream; - -import org.apache.hadoop.fs.Path; -import org.ojai.DocumentStream; -import org.ojai.json.Json; - -import com.mapr.db.Admin; -import com.mapr.db.Table; -import com.mapr.db.TableDescriptor; -import com.mapr.db.impl.MapRDBImpl; -import com.mapr.db.impl.TableDescriptorImpl; -import com.mapr.db.tests.utils.DBTests; -import com.mapr.fs.utils.ssh.TestCluster; - -/** - * This class is to generate a MapR json table of this schema: - * { - * "address" : { - * "city":"wtj", - * "state":"ho" - * } - * "contact" : { - * "email":"VcFahjRfM@gmail.com", - * "phone":"6500005583" - * } - * "id" : { - * "ssn":"100005461" - * } - * "name" : { - * "fname":"VcFahj", - * "lname":"RfM" - * } - * } - * - */ -public class LargeTableGen extends LargeTableGenBase { - - static final int SPLIT_SIZE = 5000; - private Admin admin; - - public LargeTableGen(Admin dbadmin) { - admin = dbadmin; - } - - Table createOrGetTable(String tableName, int recordNum) { - if (admin.tableExists(tableName)) { - return MapRDBImpl.getTable(tableName); - // admin.deleteTable(tableName); - } - else { - TableDescriptor desc = new TableDescriptorImpl(new Path(tableName)); - - int splits = (recordNum / SPLIT_SIZE) - (((recordNum % SPLIT_SIZE) > 1)? 0 : 1); - - String[] splitsStr = new String[splits]; - StringBuilder strBuilder = new StringBuilder("Splits:"); - for (int i = 0; i < splits; ++i) { - splitsStr[i] = String.format("%d", (i+1)*SPLIT_SIZE); - strBuilder.append(splitsStr[i] + ", "); - } - System.out.print(strBuilder.toString()); - - return admin.createTable(desc, splitsStr); - } - } - - private void createIndex(Table table, String[] indexDef) throws Exception { - if (indexDef == null) { - // don't create index here. indexes may have been created - return; - } - for (int i = 0; i < indexDef.length / 3; ++i) { - String indexCmd = String.format("maprcli table index add" - + " -path " + table.getPath() - + " -index %s" - + " -indexedfields '%s'" - + ((indexDef[3 * i + 2].length()==0)?"":" -includedfields '%s'") - + ((indexDef[3 * i].startsWith("hash"))? " -hashed true" : ""), - indexDefInCommand(indexDef[3 * i]), // index name - indexDefInCommand(indexDef[3 * i + 1]), // indexedfields - indexDefInCommand(indexDef[3 * i + 2])); // includedfields - System.out.println(indexCmd); - - TestCluster.runCommand(indexCmd); - DBTests.admin().getTableIndexes(table.getPath(), true); - } - } - - private String indexDefInCommand(String def) { - String[] splitted = def.split(","); - StringBuffer ret = new StringBuffer(); - for (String field: splitted) { - if (ret.length() == 0) { - ret.append(field); - } - else { - ret.append(",").append(field); - } - } - return ret.toString(); - } - public void generateTableWithIndex(String tablePath, int recordNumber, String[] indexDef) throws Exception { - // create index - - initRandVector(recordNumber); - initDictionary(); - DBTests.setTableStatsSendInterval(1); - - if (admin.tableExists(tablePath)) { - // admin.deleteTable(tablePath); - } - - // create Json String - int batch, i; - int BATCH_SIZE=2000; - try (Table table = createOrGetTable(tablePath, recordNumber)) { - // create index - createIndex(table, indexDef); - for (batch = 0; batch < recordNumber; batch += BATCH_SIZE) { - int batchStop = Math.min(recordNumber, batch + BATCH_SIZE); - StringBuffer strBuf = new StringBuffer(); - for (i = batch; i < batchStop; ++i) { - - strBuf.append(String.format("{\"rowid\": \"%d\", \"reverseid\": \"%d\", \"id\": {\"ssn\": \"%s\"}, \"contact\": {\"phone\": \"%s\", \"email\": \"%s\"}," + - "\"address\": {\"city\": \"%s\", \"state\": \"%s\"}, \"name\": { \"fname\": \"%s\", \"lname\": \"%s\" }," + - "\"personal\": {\"age\" : %s, \"income\": %s, \"birthdate\": {\"$dateDay\": \"%s\"} }," + - "\"activity\": {\"irs\" : { \"firstlogin\": \"%s\" } }," + - "\"driverlicense\":{\"$numberLong\": %s} } \n", - i + 1, recordNumber - i, getSSN(i), getPhone(i), getEmail(i), - getAddress(i)[2], getAddress(i)[1], getFirstName(i), getLastName(i), - getAge(i), getIncome(i), getBirthdate(i), - getFirstLogin(i), - getSSN(i))); - } - try (InputStream in = new StringBufferInputStream(strBuf.toString()); - DocumentStream stream = Json.newDocumentStream(in)) { - try { - table.insert(stream, "rowid"); // insert a batch of document in stream - } catch(Exception e) { - System.out.println(stream.toString()); - throw e; - } - } - } - table.flush(); - DBTests.waitForIndexFlush(table.getPath(), INDEX_FLUSH_TIMEOUT); - Thread.sleep(200000); - } - } -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/LargeTableGenBase.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/LargeTableGenBase.java deleted file mode 100644 index bbba08f9a04..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/LargeTableGenBase.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests.index; - -import org.apache.commons.lang3.RandomStringUtils; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Random; -import java.util.Set; - -public class LargeTableGenBase { - - private boolean dict_ready = false; - - protected List firstnames; - protected List lastnames; - protected List cities; - protected int[] randomized; - - protected synchronized void initDictionary() { - initDictionaryWithRand(); - } - - protected void initDictionaryWithRand() { - { - firstnames = new ArrayList<>(); - lastnames = new ArrayList<>(); - cities = new ArrayList<>(); - List states = new ArrayList<>(); - - int fnNum = 2000; // 2k - int lnNum = 200000; // 200k - int cityNum = 10000; // 10k - int stateNum = 50; - Random rand = new Random(2017); - int i; - try { - Set strSet = new LinkedHashSet<>(); - while (strSet.size() < stateNum) { - strSet.add(RandomStringUtils.random(2, 0, 0, true, false, null, rand)); - } - states.addAll(strSet); - - strSet = new LinkedHashSet<>(); - while (strSet.size() < cityNum) { - int len = 3 + strSet.size() % 6; - strSet.add(RandomStringUtils.random(len, 0, 0, true, false, null, rand)); - } - - Iterator it = strSet.iterator(); - for (i = 0; i < cityNum; ++i) { - cities.add(new String[]{"10000", states.get(i%stateNum), it.next()}); - } - - strSet = new LinkedHashSet<>(); - while (strSet.size() < fnNum) { - int len = 3 + strSet.size() % 6; - strSet.add(RandomStringUtils.random(len, 0, 0, true, false, null, rand)); - } - firstnames.addAll(strSet); - - strSet = new LinkedHashSet<>(); - while (strSet.size() < lnNum) { - int len = 3 + strSet.size() % 6; - strSet.add(RandomStringUtils.random(len, 0, 0, true, false, null, rand)); - } - lastnames.addAll(strSet); - } catch(Exception e) { - System.out.println("init data got exception"); - e.printStackTrace(); - } - dict_ready = true; - } - } - - protected String getFirstName(int i) { - return firstnames.get((randomized[ i%randomized.length ] + i )% firstnames.size()); - } - - protected String getLastName(int i) { - return lastnames.get((randomized[ (2*i + randomized[i%randomized.length])% randomized.length]) % lastnames.size()); - } - - protected String[] getAddress(int i) { - return cities.get((randomized[(i+ randomized[i%randomized.length])%randomized.length]) % cities.size()); - } - - protected String getSSN(int i){ - return String.format("%d", 1000*1000*100 + randomized[ i % randomized.length]); - } - - protected String getPhone(int i) { - // 80% phones are unique, - return String.format("%d", 6500*1000*1000L + randomized[ (randomized.length - i) %((int) (randomized.length * 0.8)) ]); - } - - protected String getEmail(int i){ - return getFirstName(i) + getLastName(i) + "@" + "gmail.com"; - } - - protected String getAge(int i) { - return String.format("%d",randomized[i%randomized.length] % 60 + 10); - } - - protected String getIncome(int i) {//unit should be $10k - return String.format("%d",randomized[i%randomized.length] % 47 + 1); - } - - // date yyyy-mm-dd - protected String getBirthdate(int i) { - int thisseed = randomized[i%randomized.length]; - return String.format("%d-%02d-%02d", - 2016 - (thisseed % 60 + 10), thisseed % 12 + 1, (thisseed * 31) % 28 + 1 ); - } - - // timestamp, yyyy-mm-dd HH:mm:ss - protected String getFirstLogin(int i) { - int thisseed = randomized[i%randomized.length]; - int nextseed = randomized[(i+1)%randomized.length]; - return String.format("%d-%02d-%02d %02d:%02d:%02d.0", - 2016 - (thisseed % 7), (thisseed * 31) % 12 + 1, thisseed % 28 + 1, nextseed % 24, nextseed % 60, (nextseed * 47) % 60 ); - } - - - protected String getField(String field, int i) { - if(field.equals("ssn")) { - return getSSN(i); - } - else if (field.equals("phone")) { - return getPhone(i); - } - else if(field.equals("email")) { - return getEmail(i); - } - else if(field.equals("city")) { - return getAddress(i)[1]; - } - else if(field.equals("state")) { - return getAddress(i)[0]; - } - else if(field.equals("fname")) { - return getFirstName(i); - } - else if(field.equals("lname")) { - return getLastName(i); - } - return ""; - } - - - protected void initRandVector(int recordNumber) { - int i; - Random rand = new Random(2016); - randomized = new int[recordNumber]; - for(i = 0; i 100)" - + " and (t.address.state = 'mo' or t.address.state = 'ca')"; - PlanTestBase.testPlanMatchingPatterns(explain+query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*rows=10000"} - ); - - // Top-level ORs - Cannot split top-level ORs so use defaults - query = "select * from hbase.`index_test_primary` t " - + " where (t.personal.age > 30 and t.personal.age < 100)" - + " or (t.address.state = 'mo')"; - PlanTestBase.testPlanMatchingPatterns(explain+query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*rows=10000"} - ); - - // ANDed condition - Leading index column(personal.age) and non-leading column(address.city) - query = "select * from hbase.`index_test_primary` t " - + " where (t.personal.age < 30 or t.personal.age > 100)" - + " and `address.city` = 'sf'"; - PlanTestBase.testPlanMatchingPatterns(explain+query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*rows=10000"} - ); - - // ANDed condition - Leading index columns (address.state) and (address.city) - query = "select * from hbase.`index_test_primary` t " - + " where (`address.state` = 'mo' or `address.state` = 'ca') " // Leading index column - + " and `address.city` = 'sf'"; // Non leading index column - PlanTestBase.testPlanMatchingPatterns(explain+query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*rows=10000"} - ); - - // ANDed condition - Leading index columns (address.state) and non-index column (name.fname) - query = "select * from hbase.`index_test_primary` t " - + " where (`address.state` = 'mo' or `address.state` = 'ca') " // Leading index column - + " and `name.fname` = 'VcFahj'"; // Non index column - PlanTestBase.testPlanMatchingPatterns(explain+query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*rows=10000"} - ); - - // Simple condition - LIKE predicate - query = "select t._id as rowid from hbase.`index_test_primary` as t " - + "where t.driverlicense like '100007423%'"; - PlanTestBase.testPlanMatchingPatterns(explain+query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*rows=10000"} - ); - - // Simple condition - LIKE predicate with ESCAPE clause - query = "select t._id as rowid from hbase.`index_test_primary` as t " - + "where t.driverlicense like '100007423%' ESCAPE '/'"; - PlanTestBase.testPlanMatchingPatterns(explain+query, - new String[] {".*JsonTableGroupScan.*tableName=.*index_test_primary.*rows=10000"} - ); - } -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/TableIndexCmd.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/TableIndexCmd.java deleted file mode 100644 index 46701d1882b..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/index/TableIndexCmd.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests.index; - - -import com.mapr.db.Admin; -import com.mapr.db.MapRDB; -import org.apache.drill.common.util.GuavaPatcher; - -import java.util.HashMap; -import java.util.Map; - -/** -* Copy classes to a MapR cluster node, then run a command like this: -* java -classpath /tmp/drill-cmd-1.9.0-SNAPSHOT.jar:/opt/mapr/drill/drill-1.9.0/jars/*:/opt/mapr/drill/drill-1.9.0/jars/3rdparty/*:/opt/mapr/drill/drill-1.9.0/jars/ext/* -* org.apache.drill.hbase.index.TableIndexGen -host 10.10.88.128 -port 5181 [-table pop3] [-size 1000000] -*/ - -class TestBigTable { - - Admin admin; - boolean initialized = false; - - LargeTableGen gen; - - /* - "hbase.zookeeper.quorum": "10.10.88.128", - "hbase.zookeeper.property.clientPort": "5181" - */ - void init(String host, String port) { - try { - admin = MapRDB.newAdmin(); - initialized = true; - gen = new LargeTableGen(admin); - } catch (Exception e) { - System.out.println("Connection to HBase threw" + e.getMessage()); - } - } -} - - -public class TableIndexCmd { - - public static Map parseParameter(String[] params) { - HashMap retParams = new HashMap(); - for (int i=0; i params = parseParameter(args); - if (args.length >= 2) { - if (params.get("host") != null) { - inHost = params.get("host"); - } - if (params.get("port") != null) { - inPort = params.get("port"); - } - if (params.get("table") != null) { - inTable = params.get("table"); - } - if (params.get("size") != null) { - inSize = Long.parseLong(params.get("size")); - } - if (params.get("dict") != null) { - dictPath = params.get("dict"); - } - if (params.get("wait") != null) { - String answer = params.get("wait"); - waitKeyPress = answer.startsWith("y") || answer.startsWith("t")? true : false; - } - } - if (waitKeyPress == true) { - pressEnterKeyToContinue(); - } - try { - TestBigTable tbt = new TestBigTable(); - tbt.init(inHost, inPort); - tbt.gen.generateTableWithIndex(inTable, (int)(inSize & 0xFFFFFFFFL), null); - } catch(Exception e) { - System.out.println("generate big table got exception:" + e.getMessage()); - e.printStackTrace(); - } - } -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java deleted file mode 100644 index 0dd6a4b345e..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/BaseJsonTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests.json; - -import java.util.List; - -import org.apache.drill.test.BaseTestQuery; -import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.rpc.user.QueryDataBatch; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; - -import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; - -public class BaseJsonTest extends BaseTestQuery { - protected static final String SCHEMA = "hbase.root"; - protected String format(final String sql) { - return String.format(sql, SCHEMA, getTablePath()); - } - - protected String getTablePath() { - throw new RuntimeException("unimplemented"); - } - - public static String format(final String sql, final String tablePath) { - return String.format(sql, SCHEMA, tablePath); - } - - @BeforeClass - public static void setupDefaultTestCluster() throws Exception { - // GuavaPatcher.patch is invoked in ExecTest ExecTest-->BaseTestQuery - // GuavaPatcher.patch(); - - // Since we override the class initializer of parent class, - // invoke it explicitly. This will setup a Drill cluster. - BaseTestQuery.setupDefaultTestCluster(); - - MaprDBTestsSuite.setupTests(); - MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext()); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - MaprDBTestsSuite.cleanupTests(); - } - - - protected List runHBaseSQLlWithResults(String sql) throws Exception { - return testSqlWithResults(sql); - } - - protected void runSQLAndVerifyCount(String sql, int expectedRowCount) throws Exception{ - List results = runHBaseSQLlWithResults(sql); - logResultAndVerifyRowCount(results, expectedRowCount); - } - - private void logResultAndVerifyRowCount(List results, int expectedRowCount) throws SchemaChangeException { - int rowCount = logResult(results); - if (expectedRowCount != -1) { - Assert.assertEquals(expectedRowCount, rowCount); - } - } -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestEncodedFieldPaths.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestEncodedFieldPaths.java deleted file mode 100644 index ccad57d918a..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestEncodedFieldPaths.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests.json; - -import static com.mapr.drill.maprdb.tests.MaprDBTestsSuite.INDEX_FLUSH_TIMEOUT; - -import java.io.InputStream; - -import org.apache.drill.PlanTestBase; -import org.apache.drill.exec.util.EncodedSchemaPathSet; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.ojai.DocumentStream; -import org.ojai.json.Json; - -import com.google.common.collect.ImmutableMap; -import com.mapr.db.Table; -import com.mapr.db.tests.utils.DBTests; - -public class TestEncodedFieldPaths extends BaseJsonTest { - - private static final String TABLE_NAME = "encoded_fields_userdata_table"; - private static final String INDEX_NAME = "encoded_fields_userdata_index"; - private static final String JSON_FILE_URL = "/com/mapr/drill/json/encoded_fields_userdata.json"; - - private static boolean tableCreated = false; - private static String tablePath; - - @BeforeClass - public static void setup_TestEncodedFieldPaths() throws Exception { - try (Table table = DBTests.createOrReplaceTable(TABLE_NAME, ImmutableMap.of("codes", "codes"))) { - tableCreated = true; - tablePath = table.getPath().toUri().getPath(); - - DBTests.createIndex(TABLE_NAME, INDEX_NAME, new String[] {"age"}, new String[] {"name.last", "data.salary"}); - DBTests.admin().getTableIndexes(table.getPath(), true); - - try (final InputStream in = TestEncodedFieldPaths.class.getResourceAsStream(JSON_FILE_URL); - final DocumentStream stream = Json.newDocumentStream(in);) { - table.insertOrReplace(stream); - table.flush(); - } - - // wait for the indexes to sync - DBTests.waitForRowCount(table.getPath(), 5, INDEX_FLUSH_TIMEOUT); - DBTests.waitForIndexFlush(table.getPath(), INDEX_FLUSH_TIMEOUT); - } finally { - test("ALTER SESSION SET `planner.disable_full_table_scan` = true"); - } - } - - @AfterClass - public static void cleanup_TestEncodedFieldPaths() throws Exception { - test("ALTER SESSION SET `planner.disable_full_table_scan` = false"); - if (tableCreated) { - DBTests.deleteTables(TABLE_NAME); - } - } - - @Test - public void test_encoded_fields_with_non_covering_index() throws Exception { - final String sql = String.format( - "SELECT\n" - + " t.`%s`,t.`$$document`\n" - + "FROM\n" - + " hbase.root.`%s` t\n" - + "WHERE (t.`age` > 20)\n" - + "ORDER BY t.`_id` ASC", - EncodedSchemaPathSet.encode("_id", "codes")[0], - tablePath); - - setColumnWidths(new int[] {20, 60}); - runSQLAndVerifyCount(sql, 3); - - - // plan test - final String[] expectedPlan = {"JsonTableGroupScan.*indexName=encoded_fields_userdata_index.*" + // scan on index - "columns=\\[`_id`, `age`\\]", - "RestrictedJsonTableGroupScan.*" + // restricted scan on the table with encoded name - "columns=\\[`age`, `\\$\\$ENC00L5UWIADDN5SGK4Y`, `\\$\\$document`, `_id`\\]", - "RowKeyJoin"}; // join on row_key - final String[] excludedPlan = {}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void test_encoded_fields_with_covering_index() throws Exception { - final String sql = String.format( - "SELECT\n" - + " t.`%s`,t.`$$document`\n" - + "FROM\n" - + " hbase.root.`%s` t\n" - + "WHERE (t.`age` > 10)\n" - + "ORDER BY t.`_id` ASC", - EncodedSchemaPathSet.encode("name.last", "data.salary")[0], - tablePath); - - setColumnWidths(new int[] {20, 60}); - runSQLAndVerifyCount(sql, 4); - - - // plan test - final String[] expectedPlan = {"JsonTableGroupScan.*indexName=encoded_fields_userdata_index.*", // scan on index - "columns=\\[`age`, `\\$\\$ENC00NZQW2ZJONRQXG5AAMRQXIYJOONQWYYLSPE`, `\\$\\$document`, `_id`\\]"}; - final String[] excludedPlan = {"RestrictedJsonTableGroupScan", // restricted scan on the table - "RowKeyJoin"}; // join on row_key - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestFieldPathHelper.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestFieldPathHelper.java deleted file mode 100644 index 12872979984..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestFieldPathHelper.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests.json; - -import static org.junit.Assert.assertEquals; - -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.exec.store.mapr.db.json.FieldPathHelper; -import org.apache.drill.test.BaseTest; -import org.junit.Test; -import org.ojai.FieldPath; - -public class TestFieldPathHelper extends BaseTest { - - @Test - public void simeTests() { - String[] pathStrs = {"a", "a.b", "a.b.c", "a[1].b[2].c", "a[0][1][2][3].b"}; - FieldPath[] fieldPaths = new FieldPath[pathStrs.length]; - SchemaPath[] schemaPaths = new SchemaPath[pathStrs.length]; - - // build - for (int i = 0; i < pathStrs.length; i++) { - String path = pathStrs[i]; - fieldPaths[i] = FieldPath.parseFrom(path); - schemaPaths[i] = SchemaPath.parseFromString(path); - } - - //verify - for (int i = 0; i < pathStrs.length; i++) { - FieldPath fp = FieldPathHelper.schemaPath2FieldPath(schemaPaths[i]); - assertEquals(fieldPaths[i], fp); - - SchemaPath sp = FieldPathHelper.fieldPath2SchemaPath(fieldPaths[i]); - assertEquals(schemaPaths[i], sp); - } - } - -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestScanRanges.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestScanRanges.java deleted file mode 100644 index 7ad44d1d8c8..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestScanRanges.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests.json; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.io.InputStream; -import java.util.List; - -import org.apache.drill.exec.proto.UserBitShared; -import org.apache.drill.exec.proto.UserBitShared.QueryProfile; -import org.apache.drill.exec.proto.UserBitShared.QueryType; -import org.apache.drill.exec.proto.helper.QueryIdHelper; -import org.apache.drill.exec.rpc.user.AwaitableUserResultsListener; -import org.apache.drill.exec.store.sys.PersistentStore; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.ojai.Document; -import org.ojai.DocumentStream; -import org.ojai.json.Json; - -import com.google.common.collect.Lists; -import com.mapr.db.Table; -import com.mapr.db.tests.utils.DBTests; -import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; -import com.mapr.tests.annotations.ClusterTest; - -@Category(ClusterTest.class) -public class TestScanRanges extends BaseJsonTest { - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestScanRanges.class); - - private static final int TOTAL_ROW_COUNT = 1000000; - private static final String TABLE_NAME = "large_table_TestScanRanges"; - private static final String JSON_FILE_URL = "/com/mapr/drill/json/business.json"; - - private static boolean tableCreated = false; - private static String tablePath; - protected String getTablePath() { - return tablePath; - } - - @BeforeClass - public static void setup_TestSimpleJson() throws Exception { - // We create a large table with auto-split set to disabled. - // Without intra-tablet partitioning, this test should run with only one minor fragment - try (Table table = DBTests.createOrReplaceTable(TABLE_NAME, false /*autoSplit*/); - InputStream in = MaprDBTestsSuite.getJsonStream(JSON_FILE_URL); - DocumentStream stream = Json.newDocumentStream(in)) { - tableCreated = true; - tablePath = table.getPath().toUri().getPath(); - - List docs = Lists.newArrayList(stream); - for (char ch = 'A'; ch <= 'T'; ch++) { - for (int rowIndex = 0; rowIndex < 5000; rowIndex++) { - for (int i = 0; i < docs.size(); i++) { - final Document document = docs.get(i); - final String id = String.format("%c%010d%03d", ch, rowIndex, i); - document.set("documentId", rowIndex); - table.insertOrReplace(id, document); - } - } - } - table.flush(); - DBTests.waitForRowCount(table.getPath(), TOTAL_ROW_COUNT); - - setSessionOption("planner.width.max_per_node", 5); - } - } - - @AfterClass - public static void cleanup_TestEncodedFieldPaths() throws Exception { - if (tableCreated) { - DBTests.deleteTables(TABLE_NAME); - } - } - - @Test - public void test_scan_ranges() throws Exception { - final PersistentStore completed = getDrillbitContext().getProfileStoreContext().getCompletedProfileStore(); - - setColumnWidths(new int[] {25, 40, 25, 45}); - final String sql = format("SELECT\n" - + " *\n" - + "FROM\n" - + " %s.`%s` business"); - - final SilentListener resultListener = new SilentListener(); - final AwaitableUserResultsListener listener = new AwaitableUserResultsListener(resultListener); - testWithListener(QueryType.SQL, sql, listener); - listener.await(); - - assertEquals(TOTAL_ROW_COUNT, resultListener.getRowCount()); - String queryId = QueryIdHelper.getQueryId(resultListener.getQueryId()); - - QueryProfile profile = completed.get(queryId); - String profileString = String.valueOf(profile); - logger.debug(profileString); - assertNotNull(profile); - assertTrue(profile.getTotalFragments() >= 5); // should at least as many as - } - - @Test - public void test_scan_ranges_with_filter_on_id() throws Exception { - setColumnWidths(new int[] {25, 25, 25}); - final String sql = format("SELECT\n" - + " _id, business_id, city\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " _id > 'M' AND _id < 'Q'"); - - final SilentListener resultListener = new SilentListener(); - final AwaitableUserResultsListener listener = new AwaitableUserResultsListener(resultListener); - testWithListener(QueryType.SQL, sql, listener); - listener.await(); - - assertEquals(200000, resultListener.getRowCount()); - } - - @Test - public void test_scan_ranges_with_filter_on_non_id_field() throws Exception { - setColumnWidths(new int[] {25, 25, 25}); - final String sql = format("SELECT\n" - + " _id, business_id, documentId\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " documentId >= 100 AND documentId < 150"); - - final SilentListener resultListener = new SilentListener(); - final AwaitableUserResultsListener listener = new AwaitableUserResultsListener(resultListener); - testWithListener(QueryType.SQL, sql, listener); - listener.await(); - - assertEquals(10000, resultListener.getRowCount()); - } - -} diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java deleted file mode 100644 index a5be2ab5e18..00000000000 --- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java +++ /dev/null @@ -1,485 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.mapr.drill.maprdb.tests.json; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -import java.io.InputStream; - -import org.apache.drill.PlanTestBase; -import org.apache.drill.SingleRowListener; -import org.apache.drill.exec.proto.UserBitShared.QueryType; -import org.apache.drill.exec.record.RecordBatchLoader; -import org.apache.drill.exec.rpc.user.QueryDataBatch; -import org.apache.drill.exec.util.VectorUtil; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.ojai.Document; -import org.ojai.DocumentStream; -import org.ojai.json.Json; - -import com.mapr.db.Table; -import com.mapr.db.impl.MapRDBImpl; -import com.mapr.db.tests.utils.DBTests; -import com.mapr.drill.maprdb.tests.MaprDBTestsSuite; -import com.mapr.tests.annotations.ClusterTest; - -@Category(ClusterTest.class) -public class TestSimpleJson extends BaseJsonTest { - - private static final String TABLE_NAME = "business"; - private static final String JSON_FILE_URL = "/com/mapr/drill/json/business.json"; - - private static boolean tableCreated = false; - private static String tablePath; - @Override - protected String getTablePath() { - return tablePath; - } - - @BeforeClass - public static void setup_TestSimpleJson() throws Exception { - try (Table table = DBTests.createOrReplaceTable(TABLE_NAME); - InputStream in = MaprDBTestsSuite.getJsonStream(JSON_FILE_URL); - DocumentStream stream = Json.newDocumentStream(in)) { - tableCreated = true; - tablePath = table.getPath().toUri().getPath(); - - for (Document document : stream) { - table.insert(document, "business_id"); - } - table.flush(); - } - } - - @AfterClass - public static void cleanup_TestEncodedFieldPaths() throws Exception { - if (tableCreated) { - DBTests.deleteTables(TABLE_NAME); - } - } - - @Test - public void testSelectStar() throws Exception { - final String sql = format("SELECT\n" - + " *\n" - + "FROM\n" - + " %s.`%s` business"); - runSQLAndVerifyCount(sql, 10); - } - - @Test - public void testSelectId() throws Exception { - setColumnWidths(new int[] {23}); - final String sql = format("SELECT\n" - + " _id\n" - + "FROM\n" - + " %s.`%s` business"); - runSQLAndVerifyCount(sql, 10); - } - - @Test - public void testSelectNonExistentColumns() throws Exception { - setColumnWidths(new int[] {23}); - final String sql = format("SELECT\n" - + " something\n" - + "FROM\n" - + " %s.`%s` business limit 5"); - runSQLAndVerifyCount(sql, 5); - } - - @Test - public void testKVGen() throws Exception { - setColumnWidths(new int[] {21, 10, 6}); - final String sql = format("select _id, t.parking[0].`key` K, t.parking[0].`value` V from" - + " (select _id, kvgen(b.attributes.Parking) as parking from %s.`%s` b)" - + " as t where t.parking[0].`key` = 'garage' AND t.parking[0].`value` = true"); - runSQLAndVerifyCount(sql, 1); - } - - @Test - public void testPushdownDisabled() throws Exception { - setColumnWidths(new int[] {25, 40, 40, 40}); - final String sql = format("SELECT\n" - + " _id, name, categories, full_address\n" - + "FROM\n" - + " table(%s.`%s`(type => 'maprdb', enablePushdown => false)) business\n" - + "WHERE\n" - + " name <> 'Sprint'"); - runSQLAndVerifyCount(sql, 9); - - final String[] expectedPlan = {"condition=null", "columns=\\[`\\*`\\]"}; - final String[] excludedPlan = {"condition=\\(name != \"Sprint\"\\)", "columns=\\[`name`, `_id`, `categories`, `full_address`\\]"}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testPushdownStringEqual() throws Exception { - setColumnWidths(new int[] {25, 40, 40, 40}); - final String sql = format("SELECT\n" - + " _id, name, business.hours.Monday.`open`, categories[1], years[2], full_address\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " name = 'Sprint'"); - - final Document queryResult = MapRDBImpl.newDocument(); - SingleRowListener listener = new SingleRowListener() { - @Override - protected void rowArrived(QueryDataBatch result) { - final RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); - loader.load(result.getHeader().getDef(), result.getData()); - StringBuilder sb = new StringBuilder(); - VectorUtil.appendVectorAccessibleContent(loader, sb, "|", false); - loader.clear(); - queryResult.set("result", sb.toString()); - } - }; - testWithListener(QueryType.SQL, sql, listener); - listener.waitForCompletion(); - - assertNull(queryResult.getString("error")); - assertNotNull(queryResult.getString("result")); - - String[] fields = queryResult.getString("result").split("\\|"); - assertEquals("1970-01-01T11:00:00.000", fields[2]); - assertEquals("Mobile Phones", fields[3]); - assertEquals("2016.0", fields[4]); - - final String[] expectedPlan = {"condition=\\(name = \"Sprint\"\\)"}; - final String[] excludedPlan = {}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testPushdownStringLike() throws Exception { - setColumnWidths(new int[] {25, 40, 40, 40}); - final String sql = format("SELECT\n" - + " _id, name, categories, full_address\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " name LIKE 'S%%'"); - runSQLAndVerifyCount(sql, 3); - - final String[] expectedPlan = {"condition=\\(name MATCHES \"\\^\\\\\\\\QS\\\\\\\\E\\.\\*\\$\"\\)"}; - final String[] excludedPlan = {}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testPushdownStringNotEqual() throws Exception { - setColumnWidths(new int[] {25, 40, 40, 40}); - final String sql = format("SELECT\n" - + " _id, name, categories, full_address\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " name <> 'Sprint'"); - runSQLAndVerifyCount(sql, 9); - - final String[] expectedPlan = {"condition=\\(name != \"Sprint\"\\)", "columns=\\[`name`, `_id`, `categories`, `full_address`\\]"}; - final String[] excludedPlan = {}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testPushdownLongEqual() throws Exception { - setColumnWidths(new int[] {25, 40, 40, 40}); - final String sql = format("SELECT\n" - + " _id, name, categories, full_address\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " zip = 85260"); - runSQLAndVerifyCount(sql, 1); - - final String[] expectedPlan = {"condition=\\(zip = \\{\"\\$numberLong\":85260\\}\\)"}; - final String[] excludedPlan = {}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testCompositePredicate() throws Exception { - setColumnWidths(new int[] {25, 40, 40, 40}); - final String sql = format("SELECT\n" - + " _id, name, categories, full_address\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " zip = 85260\n" - + " OR\n" - + " city = 'Las Vegas'"); - runSQLAndVerifyCount(sql, 4); - - final String[] expectedPlan = {"condition=\\(\\(zip = \\{\"\\$numberLong\":85260\\}\\) or \\(city = \"Las Vegas\"\\)\\)"}; - final String[] excludedPlan = {}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testPruneScanRange() throws Exception { - setColumnWidths(new int[] {25, 40, 40, 40}); - final String sql = format("SELECT\n" - + " _id, name, categories, full_address\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " _id = 'jFTZmywe7StuZ2hEjxyA'"); - runSQLAndVerifyCount(sql, 1); - - final String[] expectedPlan = {"condition=\\(_id = \"jFTZmywe7StuZ2hEjxyA\"\\)"}; - final String[] excludedPlan ={}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - @Ignore("Bug 27981") - public void testPruneScanRangeAndPushDownCondition() throws Exception { - // XXX/TODO: - setColumnWidths(new int[] {25, 40, 40, 40}); - final String sql = format("SELECT\n" - + " _id, name, categories, full_address\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " _id = 'jFTZmywe7StuZ2hEjxyA' AND\n" - + " name = 'Subway'"); - runSQLAndVerifyCount(sql, 1); - - final String[] expectedPlan = {"condition=\\(\\(_id = \"jFTZmywe7StuZ2hEjxyA\"\\) and \\(name = \"Subway\"\\)\\)"}; - final String[] excludedPlan ={}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testPushDownOnSubField1() throws Exception { - setColumnWidths(new int[] {25, 120, 20}); - final String sql = format("SELECT\n" - + " _id, name, b.attributes.Ambience.touristy attributes\n" - + "FROM\n" - + " %s.`%s` b\n" - + "WHERE\n" - + " b.attributes.Ambience.casual = false"); - runSQLAndVerifyCount(sql, 1); - - final String[] expectedPlan = {"condition=\\(attributes.Ambience.casual = false\\)"}; - final String[] excludedPlan ={}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testPushDownOnSubField2() throws Exception { - setColumnWidths(new int[] {25, 40, 40, 40}); - final String sql = format("SELECT\n" - + " _id, name, b.attributes.Attire attributes\n" - + "FROM\n" - + " %s.`%s` b\n" - + "WHERE\n" - + " b.attributes.Attire = 'casual'"); - runSQLAndVerifyCount(sql, 4); - - final String[] expectedPlan = {"condition=\\(attributes.Attire = \"casual\"\\)"}; - final String[] excludedPlan ={}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - @Test - public void testPushDownIsNull() throws Exception { - setColumnWidths(new int[] {25, 40, 40, 40}); - - final String sql = format("SELECT\n" - + " _id, name, attributes\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " business.attributes.Ambience.casual IS NULL"); - runSQLAndVerifyCount(sql, 7); - - final String[] expectedPlan = {"condition=\\(\\(attributes.Ambience.casual = null\\) or \\(TYPE_OF\\(attributes.Ambience.casual\\) = NULL\\)\\)"}; - final String[] excludedPlan ={}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testPushDownIsNotNull() throws Exception { - setColumnWidths(new int[] {25, 75, 75, 50}); - - final String sql = format("SELECT\n" - + " _id, name, b.attributes.Parking\n" - + "FROM\n" - + " %s.`%s` b\n" - + "WHERE\n" - + " b.attributes.Ambience.casual IS NOT NULL"); - runSQLAndVerifyCount(sql, 3); - - final String[] expectedPlan = {"condition=\\(\\(attributes.Ambience.casual != null\\) and \\(TYPE_OF\\(attributes.Ambience.casual\\) != NULL\\)\\)"}; - final String[] excludedPlan ={}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testPushDownOnSubField3() throws Exception { - setColumnWidths(new int[] {25, 40, 40, 40}); - final String sql = format("SELECT\n" - + " _id, name, b.attributes.`Accepts Credit Cards` attributes\n" - + "FROM\n" - + " %s.`%s` b\n" - + "WHERE\n" - + " b.attributes.`Accepts Credit Cards` IS NULL"); - runSQLAndVerifyCount(sql, 3); - - final String[] expectedPlan = {"condition=\\(\\(attributes.Accepts Credit Cards = null\\) or \\(TYPE_OF\\(attributes.Accepts Credit Cards\\) = NULL\\)\\)"}; - final String[] excludedPlan ={}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testPushDownLong() throws Exception { - final String sql = format("SELECT\n" - + " *\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " stars > 4.0"); - runSQLAndVerifyCount(sql, 2); - - final String[] expectedPlan = {"condition=\\(stars > 4\\)"}; - final String[] excludedPlan ={}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testPushDownSubField4() throws Exception { - final String sql = format("SELECT\n" - + " *\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " business.attributes.`Good For`.lunch = true AND" - + " stars > 4.1"); - runSQLAndVerifyCount(sql, 1); - - final String[] expectedPlan = {"condition=\\(\\(attributes.Good For.lunch = true\\) and \\(stars > 4.1\\)\\)"}; - final String[] excludedPlan ={}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - - @Test - public void testPushDownSubField5() throws Exception { - final String sql = format("SELECT\n" - + " *\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " business.hours.Tuesday.`open` < TIME '10:30:00'"); - runSQLAndVerifyCount(sql, 1); - - final String[] expectedPlan = {"condition=\\(hours.Tuesday.open < \\{\"\\$time\":\"10:30:00\"\\}\\)"}; - final String[] excludedPlan = {}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testPushDownSubField6() throws Exception { - final String sql = format("SELECT\n" - + " *\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " business.hours.Sunday.`close` > TIME '20:30:00'"); - runSQLAndVerifyCount(sql, 3); - - final String[] expectedPlan = {"condition=\\(hours.Sunday.close > \\{\"\\$time\":\"20:30:00\"\\}\\)"}; - final String[] excludedPlan = {}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testPushDownSubField7() throws Exception { - setColumnWidths(new int[] {25, 40, 25, 45}); - final String sql = format("SELECT\n" - + " _id, name, start_date, last_update\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " business.`start_date` = DATE '2012-07-14'"); - runSQLAndVerifyCount(sql, 1); - - final String[] expectedPlan = {"condition=\\(start_date = \\{\"\\$dateDay\":\"2012-07-14\"\\}\\)"}; - final String[] excludedPlan = {}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testPushDownSubField8() throws Exception { - setColumnWidths(new int[] {25, 40, 25, 45}); - final String sql = format("SELECT\n" - + " _id, name, start_date, last_update\n" - + "FROM\n" - + " %s.`%s` business\n" - + "WHERE\n" - + " business.`last_update` = TIMESTAMP '2012-10-20 07:42:46'"); - runSQLAndVerifyCount(sql, 1); - - final String[] expectedPlan = {"condition=\\(last_update = \\{\"\\$date\":\"2012-10-20T07:42:46.000Z\"\\}\\)"}; - final String[] excludedPlan = {}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - } - - @Test - public void testLimit() throws Exception { - final String sql = format("SELECT\n" - + " _id, name, start_date, last_update\n" - + "FROM\n" - + " %s.`%s` business\n" - + "limit 1" - ); - - final String[] expectedPlan = {"JsonTableGroupScan.*limit=1"}; - final String[] excludedPlan = {}; - - PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan); - runSQLAndVerifyCount(sql, 1); - } - -} diff --git a/contrib/format-maprdb/src/test/resources/com/mapr/drill/json/business.json b/contrib/format-maprdb/src/test/resources/com/mapr/drill/json/business.json deleted file mode 100644 index ab1326fef64..00000000000 --- a/contrib/format-maprdb/src/test/resources/com/mapr/drill/json/business.json +++ /dev/null @@ -1,10 +0,0 @@ -{"_version":{"$numberLong":0},"business_id":"1emggGHgoG6ipd_RMb-g","full_address":"3280 S Decatur Blvd\nWestside\nLas Vegas, NV 89102","zip":{"$numberLong":89102},"hours":{},"open":true,"categories":["Food","Convenience Stores"],"city":"Las Vegas","review_count":4,"name":"Sinclair","neighborhoods":["Westside"],"longitude":-115.2072382,"state":"NV","stars":4,"latitude":36.1305306,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Accepts Credit Cards":true,"Price Range":1},"type":"business", "start_date":{"$dateDay":"2011-07-14"}, "last_update":{"$date":"2012-10-20T07:42:46.000Z"}} -{"_version":{"$numberLong":0},"business_id":"4Pe8BZ6gj57VFL5mUE8g","full_address":"21001 North Tatum Blvd. #24\nPhoenix, AZ 85050","zip":{"$numberLong":85050},"hours":{},"open":true,"categories":["Shopping","Office Equipment"],"city":"Phoenix","review_count":5,"name":"Office Max","neighborhoods":[],"longitude":-111.9746066,"state":"AZ","stars":3,"latitude":33.678615,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business", "start_date":{"$dateDay":"2012-07-14"}} -{"_version":{"$numberLong":0},"business_id":"5jkZ3-nUPZxUvtcbr8Uw","full_address":"1336 N Scottsdale Rd\nScottsdale, AZ 85257","zip":{"$numberLong":85257},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Greek","Restaurants"],"city":"Scottsdale","review_count":42,"name":"Mika's Greek","neighborhoods":[],"longitude":-111.926908493042,"state":"AZ","stars":4.5,"latitude":33.4633733188117,"attributes":{"Take-out":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"quiet","Takes Reservations":false,"Delivery":false,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Has TV":false,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":false,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":{"$dateDay":"2013-07-14"}} -{"_version":{"$numberLong":0},"business_id":"BlvDO_RG2yElKu9XA1_g","full_address":"14870 N Northsight Blvd\nSte 103\nScottsdale, AZ 85260","zip":{"$numberLong":85260},"hours":{"Monday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Tuesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Friday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Wednesday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Thursday":{"close":{"$time":"21:00:00"},"open":{"$time":"10:30:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"12:00:00"}}},"open":true,"categories":["Sushi Bars","Hawaiian","Chinese","Restaurants"],"city":"Scottsdale","review_count":65,"name":"Asian Island","neighborhoods":[],"longitude":-111.89783602953,"state":"AZ","stars":4,"latitude":33.6205679923296,"attributes":{"Take-out":true,"Wi-Fi":"free","Good For":{"dessert":false,"latenight":false,"lunch":true,"dinner":false,"breakfast":false,"brunch":false},"Caters":true,"Noise Level":"average","Takes Reservations":false,"Has TV":false,"Delivery":true,"Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":true},"Parking":{"garage":false,"street":false,"validated":false,"lot":true,"valet":false},"Wheelchair Accessible":true,"Outdoor Seating":true,"Attire":"casual","Alcohol":"none","Waiter Service":true,"Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":{"$dateDay":"2014-07-14"}} -{"_version":{"$numberLong":0},"business_id":"Dl2rW_xO8GuYBomlg9zw","full_address":"4505 S Maryland Pkwy\nUniversity\nLas Vegas, NV 89119","zip":{"$numberLong":89119},"hours":{},"open":true,"categories":["Medical Centers","Health & Medical"],"city":"Las Vegas","review_count":6,"name":"UNLV Student Health Center","neighborhoods":["University"],"longitude":-115.1415145,"state":"NV","stars":4,"latitude":36.1109405,"attributes":{"By Appointment Only":true},"type":"business", "start_date":{"$dateDay":"2011-04-14"}} -{"_version":{"$numberLong":0},"business_id":"Ol5mVSMaW8ExtmWRUmKA","full_address":"7110 E Thomas Rd\nSte D\nScottsdale, AZ 85251","zip":{"$numberLong":85251},"hours":{},"open":true,"categories":["Barbers","Beauty & Spas"],"city":"Scottsdale","review_count":3,"name":"Dave's Barber Shop","neighborhoods":[],"longitude":-111.9289668,"state":"AZ","stars":5,"latitude":33.48051,"attributes":{"By Appointment Only":false,"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Price Range":2},"type":"business", "start_date":{"$dateDay":"2013-02-15"}} -{"_version":{"$numberLong":0},"business_id":"XBxRlD92RaV6TyUnP8Ow","full_address":"7510 W Thomas Rd Ste 108\nPhoenix, AZ 85033","zip":{"$numberLong":85033},"hours":{"Monday":{"close":{"$time":"19:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Friday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Wednesday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Thursday":{"close":{"$time":"20:00:00"},"open":{"$time":"09:00:00"}},"Sunday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}},"Saturday":{"close":{"$time":"21:00:00"},"open":{"$time":"09:00:00"}}},"open":true,"categories":["Shopping","Mobile Phones"],"city":"Phoenix","review_count":3,"name":"Sprint","neighborhoods":[],"longitude":-112.221054,"state":"AZ","stars":3.5,"latitude":33.480679,"attributes":{},"type":"business", "start_date":{"$dateDay":"2013-01-21"}, "years":[2014,2015,2016]} -{"_version":{"$numberLong":0},"business_id":"Y_2lDOtVDioX5bwF6GIw","full_address":"115 State St\nCapitol\nMadison, WI 53703","zip":{"$numberLong":53703},"hours":{"Monday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Tuesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Friday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Wednesday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Thursday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}},"Sunday":{"close":{"$time":"14:00:00"},"open":{"$time":"11:00:00"}},"Saturday":{"close":{"$time":"02:00:00"},"open":{"$time":"11:00:00"}}},"open":true,"categories":["Bars","Comfort Food","Nightlife","Restaurants"],"city":"Madison","review_count":21,"name":"Buck & Badger","neighborhoods":["Capitol"],"longitude":-89.3871119284652,"state":"WI","stars":3,"latitude":43.0747392865267,"attributes":{"Alcohol":"full_bar","Noise Level":"average","Has TV":true,"Attire":"casual","Ambience":{"romantic":false,"intimate":false,"touristy":false,"hipster":false,"divey":false,"classy":false,"trendy":false,"upscale":false,"casual":false},"Good for Kids":true,"Price Range":2,"Good For Dancing":false,"Delivery":false,"Coat Check":false,"Smoking":"no","Accepts Credit Cards":true,"Take-out":true,"Happy Hour":true,"Outdoor Seating":true,"Takes Reservations":true,"Waiter Service":true,"Wi-Fi":"no","Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Parking":{"garage":true,"street":false,"validated":false,"lot":false,"valet":false},"Music":{"dj":false,"background_music":true,"jukebox":false,"live":false,"video":false,"karaoke":false},"Good For Groups":true},"type":"business", "start_date":{"$dateDay":"2015-03-21"}} -{"_version":{"$numberLong":0},"business_id":"jFTZmywe7StuZ2hEjxyA","full_address":"3991 Dean Martin Dr\nLas Vegas, NV 89103","zip":{"$numberLong":89103},"hours":{},"open":true,"categories":["Sandwiches","Restaurants"],"city":"Las Vegas","review_count":4,"name":"Subway","neighborhoods":[],"longitude":-115.18200516700699,"state":"NV","stars":4,"latitude":36.1188189268328,"attributes":{"Take-out":true,"Good For":{"dessert":false,"latenight":false,"lunch":false,"dinner":false,"brunch":false,"breakfast":false},"Takes Reservations":false,"Delivery":false,"Outdoor Seating":false,"Attire":"casual","Accepts Credit Cards":true,"Good for Kids":true,"Good For Groups":true,"Price Range":1},"type":"business", "start_date":{"$dateDay":"2014-02-13"}} -{"_version":{"$numberLong":0},"business_id":"m1g9P1wxNblrLANfVqlA","full_address":"6 Waterloo Place\nEdinburgh EH1 3EG","hours":{},"open":true,"categories":["Bridal","Shopping"],"city":"Edinburgh","review_count":5,"name":"Caroline Castigliano","neighborhoods":[],"longitude":-3.1881974,"state":"EDH","stars":4,"latitude":55.9534049,"attributes":{"Parking":{"garage":false,"street":false,"validated":false,"lot":false,"valet":false},"Accepts Credit Cards":true,"Price Range":3},"type":"business", "start_date":{"$dateDay":"2014-02-17"}} diff --git a/contrib/format-maprdb/src/test/resources/com/mapr/drill/json/encoded_fields_userdata.json b/contrib/format-maprdb/src/test/resources/com/mapr/drill/json/encoded_fields_userdata.json deleted file mode 100644 index 3295ba5daae..00000000000 --- a/contrib/format-maprdb/src/test/resources/com/mapr/drill/json/encoded_fields_userdata.json +++ /dev/null @@ -1,5 +0,0 @@ -{"_id":"user001", "age":43, "name": {"first":"Sam", "last":"Harris"}, "codes": [1, "x@#ss2", 9.0], "data": {"salary": {"$numberLong": 125000}}} -{"_id":"user002", "age":12, "name": {"first":"Leon", "last":"Russel"}, "codes": ["JA32S"], "data": {"salary": "170200"}} -{"_id":"user003", "age":"87", "name": {"first":"David", "last":"Bowie"}, "codes": [236.35], "data": {"salary": {"$numberLong": 185200}}} -{"_id":"user004", "age":56, "name": {"first":"Bob", "last":"Dylan"}, "codes": [{"$date": "1989-07-4"}, 0], "data": {"salary": {"$numberLong": 136900}}} -{"_id":"user005", "age":54, "name": {"first":"David", "last":"Ackert"}, "codes": [{"a": 25, "b": "yas"}], "data": {"salary": {"$numberLong": 112850}}} diff --git a/contrib/format-maprdb/src/test/resources/core-site.xml b/contrib/format-maprdb/src/test/resources/core-site.xml deleted file mode 100644 index 7ea9704f2c3..00000000000 --- a/contrib/format-maprdb/src/test/resources/core-site.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - hbase.table.namespace.mappings - *:/tmp/ - - - - fs.mapr.bailout.on.library.mismatch - false - - - diff --git a/contrib/storage-hive/core/scrMapr/main/java/org/apache/drill/exec/planner/sql/logical/ConvertHiveMapRDBJsonScanToDrillMapRDBJsonScan.java b/contrib/storage-hive/core/scrMapr/main/java/org/apache/drill/exec/planner/sql/logical/ConvertHiveMapRDBJsonScanToDrillMapRDBJsonScan.java deleted file mode 100644 index daa6863d7f5..00000000000 --- a/contrib/storage-hive/core/scrMapr/main/java/org/apache/drill/exec/planner/sql/logical/ConvertHiveMapRDBJsonScanToDrillMapRDBJsonScan.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.planner.sql.logical; - -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.exec.ExecConstants; -import org.apache.drill.exec.planner.index.MapRDBStatistics; -import org.apache.drill.exec.planner.logical.DrillScanRel; -import org.apache.drill.exec.planner.logical.RelOptHelper; -import org.apache.drill.exec.planner.physical.PlannerSettings; -import org.apache.drill.exec.planner.physical.PrelUtil; -import org.apache.drill.exec.planner.types.HiveToRelDataTypeConverter; -import org.apache.drill.exec.record.metadata.TupleMetadata; -import org.apache.drill.exec.record.metadata.TupleSchema; -import org.apache.drill.exec.store.StoragePluginOptimizerRule; -import org.apache.drill.exec.store.hive.HiveMetadataProvider; -import org.apache.drill.exec.store.hive.HiveReadEntry; -import org.apache.drill.exec.store.hive.HiveScan; -import org.apache.drill.exec.store.hive.HiveUtilities; -import org.apache.drill.exec.store.mapr.db.MapRDBFormatPlugin; -import org.apache.drill.exec.store.mapr.db.MapRDBFormatPluginConfig; -import org.apache.drill.exec.store.mapr.db.json.JsonScanSpec; -import org.apache.drill.exec.store.mapr.db.json.JsonTableGroupScan; -import org.apache.drill.exec.metastore.store.FileSystemMetadataProviderManager; -import org.apache.hadoop.hive.maprdb.json.input.HiveMapRDBJsonInputFormat; -import org.ojai.DocumentConstants; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Convert Hive scan to use Drill's native MapR-DB reader instead of Hive's MapR-DB JSON Handler. - */ -public class ConvertHiveMapRDBJsonScanToDrillMapRDBJsonScan extends StoragePluginOptimizerRule { - private static final org.slf4j.Logger logger = - org.slf4j.LoggerFactory.getLogger(ConvertHiveMapRDBJsonScanToDrillMapRDBJsonScan.class); - - public static final ConvertHiveMapRDBJsonScanToDrillMapRDBJsonScan INSTANCE = - new ConvertHiveMapRDBJsonScanToDrillMapRDBJsonScan(); - - /** - * The constants from org.apache.hadoop.hive.maprdb.json.conf.MapRDBConstants - */ - private static final String MAPRDB_PFX = "maprdb."; - private static final String MAPRDB_TABLE_NAME = MAPRDB_PFX + "table.name"; - private static final String ID_KEY = DocumentConstants.ID_KEY; - private static final String MAPRDB_COLUMN_ID = MAPRDB_PFX + "column.id"; - - private ConvertHiveMapRDBJsonScanToDrillMapRDBJsonScan() { - super(RelOptHelper.any(DrillScanRel.class), "ConvertHiveScanToHiveDrillNativeScan:MapR-DB"); - } - - /** - * {@see org.apache.drill.exec.store.hive.HiveUtilities#nativeReadersRuleMatches} - */ - @Override - public boolean matches(RelOptRuleCall call) { - return HiveUtilities.nativeReadersRuleMatches(call, HiveMapRDBJsonInputFormat.class); - } - - @Override - public void onMatch(RelOptRuleCall call) { - try { - DrillScanRel hiveScanRel = call.rel(0); - PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner()); - - HiveScan hiveScan = (HiveScan) hiveScanRel.getGroupScan(); - HiveReadEntry hiveReadEntry = hiveScan.getHiveReadEntry(); - HiveMetadataProvider hiveMetadataProvider = new HiveMetadataProvider(hiveScan.getUserName(), hiveReadEntry, - hiveScan.getHiveConf()); - if (hiveMetadataProvider.getInputSplits(hiveReadEntry).isEmpty()) { - // table is empty, use original scan - return; - } - - if (hiveScan.getHiveReadEntry().getTable().isSetPartitionKeys()) { - logger.warn("Hive MapR-DB JSON Handler doesn't support table partitioning. Consider recreating table without " + - "partitions"); - } - - DrillScanRel nativeScanRel = createNativeScanRel(hiveScanRel, settings); - call.transformTo(nativeScanRel); - - /* - Drill native scan should take precedence over Hive since it's more efficient and faster. - Hive does not always give correct costing (i.e. for external tables Hive does not have number of rows - and we calculate them approximately). On the contrary, Drill calculates number of rows exactly - and thus Hive Scan can be chosen instead of Drill native scan because costings allegedly lower for Hive. - To ensure Drill MapR-DB Json scan will be chosen, reduce Hive scan importance to 0. - */ - call.getPlanner().setImportance(hiveScanRel, 0.0); - } catch (Exception e) { - // TODO: Improve error handling after allowing to throw IOException from StoragePlugin.getFormatPlugin() - logger.warn("Failed to convert HiveScan to JsonScanSpec. Fallback to HiveMapR-DB connector.", e); - } - } - - /** - * Helper method which creates a DrillScanRel with native Drill HiveScan. - */ - private DrillScanRel createNativeScanRel(DrillScanRel hiveScanRel, PlannerSettings settings) throws IOException { - RelDataTypeFactory typeFactory = hiveScanRel.getCluster().getTypeFactory(); - HiveScan hiveScan = (HiveScan) hiveScanRel.getGroupScan(); - HiveReadEntry hiveReadEntry = hiveScan.getHiveReadEntry(); - Map parameters = hiveReadEntry.getHiveTableWrapper().getParameters(); - - JsonScanSpec scanSpec = new JsonScanSpec(parameters.get(MAPRDB_TABLE_NAME), null, null); - List hiveScanCols = hiveScanRel.getColumns().stream() - .map(colNameSchemaPath -> replaceOverriddenSchemaPath(parameters, colNameSchemaPath)) - .collect(Collectors.toList()); - - // creates TupleMetadata based on Hive's schema (with optional data modes) to be used in the reader - // for the case when column type wasn't discovered - HiveToRelDataTypeConverter dataTypeConverter = new HiveToRelDataTypeConverter(typeFactory); - TupleMetadata schema = new TupleSchema(); - hiveReadEntry.getTable().getColumnListsCache().getTableSchemaColumns() - .forEach(column -> - schema.addColumn(HiveUtilities.getColumnMetadata( - replaceOverriddenColumnId(parameters, column.getName()), - dataTypeConverter.convertToNullableRelDataType(column)))); - - MapRDBFormatPluginConfig formatConfig = new MapRDBFormatPluginConfig(); - - formatConfig.readTimestampWithZoneOffset = - settings.getOptions().getBoolean(ExecConstants.HIVE_READ_MAPRDB_JSON_TIMESTAMP_WITH_TIMEZONE_OFFSET); - - formatConfig.allTextMode = settings.getOptions().getBoolean(ExecConstants.HIVE_MAPRDB_JSON_ALL_TEXT_MODE); - - JsonTableGroupScan nativeMapRDBScan = - new JsonTableGroupScan( - hiveScan.getUserName(), - hiveScan.getStoragePlugin(), - // TODO: We should use Hive format plugins here, once it will be implemented. DRILL-6621 - (MapRDBFormatPlugin) hiveScan.getStoragePlugin().getFormatPlugin(formatConfig), - scanSpec, - hiveScanCols, - new MapRDBStatistics(), - FileSystemMetadataProviderManager.getMetadataProviderForSchema(schema) - ); - - List nativeScanColNames = hiveScanRel.getRowType().getFieldList().stream() - .map(field -> replaceOverriddenColumnId(parameters, field.getName())) - .collect(Collectors.toList()); - List nativeScanColTypes = hiveScanRel.getRowType().getFieldList().stream() - .map(RelDataTypeField::getType) - .collect(Collectors.toList()); - RelDataType nativeScanRowType = typeFactory.createStructType(nativeScanColTypes, nativeScanColNames); - - return new DrillScanRel( - hiveScanRel.getCluster(), - hiveScanRel.getTraitSet(), - hiveScanRel.getTable(), - nativeMapRDBScan, - nativeScanRowType, - hiveScanCols); - } - - /** - * Hive maps column id "_id" with custom user column id name. Replace it for {@link DrillScanRel} - * - * @param parameters Hive table properties - * @param colName Hive column name - * @return original column name, null if colName is absent - */ - private String replaceOverriddenColumnId(Map parameters, String colName) { - return colName != null && colName.equals(parameters.get(MAPRDB_COLUMN_ID)) ? ID_KEY : colName; - } - - /** - * The same as above, but for {@link SchemaPath} object - * - * @param parameters Hive table properties - * @param colNameSchemaPath SchemaPath with Hive column name - * @return SchemaPath with original column name - */ - private SchemaPath replaceOverriddenSchemaPath(Map parameters, SchemaPath colNameSchemaPath) { - String hiveColumnName = colNameSchemaPath.getRootSegmentPath(); - return hiveColumnName != null && hiveColumnName.equals(parameters.get(MAPRDB_COLUMN_ID)) - ? SchemaPath.getSimplePath(ID_KEY) : colNameSchemaPath; - } -} From d6bf40e7f5862d74b5595d7f197b205e5a445159 Mon Sep 17 00:00:00 2001 From: James Turton Date: Sat, 31 Aug 2024 10:44:18 +0200 Subject: [PATCH 2/3] Print swap space usage to console before removing it. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39c3fc0aa9e..8de4b0ce001 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,7 @@ jobs: - name: Remove swap space run : | sudo sh -c " + free -h swapoff /tmp/swapfile rm /tmp/swapfile " From c32297acaaeeac9b806d02e950e31d53acc239fc Mon Sep 17 00:00:00 2001 From: James Turton Date: Tue, 17 Sep 2024 07:29:31 +0200 Subject: [PATCH 3/3] Remove Guava patching. --- .../drill/common/util/GuavaPatcher.java | 146 ----------- .../drill/common/util/TestGuavaPatcher.java | 243 ------------------ .../java/org/apache/drill/test/BaseTest.java | 6 - .../appMaster/DrillApplicationMaster.java | 6 - .../apache/drill/yarn/client/DrillOnYarn.java | 8 +- .../apache/drill/exec/server/Drillbit.java | 6 - .../java/org/apache/drill/jdbc/Driver.java | 3 +- 7 files changed, 2 insertions(+), 416 deletions(-) delete mode 100644 common/src/main/java/org/apache/drill/common/util/GuavaPatcher.java delete mode 100644 common/src/test/java/org/apache/drill/common/util/TestGuavaPatcher.java diff --git a/common/src/main/java/org/apache/drill/common/util/GuavaPatcher.java b/common/src/main/java/org/apache/drill/common/util/GuavaPatcher.java deleted file mode 100644 index ec6c1cabfc8..00000000000 --- a/common/src/main/java/org/apache/drill/common/util/GuavaPatcher.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.common.util; - -import java.lang.reflect.Modifier; - -import com.google.common.annotations.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javassist.ClassPool; -import javassist.CtClass; -import javassist.CtConstructor; -import javassist.CtMethod; -import javassist.CtNewMethod; -import javassist.scopedpool.ScopedClassPoolRepository; -import javassist.scopedpool.ScopedClassPoolRepositoryImpl; - -public class GuavaPatcher { - private static final Logger logger = LoggerFactory.getLogger(GuavaPatcher.class); - - private static boolean patchingAttempted; - - public static synchronized void patch() { - if (!patchingAttempted) { - patchingAttempted = true; - patchStopwatch(); - patchCloseables(); - patchFuturesCallback(); - } - } - - private static boolean patchingSuccessful = true; - - @VisibleForTesting - static boolean isPatchingSuccessful() { - return patchingAttempted && patchingSuccessful; - } - - /** - * Makes Guava stopwatch look like the old version for compatibility with hbase-server (for test purposes). - */ - private static void patchStopwatch() { - try { - ClassPool cp = getClassPool(); - CtClass cc = cp.get("com.google.common.base.Stopwatch"); - - // Expose the constructor for Stopwatch for old libraries who use the pattern new Stopwatch().start(). - for (CtConstructor c : cc.getConstructors()) { - if (!Modifier.isStatic(c.getModifiers())) { - c.setModifiers(Modifier.PUBLIC); - } - } - - // Add back the Stopwatch.elapsedMillis() method for old consumers. - CtMethod newMethod = CtNewMethod.make( - "public long elapsedMillis() { return elapsed(java.util.concurrent.TimeUnit.MILLISECONDS); }", cc); - cc.addMethod(newMethod); - - // Load the modified class instead of the original. - cc.toClass(); - - logger.info("Google's Stopwatch patched for old HBase Guava version."); - } catch (Exception e) { - logUnableToPatchException(e); - } - } - - private static void logUnableToPatchException(Exception e) { - patchingSuccessful = false; - logger.warn("Unable to patch Guava classes: {}", e.getMessage()); - logger.debug("Exception:", e); - } - - private static void patchCloseables() { - try { - ClassPool cp = getClassPool(); - CtClass cc = cp.get("com.google.common.io.Closeables"); - - // Add back the Closeables.closeQuietly() method for old consumers. - CtMethod newMethod = CtNewMethod.make( - "public static void closeQuietly(java.io.Closeable closeable) { try{closeable.close();}catch(Exception e){} }", - cc); - cc.addMethod(newMethod); - - // Load the modified class instead of the original. - cc.toClass(); - - logger.info("Google's Closeables patched for old HBase Guava version."); - } catch (Exception e) { - logUnableToPatchException(e); - } - } - - /** - * Returns {@link javassist.scopedpool.ScopedClassPool} instance which uses the same class loader - * which was used for loading current class. - * - * @return {@link javassist.scopedpool.ScopedClassPool} instance - */ - private static ClassPool getClassPool() { - ScopedClassPoolRepository classPoolRepository = ScopedClassPoolRepositoryImpl.getInstance(); - // sets prune flag to false to avoid freezing and pruning classes right after obtaining CtClass instance - classPoolRepository.setPrune(false); - return classPoolRepository.createScopedClassPool(GuavaPatcher.class.getClassLoader(), null); - } - - /** - * Makes Guava Futures#addCallback look like the old version for compatibility with hadoop-hdfs (for test purposes). - */ - private static void patchFuturesCallback() { - try { - ClassPool cp = getClassPool(); - CtClass cc = cp.get("com.google.common.util.concurrent.Futures"); - - // Add back the Futures.addCallback(ListenableFuture future, FutureCallback callback) method for old consumers. - CtMethod newMethod = CtNewMethod.make( - "public static void addCallback(" + - "com.google.common.util.concurrent.ListenableFuture future, " + - "com.google.common.util.concurrent.FutureCallback callback" + - ") { addCallback(future, callback, com.google.common.util.concurrent.MoreExecutors.directExecutor()); }", cc); - cc.addMethod(newMethod); - - // Load the modified class instead of the original. - cc.toClass(); - logger.info("Google's Futures#addCallback patched for old Hadoop Guava version."); - } catch (Exception e) { - logUnableToPatchException(e); - } - } -} diff --git a/common/src/test/java/org/apache/drill/common/util/TestGuavaPatcher.java b/common/src/test/java/org/apache/drill/common/util/TestGuavaPatcher.java deleted file mode 100644 index 5dbdf537e7a..00000000000 --- a/common/src/test/java/org/apache/drill/common/util/TestGuavaPatcher.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.common.util; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.Closeable; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.Arrays; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.apache.drill.test.BaseTest; -import org.junit.Test; - -// Easier to test Guava patching if we can reference Guava classes directly for the tests -// CHECKSTYLE:OFF -import com.google.common.base.Preconditions; -import com.google.common.base.Stopwatch; -import com.google.common.base.Ticker; -import com.google.common.io.Closeables; -//CHECKSTYLE:ON - -/** - * Test class for {@code GuavaPatcher} - * - */ -public class TestGuavaPatcher extends BaseTest { - - @Test - public void checkSuccessfulPatching() { - // Catch-all test to see if Guava patching was successful - assertTrue("Guava Patcher ran with errors, check error log messages", - GuavaPatcher.isPatchingSuccessful()); - } - - @Test - public void checkStopwatchEllapsedMillis() throws Exception { - long[] currentTimeMillis = new long[] { 0L }; - final Ticker ticker = new Ticker() { - @Override - public long read() { - return TimeUnit.MILLISECONDS.toNanos(currentTimeMillis[0]); - } - }; - final Stopwatch stopwatch = Stopwatch.createStarted(ticker); - currentTimeMillis[0] = 12345L; - stopwatch.stop(); - - assertEquals(currentTimeMillis[0], - stopwatch.elapsed(TimeUnit.MILLISECONDS)); - assertEquals(currentTimeMillis[0], (long) invokeMethod(Stopwatch.class, - "elapsedMillis", new Class[] {}, stopwatch)); - } - - @Test - public void checkCloseablesCloseQuietly() throws Exception { - final Closeable alwaysThrows = () -> { - throw new IOException("Always fail"); - }; - - invokeMethod(Closeables.class, "closeQuietly", - new Class[] { Closeable.class }, null, alwaysThrows); - } - - // All the preconditions checks are method which were previously added for - // compatibility with Apache Iceberg but are not necessary anymore because - // Guava's version has been updated since. - - @Test - public void checkPreconditionsCheckArgumentIntParam() throws Exception { - invokeMethod(Preconditions.class, "checkArgument", - new Class[] { boolean.class, String.class, int.class }, null, true, - "Error Message %s", 1); - try { - invokeMethod(Preconditions.class, "checkArgument", - new Class[] { boolean.class, String.class, int.class }, null, false, - "Error Message %s", 1); - fail(); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause(); - assertEquals(IllegalArgumentException.class, cause.getClass()); - assertEquals("Error Message 1", cause.getMessage()); - } - } - - @Test - public void checkPreconditionsCheckArgumentLongParam() throws Exception { - invokeMethod(Preconditions.class, "checkArgument", - new Class[] { boolean.class, String.class, long.class }, null, true, - "Error Message %s", 2L); - try { - invokeMethod(Preconditions.class, "checkArgument", - new Class[] { boolean.class, String.class, long.class }, null, false, - "Error Message %s", 2L); - fail(); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause(); - assertEquals(IllegalArgumentException.class, cause.getClass()); - assertEquals("Error Message 2", cause.getMessage()); - } - } - - @Test - public void checkPreconditionsCheckArgumentLongLongParam() throws Exception { - invokeMethod(Preconditions.class, "checkArgument", - new Class[] { boolean.class, String.class, long.class, long.class }, - null, true, "Error Message %s %s", 3L, 4L); - try { - invokeMethod(Preconditions.class, "checkArgument", - new Class[] { boolean.class, String.class, long.class, long.class }, - null, false, "Error Message %s %s", 3L, 4L); - fail(); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause(); - assertEquals(IllegalArgumentException.class, cause.getClass()); - assertEquals("Error Message 3 4", cause.getMessage()); - } - } - - @Test - public void checkPreconditionsCheckNotNullIntParam() throws Exception { - invokeMethod(Preconditions.class, "checkNotNull", - new Class[] { Object.class, String.class, int.class }, null, this, - "Error Message %s", 5); - try { - invokeMethod(Preconditions.class, "checkNotNull", - new Class[] { Object.class, String.class, int.class }, null, null, - "Error Message %s", 5); - fail(); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause(); - assertEquals(NullPointerException.class, cause.getClass()); - assertEquals("Error Message 5", cause.getMessage()); - } - } - - @Test - public void checkPreconditionsCheckStateIntParam() throws Exception { - invokeMethod(Preconditions.class, "checkState", - new Class[] { boolean.class, String.class, int.class }, null, true, - "Error Message %s", 6); - try { - invokeMethod(Preconditions.class, "checkState", - new Class[] { boolean.class, String.class, int.class }, null, false, - "Error Message %s", 6); - fail(); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause(); - assertEquals(IllegalStateException.class, cause.getClass()); - assertEquals("Error Message 6", cause.getMessage()); - } - } - - @Test - public void checkPreconditionsCheckNotNullVarargs() throws Exception { - checkPreconditionsCheckVarargMethod("checkNotNull", Object.class, this, - null, NullPointerException.class); - } - - @Test - public void checkPreconditionsCheckArgumentVarargs() throws Exception { - checkPreconditionsCheckVarargMethod("checkArgument", boolean.class, true, - false, IllegalArgumentException.class); - } - - @Test - public void checkPreconditionsCheckStateVarargs() throws Exception { - checkPreconditionsCheckVarargMethod("checkState", boolean.class, true, - false, IllegalStateException.class); - } - - private void checkPreconditionsCheckVarargMethod(String methodName, - Class argClass, T goodValue, T badValue, - Class exceptionClass) throws Exception { - for (int i = 1; i < 5; i++) { - String[] templatePlaceholders = new String[i]; - Arrays.fill(templatePlaceholders, "%s"); - - Object[] templateArguments = new Object[i]; - Arrays.setAll(templateArguments, Integer::valueOf); - - String template = "Error message: " - + Stream.of(templatePlaceholders).collect(Collectors.joining(",")); - String message = "Error message: " + Stream.of(templateArguments) - .map(Object::toString).collect(Collectors.joining(",")); - - Class[] parameterTypes = new Class[2 + i]; - parameterTypes[0] = argClass; - parameterTypes[1] = String.class; - Arrays.fill(parameterTypes, 2, parameterTypes.length, Object.class); - - Object[] parameters = new Object[2 + i]; - parameters[0] = goodValue; - parameters[1] = template; - System.arraycopy(templateArguments, 0, parameters, 2, i); - - // Check successful call - invokeMethod(Preconditions.class, methodName, parameterTypes, null, - parameters); - - try { - parameters[0] = badValue; - invokeMethod(Preconditions.class, methodName, parameterTypes, null, - parameters); - fail(); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause(); - assertEquals(exceptionClass, cause.getClass()); - assertEquals(message, cause.getMessage()); - } - } - } - - @SuppressWarnings("unchecked") - private static T invokeMethod(Class clazz, String methodName, - Class[] argumentTypes, Object object, Object... arguments) - throws Exception { - Method method = clazz.getMethod(methodName, argumentTypes); - return (T) method.invoke(object, arguments); - } - -} diff --git a/common/src/test/java/org/apache/drill/test/BaseTest.java b/common/src/test/java/org/apache/drill/test/BaseTest.java index 6256b5e74c9..92f0d1c26f1 100644 --- a/common/src/test/java/org/apache/drill/test/BaseTest.java +++ b/common/src/test/java/org/apache/drill/test/BaseTest.java @@ -17,7 +17,6 @@ */ package org.apache.drill.test; -import org.apache.drill.common.util.GuavaPatcher; import org.apache.drill.common.util.ProtobufPatcher; /** @@ -33,10 +32,5 @@ public class BaseTest { * This code removes these final modifiers. */ ProtobufPatcher.patch(); - /* - * Some libraries, such as Hadoop, HBase, Iceberg depend on incompatible versions of Guava. - * This code adds back some methods to so that the libraries can work with single Guava version. - */ - GuavaPatcher.patch(); } } diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/DrillApplicationMaster.java b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/DrillApplicationMaster.java index e144bff48c1..a74281ba1ff 100644 --- a/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/DrillApplicationMaster.java +++ b/drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/DrillApplicationMaster.java @@ -19,7 +19,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.drill.common.util.GuavaPatcher; import org.apache.drill.common.util.ProtobufPatcher; import org.apache.drill.yarn.appMaster.ControllerFactory.ControllerFactoryException; import org.apache.drill.yarn.appMaster.http.WebServer; @@ -54,11 +53,6 @@ public class DrillApplicationMaster { * This code removes these final modifiers. */ ProtobufPatcher.patch(); - /* - * Some libraries, such as Hadoop or HBase, depend on incompatible versions of Guava. - * This code adds back some methods to so that the libraries can work with single Guava version. - */ - GuavaPatcher.patch(); } private static final Log LOG = LogFactory diff --git a/drill-yarn/src/main/java/org/apache/drill/yarn/client/DrillOnYarn.java b/drill-yarn/src/main/java/org/apache/drill/yarn/client/DrillOnYarn.java index 5f79f5da7f1..650a698eb15 100644 --- a/drill-yarn/src/main/java/org/apache/drill/yarn/client/DrillOnYarn.java +++ b/drill-yarn/src/main/java/org/apache/drill/yarn/client/DrillOnYarn.java @@ -18,7 +18,6 @@ package org.apache.drill.yarn.client; -import org.apache.drill.common.util.GuavaPatcher; import org.apache.drill.common.util.ProtobufPatcher; import org.apache.drill.yarn.core.DoyConfigException; import org.apache.drill.yarn.core.DrillOnYarnConfig; @@ -83,11 +82,6 @@ public class DrillOnYarn { * This code removes these final modifiers. */ ProtobufPatcher.patch(); - /* - * Some libraries, such as Hadoop or HBase, depend on incompatible versions of Guava. - * This code adds back some methods to so that the libraries can work with single Guava version. - */ - GuavaPatcher.patch(); } public static void main(String argv[]) { @@ -190,4 +184,4 @@ private static void displayError(CommandLineOptions opts, ClientException e) { e.printStackTrace(ClientContext.err); } } -} \ No newline at end of file +} diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java index cb78340f7e4..b3ac6eaa1e6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/Drillbit.java @@ -47,7 +47,6 @@ import org.apache.drill.exec.store.sys.store.provider.CachingPersistentStoreProvider; import org.apache.drill.exec.store.sys.store.provider.InMemoryStoreProvider; import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider; -import org.apache.drill.common.util.GuavaPatcher; import org.apache.drill.common.util.ProtobufPatcher; import org.apache.drill.exec.work.WorkManager; import com.google.common.annotations.VisibleForTesting; @@ -85,11 +84,6 @@ public class Drillbit implements AutoCloseable { * This code removes these final modifiers. */ ProtobufPatcher.patch(); - /* - * Some libraries, such as Hadoop or HBase, depend on incompatible versions of Guava. - * This code adds back some methods to so that the libraries can work with single Guava version. - */ - GuavaPatcher.patch(); Environment.logEnv("Drillbit environment: ", logger); // Jersey uses java.util.logging - create bridge: jul to slf4j SLF4JBridgeHandler.removeHandlersForRootLogger(); diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/Driver.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/Driver.java index 6b8dd32970d..74daa3d14c8 100644 --- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/Driver.java +++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/Driver.java @@ -44,12 +44,11 @@ public class Driver implements java.sql.Driver { static { ProtobufPatcher.patch(); - GuavaPatcher.patch(); // Upon loading of class, register an instance with DriverManager. try { DriverManager.registerDriver(new Driver()); } catch (Error | SQLException e) { - logger.warn("Error in registering Drill JDBC driver {}: {}", Driver.class, e, e); + logger.error("Error in registering Drill JDBC driver {}: {}", Driver.class, e, e); } }