Skip to content

Commit 0033c69

Browse files
anloweeanlowee
committed
Add unit tests
1 parent bea15e7 commit 0033c69

File tree

3 files changed

+365
-2
lines changed

3 files changed

+365
-2
lines changed

‎presto-clp/src/main/java/com/facebook/presto/plugin/clp/optimization/ClpFilterToKqlConverter.java‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import com.facebook.presto.common.function.OperatorType;
1717
import com.facebook.presto.common.type.DecimalType;
1818
import com.facebook.presto.common.type.RowType;
19+
import com.facebook.presto.common.type.TimestampType;
1920
import com.facebook.presto.common.type.Type;
2021
import com.facebook.presto.common.type.VarcharType;
2122
import com.facebook.presto.plugin.clp.ClpColumnHandle;
@@ -57,12 +58,15 @@
5758
import static com.facebook.presto.common.type.IntegerType.INTEGER;
5859
import static com.facebook.presto.common.type.RealType.REAL;
5960
import static com.facebook.presto.common.type.SmallintType.SMALLINT;
61+
import static com.facebook.presto.common.type.TimestampType.TIMESTAMP;
62+
import static com.facebook.presto.common.type.TimestampType.TIMESTAMP_MICROSECONDS;
6063
import static com.facebook.presto.common.type.TinyintType.TINYINT;
6164
import static com.facebook.presto.plugin.clp.ClpErrorCode.CLP_PUSHDOWN_UNSUPPORTED_EXPRESSION;
6265
import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.AND;
6366
import static java.lang.Integer.parseInt;
6467
import static java.lang.String.format;
6568
import static java.util.Objects.requireNonNull;
69+
import static java.util.concurrent.TimeUnit.SECONDS;
6670

6771
/**
6872
* A translator to translate Presto {@link RowExpression}s into:
@@ -256,8 +260,10 @@ private ClpExpression handleBetween(CallExpression node)
256260
}
257261

258262
String variable = variableOpt.get();
259-
String lowerBound = getLiteralString((ConstantExpression) second);
260-
String upperBound = getLiteralString((ConstantExpression) third);
263+
Type lowerBoundType = second.getType();
264+
String lowerBound = tryEnsureNanosecondTimestamp(lowerBoundType, getLiteralString((ConstantExpression) second));
265+
Type upperBoundType = third.getType();
266+
String upperBound = tryEnsureNanosecondTimestamp(upperBoundType, getLiteralString((ConstantExpression) third));
261267
String kql = String.format("%s >= %s AND %s <= %s", variable, lowerBound, variable, upperBound);
262268
String metadataSqlQuery = metadataFilterColumns.contains(variable)
263269
? String.format("\"%s\" >= %s AND \"%s\" <= %s", variable, lowerBound, variable, upperBound)
@@ -440,6 +446,7 @@ private ClpExpression buildClpExpression(
440446
RowExpression originalNode)
441447
{
442448
String metadataSqlQuery = null;
449+
literalString = tryEnsureNanosecondTimestamp(literalType, literalString);
443450
if (operator.equals(EQUAL)) {
444451
if (literalType instanceof VarcharType) {
445452
return new ClpExpression(format("%s: \"%s\"", variableName, escapeKqlSpecialCharsForStringValue(literalString)));
@@ -914,9 +921,31 @@ public static boolean isClpCompatibleNumericType(Type type)
914921
|| type.equals(TINYINT)
915922
|| type.equals(DOUBLE)
916923
|| type.equals(REAL)
924+
|| type.equals(TIMESTAMP)
925+
|| type.equals(TIMESTAMP_MICROSECONDS)
917926
|| type instanceof DecimalType;
918927
}
919928

929+
private static String tryEnsureNanosecondTimestamp(Type type, String literalString)
930+
{
931+
if (type == TIMESTAMP) {
932+
return ensureNanosecondTimestamp(TIMESTAMP, literalString);
933+
}
934+
else if (type == TIMESTAMP_MICROSECONDS) {
935+
return ensureNanosecondTimestamp(TIMESTAMP_MICROSECONDS, literalString);
936+
}
937+
return literalString;
938+
}
939+
940+
private static String ensureNanosecondTimestamp(TimestampType type, String literalString)
941+
{
942+
long literalNumber = Long.parseLong(literalString);
943+
long seconds = type.getEpochSecond(literalNumber);
944+
long nanosecondFraction = type.getNanos(literalNumber);
945+
long nanoseconds = SECONDS.toNanos(seconds) + nanosecondFraction;
946+
return Long.toString(nanoseconds);
947+
}
948+
920949
private static class SubstrInfo
921950
{
922951
String variableName;

‎presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpPushDown.java‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.facebook.presto.plugin.clp;
15+
16+
import com.facebook.presto.dispatcher.DispatchManager;
17+
import com.facebook.presto.execution.QueryManager;
18+
import com.facebook.presto.plugin.clp.metadata.ClpMetadataProvider;
19+
import com.facebook.presto.plugin.clp.metadata.ClpMySqlMetadataProvider;
20+
import com.facebook.presto.plugin.clp.mockdb.ClpMockMetadataDatabase;
21+
import com.facebook.presto.plugin.clp.mockdb.table.ColumnMetadataTableRows;
22+
import com.facebook.presto.spi.QueryId;
23+
import com.facebook.presto.spi.plan.PlanNode;
24+
import com.facebook.presto.spi.plan.PlanNodeId;
25+
import com.facebook.presto.spi.plan.TableScanNode;
26+
import com.facebook.presto.tests.DistributedQueryRunner;
27+
import com.google.common.collect.ImmutableList;
28+
import com.google.common.collect.ImmutableMap;
29+
import org.intellij.lang.annotations.Language;
30+
import org.testng.annotations.AfterMethod;
31+
import org.testng.annotations.BeforeMethod;
32+
import org.testng.annotations.Test;
33+
34+
import java.util.Map;
35+
import java.util.Optional;
36+
37+
import static com.facebook.presto.execution.QueryState.RUNNING;
38+
import static com.facebook.presto.plugin.clp.ClpQueryRunner.createQueryRunner;
39+
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.Boolean;
40+
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.DateString;
41+
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.Float;
42+
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.Integer;
43+
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.VarString;
44+
import static java.lang.String.format;
45+
import static org.testng.Assert.assertEquals;
46+
import static org.testng.Assert.assertTrue;
47+
import static org.testng.Assert.fail;
48+
49+
@Test(singleThreaded = true)
50+
public class TestClpPushDown
51+
{
52+
private static final String TABLE_NAME = "test_pushdown";
53+
private static final Long TEST_TS_SECONDS = 1746003005L;
54+
private static final Long TEST_TS_NANOSECONDS = 1746003005000000000L;
55+
56+
private ClpMockMetadataDatabase mockMetadataDatabase;
57+
private DistributedQueryRunner queryRunner;
58+
private QueryManager queryManager;
59+
private DispatchManager dispatchManager;
60+
61+
@BeforeMethod
62+
public void setUp() throws Exception {
63+
mockMetadataDatabase = ClpMockMetadataDatabase
64+
.builder()
65+
.build();
66+
mockMetadataDatabase.addTableToDatasetsTableIfNotExist(ImmutableList.of(TABLE_NAME));
67+
mockMetadataDatabase.addColumnMetadata(ImmutableMap.of(TABLE_NAME, new ColumnMetadataTableRows(
68+
ImmutableList.of(
69+
"city.Name",
70+
"city.Region.id",
71+
"city.Region.Name",
72+
"fare",
73+
"isHoliday",
74+
"ts"),
75+
ImmutableList.of(
76+
VarString,
77+
Integer,
78+
VarString,
79+
Float,
80+
Boolean,
81+
DateString))));
82+
ClpConfig config = new ClpConfig()
83+
.setPolymorphicTypeEnabled(true)
84+
.setMetadataDbUrl(mockMetadataDatabase.getUrl())
85+
.setMetadataDbUser(mockMetadataDatabase.getUsername())
86+
.setMetadataDbPassword(mockMetadataDatabase.getPassword())
87+
.setMetadataTablePrefix(mockMetadataDatabase.getTablePrefix());
88+
ClpMetadataProvider metadataProvider = new ClpMySqlMetadataProvider(config);
89+
queryRunner = createQueryRunner(
90+
mockMetadataDatabase.getUrl(),
91+
mockMetadataDatabase.getUsername(),
92+
mockMetadataDatabase.getPassword(),
93+
mockMetadataDatabase.getTablePrefix(),
94+
Optional.of(0),
95+
Optional.empty());
96+
queryManager = queryRunner.getCoordinator().getQueryManager();
97+
dispatchManager = queryRunner.getCoordinator().getDispatchManager();
98+
}
99+
100+
@AfterMethod
101+
public void tearDown() throws InterruptedException {
102+
long maxCleanUpTime = 5 * 1000L; // 5 seconds
103+
long currentCleanUpTime = 0L;
104+
while (!queryManager.getQueries().isEmpty() && currentCleanUpTime < maxCleanUpTime) {
105+
Thread.sleep(1000L);
106+
currentCleanUpTime += 1000L;
107+
}
108+
if (null != mockMetadataDatabase) {
109+
mockMetadataDatabase.teardown();
110+
}
111+
}
112+
113+
public void testTimestampComparisons()
114+
{
115+
testPushDown(format("ts > from_unixtime(%s)", TEST_TS_SECONDS), format("ts > %s", TEST_TS_NANOSECONDS));
116+
testPushDown(format("ts >= from_unixtime(%s)", TEST_TS_SECONDS), format("ts >= %s", TEST_TS_NANOSECONDS));
117+
testPushDown(format("ts < from_unixtime(%s)", TEST_TS_SECONDS), format("ts < %s", TEST_TS_NANOSECONDS));
118+
testPushDown(format("ts <= from_unixtime(%s)", TEST_TS_SECONDS), format("ts <= %s", TEST_TS_NANOSECONDS));
119+
testPushDown(format("ts = from_unixtime(%s)", TEST_TS_SECONDS), format("ts: %s", TEST_TS_NANOSECONDS));
120+
testPushDown(format("ts != from_unixtime(%s)", TEST_TS_SECONDS), format("NOT ts: %s", TEST_TS_NANOSECONDS));
121+
testPushDown(format("ts <> from_unixtime(%s)", TEST_TS_SECONDS), format("NOT ts: %s", TEST_TS_NANOSECONDS));
122+
testPushDown(format("from_unixtime(%s) < ts", TEST_TS_SECONDS), format("ts > %s", TEST_TS_NANOSECONDS));
123+
testPushDown(format("from_unixtime(%s) <= ts", TEST_TS_SECONDS), format("ts >= %s", TEST_TS_NANOSECONDS));
124+
testPushDown(format("from_unixtime(%s) > ts", TEST_TS_SECONDS), format("ts < %s", TEST_TS_NANOSECONDS));
125+
testPushDown(format("from_unixtime(%s) >= ts", TEST_TS_SECONDS), format("ts <= %s", TEST_TS_NANOSECONDS));
126+
testPushDown(format("from_unixtime(%s) = ts", TEST_TS_SECONDS), format("ts: %s", TEST_TS_NANOSECONDS));
127+
testPushDown(format("from_unixtime(%s) != ts", TEST_TS_SECONDS), format("NOT ts: %s", TEST_TS_NANOSECONDS));
128+
testPushDown(format("from_unixtime(%s) <> ts", TEST_TS_SECONDS), format("NOT ts: %s", TEST_TS_NANOSECONDS));
129+
}
130+
131+
private void testPushDown(String filter, String expectedPushDown) {
132+
try {
133+
QueryId id = queryRunner.getCoordinator().getDispatchManager().createQueryId();
134+
@Language("SQL") String sql = format("SELECT * FROM clp.default.test_pushdown WHERE %s LIMIT 1", filter);
135+
dispatchManager.createQuery(
136+
id,
137+
"slug",
138+
0,
139+
new TestingClpSessionContext(queryRunner.getDefaultSession()),
140+
sql).get();
141+
long maxDispatchingAndPlanningTime = 60 * 1000L; // 1 minute
142+
long currentWaitingTime = 0L;
143+
while (dispatchManager.getQueryInfo(id).getState().ordinal() != RUNNING.ordinal() && currentWaitingTime < maxDispatchingAndPlanningTime) {
144+
Thread.sleep(1000L);
145+
currentWaitingTime += 1000L;
146+
}
147+
assertTrue(currentWaitingTime < maxDispatchingAndPlanningTime);
148+
boolean isPushDownGenerated = false;
149+
for (Map.Entry<PlanNodeId, PlanNode> entry : queryManager.getFullQueryInfo(id).getPlanIdNodeMap().entrySet()) {
150+
if (!(entry.getValue() instanceof TableScanNode)
151+
|| (!(((TableScanNode) entry.getValue()).getTable().getLayout().orElse(null) instanceof ClpTableLayoutHandle))) {
152+
continue;
153+
}
154+
ClpTableLayoutHandle clpTableLayoutHandle = (ClpTableLayoutHandle) ((TableScanNode) entry.getValue()).getTable().getLayout().orElseThrow(AssertionError::new);
155+
String actualPushDown = clpTableLayoutHandle.getKqlQuery().orElse(null);
156+
assertEquals(actualPushDown, expectedPushDown);
157+
isPushDownGenerated = true;
158+
break;
159+
}
160+
assertTrue(isPushDownGenerated);
161+
queryManager.cancelQuery(id);
162+
} catch (Exception e) {
163+
fail(e.getMessage());
164+
}
165+
}
166+
}

0 commit comments

Comments
 (0)