Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add DATE_TRUNC Optimizer #14385

Draft
wants to merge 25 commits into
base: master
Choose a base branch
from
Draft

Add DATE_TRUNC Optimizer #14385

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
e3efa65
add floor and ceil func
ashishjayamohan Nov 4, 2024
731b943
Merge branch 'apache:master' into optimize-date-trunc
ashishjayamohan Nov 5, 2024
749bbbb
add date trunc optimization
ashishjayamohan Nov 5, 2024
c2dbe08
fix style
ashishjayamohan Nov 5, 2024
8dda15e
fix range query column expr
ashishjayamohan Nov 5, 2024
2edad2a
add support for between
ashishjayamohan Nov 5, 2024
52490a3
refactor test functions
ashishjayamohan Nov 5, 2024
cb387da
refactor back and add test
ashishjayamohan Nov 5, 2024
0b41d25
add tests for range pred
ashishjayamohan Nov 5, 2024
9391b48
style
ashishjayamohan Nov 5, 2024
36e7639
fix algo and tests
ashishjayamohan Nov 5, 2024
bc08f03
fix between operator
ashishjayamohan Nov 6, 2024
b09fc49
add tests
ashishjayamohan Nov 6, 2024
eec0e7f
use correct output format
ashishjayamohan Nov 6, 2024
614e176
add helper function
ashishjayamohan Nov 6, 2024
624e360
make unit step less brittle
ashishjayamohan Nov 6, 2024
5e87ed7
correct conversion to output unit
ashishjayamohan Nov 6, 2024
4b836c7
add javadocs
ashishjayamohan Nov 6, 2024
470d4bb
fix short circuit
ashishjayamohan Nov 7, 2024
321cc6a
fix range settings
ashishjayamohan Nov 7, 2024
e3a6651
everything but time zones
ashishjayamohan Nov 11, 2024
2e6bb78
just logging for future
ashishjayamohan Nov 12, 2024
8b58e9e
update, still not working
ashishjayamohan Nov 21, 2024
f38e322
Merge branch 'apache:master' into optimize-date-trunc
ashishjayamohan Jan 25, 2025
90e87c6
Merge branch 'apache:master' into optimize-date-trunc
ashishjayamohan Feb 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 113 additions & 0 deletions .../main/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,23 @@
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.common.function.DateTimeUtils;
import org.apache.pinot.common.function.TimeZoneKey;
import org.apache.pinot.common.request.Expression;
import org.apache.pinot.common.request.ExpressionType;
import org.apache.pinot.common.request.Function;
import org.apache.pinot.common.request.Literal;
import org.apache.pinot.common.utils.request.RequestUtils;
import org.apache.pinot.core.operator.transform.function.DateTimeConversionTransformFunction;
import org.apache.pinot.core.operator.transform.function.DateTruncTransformFunction;
import org.apache.pinot.core.operator.transform.function.TimeConversionTransformFunction;
import org.apache.pinot.spi.data.DateTimeFieldSpec.TimeFormat;
import org.apache.pinot.spi.data.DateTimeFormatSpec;
import org.apache.pinot.spi.data.DateTimeGranularitySpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.utils.TimeUtils;
import org.apache.pinot.sql.FilterKind;
import org.joda.time.chrono.ISOChronology;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -84,6 +88,8 @@ Expression optimize(Expression filterExpression) {
optimizeTimeConvert(filterFunction, filterKind);
} else if (functionName.equalsIgnoreCase(DateTimeConversionTransformFunction.FUNCTION_NAME)) {
optimizeDateTimeConvert(filterFunction, filterKind);
} else if (functionName.equalsIgnoreCase(DateTruncTransformFunction.FUNCTION_NAME)) {
ashishjayamohan marked this conversation as resolved.
Show resolved Hide resolved
optimizeDateTrunc(filterFunction, filterKind);
}
}
}
Expand Down Expand Up @@ -411,6 +417,96 @@ && isStringLiteral(dateTimeConvertOperands.get(3)),
}
}

private void optimizeDateTrunc(Function filterFunction, FilterKind filterKind) {
List<Expression> filterOperands = filterFunction.getOperands();
List<Expression> dateTruncOperands = filterOperands.get(0).getFunctionCall().getOperands();

// Check if date trunc function is being applied on a literal value
if (dateTruncOperands.get(1).isSetLiteral()) {
ashishjayamohan marked this conversation as resolved.
Show resolved Hide resolved
return;
}

Long lowerMillis = null;
Long upperMillis = null;
DateTimeFormatSpec inputFormat = (dateTruncOperands.size() >= 3)
? new DateTimeFormatSpec(dateTruncOperands.get(2).getLiteral().getStringValue())
: new DateTimeFormatSpec("TIMESTAMP");
boolean lowerInclusive = true;
boolean upperInclusive = true;
List<Expression> operands = new ArrayList<>(dateTruncOperands);
switch (filterKind) {
case EQUALS:
operands.set(1, filterOperands.get(1));
lowerMillis = dateTruncFloor(operands);
upperMillis = dateTruncCeil(operands);
ashishjayamohan marked this conversation as resolved.
Show resolved Hide resolved
// Check if it is impossible to obtain literal equality
if (lowerMillis != inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())) {
lowerMillis = Long.MAX_VALUE;
upperMillis = Long.MIN_VALUE;
}
break;
case GREATER_THAN:
lowerInclusive = false;
operands.set(1, filterOperands.get(1));
lowerMillis = dateTruncCeil(operands);
break;
case GREATER_THAN_OR_EQUAL:
operands.set(1, filterOperands.get(1));
lowerInclusive = false;
lowerMillis = dateTruncCeil(operands);
if (dateTruncFloor(operands)
== inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())) {
lowerInclusive = true;
lowerMillis = dateTruncFloor(operands);
}
break;
case LESS_THAN:
upperInclusive = false;
operands.set(1, filterOperands.get(1));
upperMillis = dateTruncFloor(operands);
if (upperMillis != inputFormat.fromFormatToMillis(filterOperands.get(1).getLiteral().getLongValue())) {
upperInclusive = true;
upperMillis = dateTruncCeil(operands);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why is this recomputed here?

}
break;
case LESS_THAN_OR_EQUAL:
operands.set(1, filterOperands.get(1));
upperMillis = dateTruncCeil(operands);
break;
case BETWEEN:
Literal lowerLiteral = new Literal();
if (filterOperands.get(1).getLiteral().isSetStringValue()) {
lowerLiteral.setLongValue(new DateTimeFormatSpec("TIMESTAMP").fromFormatToMillis(
filterOperands.get(1).getLiteral().getStringValue()));
} else {
lowerLiteral.setLongValue(getLongValue(filterOperands.get(1)));
}
Expression lowerExpression = new Expression(ExpressionType.LITERAL);
lowerExpression.setLiteral(lowerLiteral);
operands.set(1, lowerExpression);
lowerMillis = dateTruncCeil(operands);
Literal upperLiteral = new Literal();
if (filterOperands.get(2).getLiteral().isSetStringValue()) {
upperLiteral.setLongValue(new DateTimeFormatSpec("TIMESTAMP").fromFormatToMillis(
ashishjayamohan marked this conversation as resolved.
Show resolved Hide resolved
filterOperands.get(2).getLiteral().getStringValue()));
} else {
upperLiteral.setLongValue(getLongValue(filterOperands.get(2)));
}
Expression upperExpression = new Expression(ExpressionType.LITERAL);
upperExpression.setLiteral(upperLiteral);
operands.set(1, upperExpression);
upperMillis = dateTruncCeil(operands);
break;
default:
throw new IllegalStateException();
ashishjayamohan marked this conversation as resolved.
Show resolved Hide resolved
}
lowerMillis = (lowerMillis == null) ? 0 : lowerMillis;
upperMillis = (upperMillis == null) ? Long.MAX_VALUE : upperMillis;

String rangeString = new Range(lowerMillis, lowerInclusive, upperMillis, upperInclusive).getRangeString();
rewriteToRange(filterFunction, dateTruncOperands.get(1), rangeString);
}

private boolean isStringLiteral(Expression expression) {
Literal literal = expression.getLiteral();
return literal != null && literal.isSetStringValue();
Expand Down Expand Up @@ -446,4 +542,21 @@ private static void rewriteToRange(Function filterFunction, Expression expressio
newOperands.add(RequestUtils.getLiteralExpression(rangeString));
filterFunction.setOperands(newOperands);
}
private static long dateTruncFloor(List<Expression> operands) {
String unit = operands.get(0).getLiteral().getStringValue();
long timeValue = operands.get(1).getLiteral().getLongValue();
String inputTimeUnit = (operands.size() >= 3) ? operands.get(2).getLiteral().getStringValue()
: TimeUnit.MILLISECONDS.name();
ISOChronology chronology = (operands.size() >= 4) ? DateTimeUtils.getChronology(TimeZoneKey
.getTimeZoneKey(operands.get(3).getLiteral().getStringValue())) : ISOChronology.getInstanceUTC();
String outputTimeUnit = (operands.size() == 5) ? operands.get(4).getLiteral().getStringValue() : inputTimeUnit;
return TimeUnit.valueOf(outputTimeUnit.toUpperCase()).convert(DateTimeUtils.getTimestampField(chronology, unit)
.roundFloor(TimeUnit.MILLISECONDS.convert(timeValue, TimeUnit.valueOf(inputTimeUnit.toUpperCase()))),
TimeUnit.MILLISECONDS);
}

private static long dateTruncCeil(List<Expression> operands) {
String unit = operands.get(0).getLiteral().getStringValue();
return dateTruncFloor(operands) + TimeUnit.MILLISECONDS.convert(1, TimeUnit.valueOf(unit.toUpperCase() + 'S')) - 1;
}
}
93 changes: 76 additions & 17 deletions ...t/java/org/apache/pinot/core/query/optimizer/filter/TimePredicateFilterOptimizerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,18 @@ public void testTimeConvert() {
new Range(1620777600000L, true, 1620864000000L, false));

// Other input format
testTimeConvert("timeConvert(col, 'MINUTES', 'SECONDS') > 1620830760", new Range(27013846L, false, null, false));
testTimeConvert("timeConvert(col, 'HOURS', 'MINUTES') < 27015286", new Range(null, false, 450254L, true));
testTimeConvert("timeConvert(col, 'MINUTES', 'SECONDS') > 1620830760",
ashishjayamohan marked this conversation as resolved.
Show resolved Hide resolved
new Range(27013846L, false, null, false));
testTimeConvert("timeConvert(col, 'HOURS', 'MINUTES') < 27015286",
new Range(null, false, 450254L, true));
testTimeConvert("timeConvert(col, 'DAYS', 'HOURS') BETWEEN 450230 AND 450254",
new Range(18759L, false, 18760L, true));
testTimeConvert("timeConvert(col, 'SECONDS', 'DAYS') = 18759", new Range(1620777600L, true, 1620864000L, false));
testTimeConvert("timeConvert(col, 'SECONDS', 'DAYS') = 18759",
new Range(1620777600L, true, 1620864000L, false));

// Invalid time
testInvalidTimeConvert("timeConvert(col, 'MINUTES', 'SECONDS') > 1620830760.5");
testInvalidTimeConvert("timeConvert(col, 'HOURS', 'MINUTES') > 1620830760");
testInvalidFilterOptimizer("timeConvert(col, 'MINUTES', 'SECONDS') > 1620830760.5");
testInvalidFilterOptimizer("timeConvert(col, 'HOURS', 'MINUTES') > 1620830760");
}

@Test
Expand Down Expand Up @@ -92,14 +95,17 @@ public void testEpochToEpochDateTimeConvert() {
new Range(1620831600000L, true, 1620833400000L, false));

// Other output format
testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES') > 27013846",
testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES')"
+ " > 27013846",
new Range(1620831600000L, true, null, false));
testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '10:MINUTES:EPOCH', '30:MINUTES') < 2701528",
testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '10:MINUTES:EPOCH', '30:MINUTES') "
+ "< 2701528",
new Range(null, false, 1620918000000L, false));
testTimeConvert(
"dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:SECONDS:EPOCH', '30:MINUTES') BETWEEN 1620830760 AND "
+ "1620917160", new Range(1620831600000L, true, 1620918000000L, false));
testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') > 900462",
testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') "
+ "> 900462",
new Range(1620833400000L, true, null, false));
testTimeConvert("dateTimeConvert(col, '1:MILLISECONDS:EPOCH', '1:HOURS:EPOCH', '30:MINUTES') < 450255",
new Range(null, false, 1620918000000L, false));
Expand All @@ -121,23 +127,25 @@ public void testEpochToEpochDateTimeConvert() {
new Range(1620833400L, true, null, false));
testTimeConvert("dateTimeConvert(col, '1:MINUTES:EPOCH', '1:HOURS:EPOCH', '30:MINUTES') < 450255",
new Range(null, false, 27015300L, false));
testTimeConvert("dateTimeConvert(col, '1:DAYS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') BETWEEN 18759 AND 18760",
testTimeConvert("dateTimeConvert(col, '1:DAYS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') "
+ "BETWEEN 18759 AND 18760",
new Range(18759L, true, 18761L, false));
testTimeConvert("dateTimeConvert(col, '1:DAYS:EPOCH', '1:DAYS:EPOCH', '30:MINUTES') = 18759",
new Range(18759L, true, 18760L, false));

// Invalid time
testInvalidTimeConvert("dateTimeConvert(col, '1:SECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES') > 27013846.5");
testInvalidTimeConvert("dateTimeConvert(col, '1:SECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') > 27013846");
testInvalidFilterOptimizer("dateTimeConvert(col, '1:SECONDS:EPOCH', '1:MINUTES:EPOCH', '30:MINUTES') > 27013846.5");
testInvalidFilterOptimizer("dateTimeConvert(col, '1:SECONDS:EPOCH', '30:MINUTES:EPOCH', '30:MINUTES') > 27013846");
}

@Test
public void testSDFToEpochDateTimeConvert() {
testTimeConvert(
"dateTimeConvert(col, '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', '1:MILLISECONDS:EPOCH', "
+ "'30:MINUTES') > 1620830760000", new Range("2021-05-12 15:00:00.000", true, null, false));
testTimeConvert("dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', '1:MILLISECONDS:EPOCH', "
+ "'30:MINUTES') < 1620917160000", new Range(null, false, "2021-05-13 15:00:00", false));
testTimeConvert("dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', "
+ "'1:MILLISECONDS:EPOCH', '30:MINUTES') < 1620917160000", new Range(null, false,
"2021-05-13 15:00:00", false));
testTimeConvert(
"dateTimeConvert(col, '1:MINUTES:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm', '1:MILLISECONDS:EPOCH', '30:MINUTES') "
+ "BETWEEN 1620830760000 AND 1620917160000",
Expand All @@ -147,14 +155,65 @@ public void testSDFToEpochDateTimeConvert() {
+ "1620830760000", new Range("2021-05-12", false, "2021-05-12", true));

// Invalid time
testInvalidTimeConvert(
testInvalidFilterOptimizer(
"dateTimeConvert(col, '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', '1:MILLISECONDS:EPOCH', "
+ "'30:MINUTES') > 1620830760000.5");
testInvalidTimeConvert(
testInvalidFilterOptimizer(
"dateTimeConvert(col, '1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss', '1:MILLISECONDS:EPOCH', "
+ "'30:MINUTES') < 1620917160");
}


@Test
public void testDateTruncOptimizer() {
testDateTrunc(
"datetrunc('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false));
ashishjayamohan marked this conversation as resolved.
Show resolved Hide resolved
testDateTrunc(
"dateTrunc('DAY', col) < 1620777600000", new Range("0", true, "1620777600000", false));
testDateTrunc(
"DATETRUNC('DAY', col) < 1620777600010", new Range("0", true, "1620863999999", true));
testDateTrunc(
"DATE_TRUNC('DAY', col) < 1620863999999", new Range("0", true, "1620863999999", true));

testDateTrunc(
"datetrunc('DAY', col) <= 1620777600000", new Range("0", true, "1620863999999", true));
testDateTrunc(
"datetrunc('DAY', col) <= 1620777600010", new Range("0", true, "1620863999999", true));

testDateTrunc(
"datetrunc('DAY', col) > 1620777600000", new Range("1620863999999", false, Long.MAX_VALUE, true));
testDateTrunc(
"dateTrunc('DAY', col) > 1620863999999", new Range("1620863999999", false, Long.MAX_VALUE, true));
testDateTrunc(
"DATETRUNC('DAY', col) > 1620864000000", new Range("1620950399999", false, Long.MAX_VALUE, true));

testDateTrunc(
"datetrunc('DAY', col) >= 1620863999999", new Range("1620863999999", false, Long.MAX_VALUE, true));
testDateTrunc(
"datetrunc('DAY', col) >= 1620777600000", new Range("1620777600000", true, Long.MAX_VALUE, true));

testDateTrunc(
"datetrunc('DAY', col) = 1620777600000", new Range("1620777600000", true, "1620863999999", true));
testDateTrunc(
"dateTrunc('DAY', col) = 1620777600001", new Range(Long.MAX_VALUE, true, Long.MIN_VALUE, true));
}

/**
* Helper method to test optimizing DATE_TRUNC on the given filter.
*/
private void testDateTrunc(String filterString, Range expectedRange) {
Expression originalExpression = CalciteSqlParser.compileToExpression(filterString);
Expression optimizedFilterExpression = OPTIMIZER.optimize(CalciteSqlParser.compileToExpression(filterString));
Function function = optimizedFilterExpression.getFunctionCall();
assertEquals(function.getOperator(), FilterKind.RANGE.name());
List<Expression> operands = function.getOperands();
assertEquals(operands.size(), 2);
assertEquals(operands.get(0),
originalExpression.getFunctionCall().getOperands().get(0).getFunctionCall().getOperands().get(1));
String rangeString = operands.get(1).getLiteral().getStringValue();
assertEquals(rangeString, expectedRange.getRangeString());
}

/**
* Helper method to test no-op TIME_CONVERT filter (same input and output time unit).
*/
Expand Down Expand Up @@ -192,9 +251,9 @@ private void testTimeConvert(String filterString, Range expectedRange) {
}

/**
* Helper method to test optimizing TIME_CONVERT/DATE_TIME_CONVERT with invalid time in filter.
* Helper method to test optimizing TIME_CONVERT/DATE_TIME_CONVERT/DATE_TRUNC with invalid time in filter.
*/
private void testInvalidTimeConvert(String filterString) {
private void testInvalidFilterOptimizer(String filterString) {
Expression originalExpression = CalciteSqlParser.compileToExpression(filterString);
Expression optimizedFilterExpression = OPTIMIZER.optimize(CalciteSqlParser.compileToExpression(filterString));
assertEquals(optimizedFilterExpression, originalExpression);
Expand Down
Loading