Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add support for distinct aggregates #1261

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

feat: Add support for distinct aggregates #1261

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ca593a2
Add support for distinct aggregates
andygrove Jan 10, 2025
160b57f
clippy
andygrove Jan 10, 2025
faa62bf
fixes
andygrove Jan 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 11 additions & 10 deletions native/core/src/execution/planner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ use jni::objects::GlobalRef;
use num::{BigInt, ToPrimitive};
use std::cmp::max;
use std::{collections::HashMap, sync::Arc};
use datafusion_expr::test::function_stub::count_udaf;

// For clippy error on type_complexity.
type PhyAggResult = Result<Vec<AggregateFunctionExpr>, ExecutionError>;
Expand Down Expand Up @@ -1521,7 +1522,7 @@ impl PhysicalPlanner {
.schema(schema)
.alias("count")
.with_ignore_nulls(false)
.with_distinct(false)
.with_distinct(spark_expr.distinct)
.build()
.map_err(|e| ExecutionError::DataFusionError(e.to_string()))
}
Expand All @@ -1534,7 +1535,7 @@ impl PhysicalPlanner {
.schema(schema)
.alias("min")
.with_ignore_nulls(false)
.with_distinct(false)
.with_distinct(spark_expr.distinct)
.build()
.map_err(|e| ExecutionError::DataFusionError(e.to_string()))
}
Expand All @@ -1547,7 +1548,7 @@ impl PhysicalPlanner {
.schema(schema)
.alias("max")
.with_ignore_nulls(false)
.with_distinct(false)
.with_distinct(spark_expr.distinct)
.build()
.map_err(|e| ExecutionError::DataFusionError(e.to_string()))
}
Expand All @@ -1572,7 +1573,7 @@ impl PhysicalPlanner {
.schema(schema)
.alias("sum")
.with_ignore_nulls(false)
.with_distinct(false)
.with_distinct(spark_expr.distinct)
.build()
.map_err(|e| e.into())
}
Expand Down Expand Up @@ -1600,7 +1601,7 @@ impl PhysicalPlanner {
.schema(schema)
.alias("avg")
.with_ignore_nulls(false)
.with_distinct(false)
.with_distinct(spark_expr.distinct)
.build()
.map_err(|e| e.into())
}
Expand All @@ -1612,7 +1613,7 @@ impl PhysicalPlanner {
.schema(schema)
.alias("first")
.with_ignore_nulls(false)
.with_distinct(false)
.with_distinct(spark_expr.distinct)
.build()
.map_err(|e| e.into())
}
Expand All @@ -1624,7 +1625,7 @@ impl PhysicalPlanner {
.schema(schema)
.alias("last")
.with_ignore_nulls(false)
.with_distinct(false)
.with_distinct(spark_expr.distinct)
.build()
.map_err(|e| e.into())
}
Expand All @@ -1635,7 +1636,7 @@ impl PhysicalPlanner {
.schema(schema)
.alias("bit_and")
.with_ignore_nulls(false)
.with_distinct(false)
.with_distinct(spark_expr.distinct)
.build()
.map_err(|e| e.into())
}
Expand All @@ -1646,7 +1647,7 @@ impl PhysicalPlanner {
.schema(schema)
.alias("bit_or")
.with_ignore_nulls(false)
.with_distinct(false)
.with_distinct(spark_expr.distinct)
.build()
.map_err(|e| e.into())
}
Expand All @@ -1657,7 +1658,7 @@ impl PhysicalPlanner {
.schema(schema)
.alias("bit_xor")
.with_ignore_nulls(false)
.with_distinct(false)
.with_distinct(spark_expr.distinct)
.build()
.map_err(|e| e.into())
}
Expand Down
1 change: 1 addition & 0 deletions native/proto/src/proto/expr.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ message Expr {
}

message AggExpr {
bool distinct = 1;
oneof expr_struct {
Count count = 2;
Sum sum = 3;
Expand Down
7 changes: 7 additions & 0 deletions spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,7 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde with CometExprShim
ExprOuterClass.AggExpr
.newBuilder()
.setSum(sumBuilder)
.setDistinct(aggExpr.isDistinct)
.build())
} else {
if (dataType.isEmpty) {
Expand Down Expand Up @@ -432,6 +433,11 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde with CometExprShim
None
}
case Count(children) =>
if (children.length > 1 && aggExpr.isDistinct) {
withInfo(aggExpr, "no support for count distinct with multiple expressions")
return None
}

val exprChildren = children.map(exprToProto(_, inputs, binding))

if (exprChildren.forall(_.isDefined)) {
Expand All @@ -442,6 +448,7 @@ object QueryPlanSerde extends Logging with ShimQueryPlanSerde with CometExprShim
ExprOuterClass.AggExpr
.newBuilder()
.setCount(countBuilder)
.setDistinct(aggExpr.isDistinct)
.build())
} else {
withInfo(aggExpr, children: _*)
Expand Down
6 changes: 3 additions & 3 deletions spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
val cometShuffles = collect(df2.queryExecution.executedPlan) {
case _: CometShuffleExchangeExec => true
}
if (shuffleMode == "jvm") {
if (shuffleMode == "jvm" || shuffleMode == "auto") {
assert(cometShuffles.length == 1)
} else {
// we fall back to Spark for shuffle because we do not support
Expand Down Expand Up @@ -608,8 +608,8 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
withView("v") {
sql("CREATE TEMP VIEW v AS SELECT _1, _2 FROM tbl ORDER BY _1")
checkSparkAnswer(
"SELECT _2, SUM(_1), SUM(DISTINCT _1), MIN(_1), MAX(_1), COUNT(_1)," +
" COUNT(DISTINCT _1), AVG(_1), FIRST(_1), LAST(_1) FROM v GROUP BY _2")
"SELECT _2, SUM(_1), SUM(DISTINCT _2), MIN(_1), MAX(_1), COUNT(_1)," +
" COUNT(DISTINCT _2), AVG(_1), FIRST(_1), LAST(_1) FROM v GROUP BY _2 ORDER BY _2")
}
}
}
Expand Down
Loading