[opt](nereids) some topn-join rules are both used in rbo and cbo (apa…

…che#46773) ### What problem does this PR solve? the following rules only applies on pattern: topn->outerJoin if they are used a rbo rules, we miss the opportunity to optimize the plan, when the initial plan pattern is topn->innerJoin. to utilize the join reorder, the are moved to cbo rules, and when bottom outer join reorders as the root of join cluster, these rules could be applied. PushDownTopNThroughJoin PushDownLimitDistinctThroughJoin PushDownTopNDistinctThroughJoin
amorynan · Jan 15, 2025 · 9c3b634 · 9c3b634
1 parent aeebe49
commit 9c3b634
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 1 deletion.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -107,7 +107,10 @@
 import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughSort;
 import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughWindow;
 import org.apache.doris.nereids.rules.rewrite.PushDownJoinOtherCondition;
+import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughJoin;
 import org.apache.doris.nereids.rules.rewrite.PushDownProjectThroughLimit;
+import org.apache.doris.nereids.rules.rewrite.PushDownTopNDistinctThroughJoin;
+import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughJoin;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableList.Builder;
@@ -132,6 +135,9 @@ public class RuleSet {
             .add(PushDownProjectThroughInnerOuterJoin.INSTANCE)
             .add(PushDownProjectThroughSemiJoin.INSTANCE)
             .add(TransposeAggSemiJoinProject.INSTANCE)
+            .addAll(new PushDownTopNThroughJoin().buildRules())
+            .addAll(new PushDownLimitDistinctThroughJoin().buildRules())
+            .addAll(new PushDownTopNDistinctThroughJoin().buildRules())
             .build();
 
     public static final List<RuleFactory> PUSH_DOWN_FILTERS = ImmutableList.of(

diff --git a/...rc/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoin.java b/...rc/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoin.java
@@ -25,6 +25,7 @@
 import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
 import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.qe.ConnectContext;
 
 import com.google.common.collect.ImmutableList;
 
@@ -42,6 +43,10 @@ public List<Rule> buildRules() {
                 // limit -> distinct -> join
                 logicalLimit(logicalAggregate(logicalJoin())
                         .when(LogicalAggregate::isDistinct))
+                        .when(limit ->
+                                ConnectContext.get() != null
+                                        && ConnectContext.get().getSessionVariable().topnOptLimitThreshold
+                                        >= limit.getLimit() + limit.getOffset())
                         .then(limit -> {
                             LogicalAggregate<LogicalJoin<Plan, Plan>> agg = limit.child();
                             LogicalJoin<Plan, Plan> join = agg.child();

diff --git a/...src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoin.java b/...src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoin.java
@@ -27,6 +27,7 @@
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
 import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
 import org.apache.doris.nereids.util.PlanUtils;
+import org.apache.doris.qe.ConnectContext;
 
 import com.google.common.collect.ImmutableList;
 
@@ -45,6 +46,10 @@ public List<Rule> buildRules() {
                 // topN -> join
                 logicalTopN(logicalAggregate(logicalJoin()).when(LogicalAggregate::isDistinct))
                         // TODO: complex order by
+                        .when(topn ->
+                                ConnectContext.get() != null
+                                        && ConnectContext.get().getSessionVariable().topnOptLimitThreshold
+                                        >= topn.getLimit() + topn.getOffset())
                         .when(topN -> topN.getOrderKeys().stream().map(OrderKey::getExpr)
                                 .allMatch(Slot.class::isInstance))
                         .then(topN -> {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoin.java
@@ -25,6 +25,7 @@
 import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
 import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
+import org.apache.doris.qe.ConnectContext;
 
 import com.google.common.collect.ImmutableList;
 
@@ -43,6 +44,10 @@ public List<Rule> buildRules() {
                 // topN -> join
                 logicalTopN(logicalJoin())
                         // TODO: complex orderby
+                        .when(topn ->
+                                ConnectContext.get() != null
+                                        && ConnectContext.get().getSessionVariable().topnOptLimitThreshold
+                                        >= topn.getLimit() + topn.getOffset())
                         .when(topN -> topN.getOrderKeys().stream().map(OrderKey::getExpr)
                                 .allMatch(Slot.class::isInstance))
                         .then(topN -> {
@@ -102,7 +107,6 @@ private Plan pushLimitThroughJoin(LogicalTopN<? extends Plan> topN, LogicalJoin<
                 }
                 return null;
             case CROSS_JOIN:
-
                 if (join.left().getOutputSet().containsAll(orderbySlots)) {
                     return join.withChildren(
                             topN.withLimitChild(topN.getLimit() + topN.getOffset(), 0, join.left()),