From 16a1b873da80e4a2b046007683144a7208f6c712 Mon Sep 17 00:00:00 2001 From: seawinde Date: Mon, 10 Feb 2025 20:44:01 +0800 Subject: [PATCH 1/5] [fix](mv) Fix sync mv add default select limit wrongly --- .../java/org/apache/doris/catalog/MTMV.java | 6 +- .../java/org/apache/doris/mtmv/MTMVCache.java | 16 +++-- .../org/apache/doris/mtmv/MTMVPlanUtil.java | 27 +++++--- .../mv/InitMaterializationContextHook.java | 6 +- .../exploration/mv/MaterializedViewUtils.java | 48 --------------- .../sql_default_limit/sql_default_limit.out | 8 +++ .../sql_default_limit.groovy | 61 +++++++++++++++++++ 7 files changed, 107 insertions(+), 65 deletions(-) create mode 100644 regression-test/data/mv_p0/sql_default_limit/sql_default_limit.out create mode 100644 regression-test/suites/mv_p0/sql_default_limit/sql_default_limit.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 19058df1eb904b..971a7049bfa3ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -201,7 +201,8 @@ public void addTaskResult(MTMVTask task, MTMVRelation relation, // to connection issues such as S3, so it is directly set to null if (!isReplay) { // shouldn't do this while holding mvWriteLock - mtmvCache = MTMVCache.from(this, MTMVPlanUtil.createMTMVContext(this), true, true); + mtmvCache = MTMVCache.from(this.getQuerySql(), MTMVPlanUtil.createMTMVContext(this), true, + true); } } catch (Throwable e) { mtmvCache = null; @@ -323,7 +324,8 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws Ana MTMVCache mtmvCache; try { // Should new context with ADMIN user - mtmvCache = MTMVCache.from(this, MTMVPlanUtil.createMTMVContext(this), true, false); + mtmvCache = MTMVCache.from(this.getQuerySql(), MTMVPlanUtil.createMTMVContext(this), true, + false); } finally { connectionContext.setThreadLocalInfo(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java index b185000c14897e..f5554f889d56e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java @@ -17,7 +17,6 @@ package org.apache.doris.mtmv; -import org.apache.doris.catalog.MTMV; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; @@ -87,16 +86,23 @@ public StructInfo getStructInfo() { return structInfo; } - public static MTMVCache from(MTMV mtmv, ConnectContext connectContext, boolean needCost, boolean needLock) { + /** + * @param defSql the def sql of materialization + * @param connectContext should create new connectContext use MTMVPlanUtil createMTMVContext + * or createBasicMvContext + * @param needCost the plan from def sql should calc cost or not + * @param needLock should skip lock when create mtmv cache + */ + public static MTMVCache from(String defSql, ConnectContext connectContext, boolean needCost, boolean needLock) { StatementContext mvSqlStatementContext = new StatementContext(connectContext, - new OriginStatement(mtmv.getQuerySql(), 0)); - if (needLock) { + new OriginStatement(defSql, 0)); + if (!needLock) { mvSqlStatementContext.setNeedLockTables(false); } if (mvSqlStatementContext.getConnectContext().getStatementContext() == null) { mvSqlStatementContext.getConnectContext().setStatementContext(mvSqlStatementContext); } - LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(mtmv.getQuerySql()); + LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(defSql); NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); boolean originalRewriteFlag = connectContext.getSessionVariable().enableMaterializedViewRewrite; connectContext.getSessionVariable().enableMaterializedViewRewrite = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 585d9d24830217..f15ae8db407bdc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -43,10 +43,24 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import javax.annotation.Nullable; public class MTMVPlanUtil { public static ConnectContext createMTMVContext(MTMV mtmv) { + ConnectContext ctx = createMTMVContext(null); + Optional workloadGroup = mtmv.getWorkloadGroup(); + if (workloadGroup.isPresent()) { + ctx.getSessionVariable().setWorkloadGroup(workloadGroup.get()); + } + // Set db&catalog to be used when creating materialized views to avoid SQL statements not writing the full path + // After https://github.com/apache/doris/pull/36543, + // After 1, this logic is no longer needed. This is to be compatible with older versions + setCatalogAndDb(ctx, mtmv); + return ctx; + } + + public static ConnectContext createBasicMvContext(@Nullable ConnectContext parentContext) { ConnectContext ctx = new ConnectContext(); ctx.setEnv(Env.getCurrentEnv()); ctx.setQualifiedUser(Auth.ADMIN_USER); @@ -65,18 +79,15 @@ public static ConnectContext createMTMVContext(MTMV mtmv) { String.join(",", ImmutableSet.of( "COMPRESSED_MATERIALIZE_AGG", "COMPRESSED_MATERIALIZE_SORT", RuleType.ADD_DEFAULT_LIMIT.name()))); - Optional workloadGroup = mtmv.getWorkloadGroup(); - if (workloadGroup.isPresent()) { - ctx.getSessionVariable().setWorkloadGroup(workloadGroup.get()); - } ctx.setStartTime(); - // Set db&catalog to be used when creating materialized views to avoid SQL statements not writing the full path - // After https://github.com/apache/doris/pull/36543, - // After 1, this logic is no longer needed. This is to be compatible with older versions - setCatalogAndDb(ctx, mtmv); + if (parentContext != null) { + ctx.changeDefaultCatalog(parentContext.getDefaultCatalog()); + ctx.setDatabase(parentContext.getDatabase()); + } return ctx; } + private static void setCatalogAndDb(ConnectContext ctx, MTMV mtmv) { EnvInfo envInfo = mtmv.getEnvInfo(); if (envInfo == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 7294136d3bce49..b6798be6d6362c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -28,6 +28,7 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVCache; +import org.apache.doris.mtmv.MTMVPlanUtil; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; @@ -256,8 +257,9 @@ private List createSyncMvContexts(OlapTable olapTable, LOG.warn(String.format("can't parse %s ", createMvSql)); continue; } - MTMVCache mtmvCache = MaterializedViewUtils.createMTMVCache(querySql.get(), - cascadesContext.getConnectContext()); + MTMVCache mtmvCache = MTMVCache.from(querySql.get(), + MTMVPlanUtil.createBasicMvContext(cascadesContext.getConnectContext()), true, + false); contexts.add(new SyncMaterializationContext(mtmvCache.getLogicalPlan(), mtmvCache.getOriginalPlan(), olapTable, meta.getIndexId(), indexName, cascadesContext, mtmvCache.getStatistics())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 20aad9ecdb25c6..f5254eb9772cdc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -26,21 +26,14 @@ import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.mtmv.BaseTableInfo; -import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.NereidsPlanner; -import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.StructInfoMap; -import org.apache.doris.nereids.parser.NereidsParser; -import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.analysis.BindRelation; import org.apache.doris.nereids.rules.expression.ExpressionNormalization; import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; -import org.apache.doris.nereids.rules.rewrite.EliminateSort; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; @@ -55,7 +48,6 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PreAggStatus; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; -import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; @@ -63,17 +55,13 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; -import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.trees.plans.visitor.NondeterministicFunctionCollector; import org.apache.doris.nereids.util.ExpressionUtils; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.SessionVariable; import com.google.common.collect.HashMultimap; @@ -312,42 +300,6 @@ public static List extractNondeterministicFunction(Plan plan) { return nondeterministicFunctions; } - /** - * createMTMVCache from querySql - */ - public static MTMVCache createMTMVCache(String querySql, ConnectContext connectContext) { - LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(querySql); - StatementContext mvSqlStatementContext = new StatementContext(connectContext, - new OriginStatement(querySql, 0)); - mvSqlStatementContext.setNeedLockTables(false); - NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); - if (mvSqlStatementContext.getConnectContext().getStatementContext() == null) { - mvSqlStatementContext.getConnectContext().setStatementContext(mvSqlStatementContext); - } - // Can not convert to table sink, because use the same column from different table when self join - // the out slot is wrong - planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainCommand.ExplainLevel.ALL_PLAN); - Plan originPlan = planner.getRewrittenPlan(); - // Eliminate result sink because sink operator is useless in query rewrite by materialized view - // and the top sort can also be removed - Plan mvPlan = originPlan.accept(new DefaultPlanRewriter() { - @Override - public Plan visitLogicalResultSink(LogicalResultSink logicalResultSink, Object context) { - return logicalResultSink.child().accept(this, context); - } - }, null); - // Optimize by rules to remove top sort - CascadesContext parentCascadesContext = CascadesContext.initContext(mvSqlStatementContext, mvPlan, - PhysicalProperties.ANY); - mvPlan = MaterializedViewUtils.rewriteByRules(parentCascadesContext, childContext -> { - Rewriter.getCteChildrenRewriter(childContext, - ImmutableList.of(Rewriter.custom(RuleType.ELIMINATE_SORT, EliminateSort::new))).execute(); - return childContext.getRewritePlan(); - }, mvPlan, originPlan); - return new MTMVCache(mvPlan, originPlan, planner.getAnalyzedPlan(), - planner.getCascadesContext().getMemo().getRoot().getStatistics(), null); - } - /** * Check the query if Contains query operator * Such sql as following should return true diff --git a/regression-test/data/mv_p0/sql_default_limit/sql_default_limit.out b/regression-test/data/mv_p0/sql_default_limit/sql_default_limit.out new file mode 100644 index 00000000000000..2e1553fc05e316 --- /dev/null +++ b/regression-test/data/mv_p0/sql_default_limit/sql_default_limit.out @@ -0,0 +1,8 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1 -- +1 5 + +-- !query2 -- +1 5 +2 1 + diff --git a/regression-test/suites/mv_p0/sql_default_limit/sql_default_limit.groovy b/regression-test/suites/mv_p0/sql_default_limit/sql_default_limit.groovy new file mode 100644 index 00000000000000..1897b74a7836b4 --- /dev/null +++ b/regression-test/suites/mv_p0/sql_default_limit/sql_default_limit.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite ("sql_default_limit") { + + String db = context.config.getDbNameByFile(context.file) + sql """use ${db}""" + + sql """ DROP TABLE IF EXISTS sql_default_limit_table; """ + + sql """ + create table sql_default_limit_table + ( + id1 int, + id2 int, + id3 int, + sale_date date, + sale_amt bigint + ) + distributed by hash(id1) + properties("replication_num" = "1"); + """ + + sql """insert into sql_default_limit_table values(1,1,1,'2020-02-02',1);""" + sql """insert into sql_default_limit_table values(1,1,1,'2020-02-02',1);""" + sql """insert into sql_default_limit_table values(1,1,1,'2020-02-02',1);""" + sql """insert into sql_default_limit_table values(1,1,1,'2020-02-02',1);""" + sql """insert into sql_default_limit_table values(2,1,1,'2020-02-02',1);""" + sql """insert into sql_default_limit_table values(1,1,1,'2020-02-02',1);""" + + create_sync_mv(db, "sql_default_limit_table", "test_mv", + """select id1, sum(sale_amt) from sql_default_limit_table group by id1""") + + sql """analyze table sql_default_limit_table with sync;""" + sql """alter table sql_default_limit_table modify column sale_amt set stats ('row_count'='6');""" + + + sql """set enable_stats=true;""" + sql """set sql_select_limit = 1;""" + mv_rewrite_success("select id1, sum(sale_amt) from sql_default_limit_table group by id1;", "test_mv") + order_qt_query1 """select id1, sum(sale_amt) from sql_default_limit_table group by id1;""" + sql """set sql_select_limit = -1;""" + + sql """set default_order_by_limit = 2;""" + mv_rewrite_success("select id1, sum(sale_amt) from sql_default_limit_table group by id1;", "test_mv") + order_qt_query2 """select id1, sum(sale_amt) from sql_default_limit_table group by id1;""" +} From 3a0b2090465f0802930f27a5cf37099543511748 Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 11 Feb 2025 14:37:46 +0800 Subject: [PATCH 2/5] fix test --- .../src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index f15ae8db407bdc..13a58d82901183 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -48,7 +48,7 @@ public class MTMVPlanUtil { public static ConnectContext createMTMVContext(MTMV mtmv) { - ConnectContext ctx = createMTMVContext(null); + ConnectContext ctx = createBasicMvContext(null); Optional workloadGroup = mtmv.getWorkloadGroup(); if (workloadGroup.isPresent()) { ctx.getSessionVariable().setWorkloadGroup(workloadGroup.get()); From e4cc63607c46e23b50d8a99236b6b66803480d23 Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 12 Feb 2025 14:43:00 +0800 Subject: [PATCH 3/5] fix test --- .../java/org/apache/doris/catalog/MTMV.java | 12 +--- .../java/org/apache/doris/mtmv/MTMVCache.java | 68 +++++++++++-------- .../mv/InitMaterializationContextHook.java | 2 +- .../sum_divede_count/sum_devide_count.out | 5 ++ .../test_dup_mv_repeat/test_dup_mv_repeat.out | 4 +- .../test_insert_multi/test_insert_multi.out | 8 ++- .../sum_divede_count/sum_devide_count.groovy | 7 +- .../test_dup_mv_repeat.groovy | 12 +++- .../test_insert_multi.groovy | 12 +++- 9 files changed, 86 insertions(+), 44 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 971a7049bfa3ee..c540ee5c5417dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -202,7 +202,7 @@ public void addTaskResult(MTMVTask task, MTMVRelation relation, if (!isReplay) { // shouldn't do this while holding mvWriteLock mtmvCache = MTMVCache.from(this.getQuerySql(), MTMVPlanUtil.createMTMVContext(this), true, - true); + true, null); } } catch (Throwable e) { mtmvCache = null; @@ -321,14 +321,8 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws Ana } // Concurrent situations may result in duplicate cache generation, // but we tolerate this in order to prevent nested use of readLock and write MvLock for the table - MTMVCache mtmvCache; - try { - // Should new context with ADMIN user - mtmvCache = MTMVCache.from(this.getQuerySql(), MTMVPlanUtil.createMTMVContext(this), true, - false); - } finally { - connectionContext.setThreadLocalInfo(); - } + MTMVCache mtmvCache = MTMVCache.from(this.getQuerySql(), MTMVPlanUtil.createMTMVContext(this), true, + false, connectionContext); writeMvLock(); try { this.cache = mtmvCache; diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java index f5554f889d56e2..517acb048e0a2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java @@ -88,13 +88,17 @@ public StructInfo getStructInfo() { /** * @param defSql the def sql of materialization - * @param connectContext should create new connectContext use MTMVPlanUtil createMTMVContext + * @param createCacheContext should create new createCacheContext use MTMVPlanUtil createMTMVContext * or createBasicMvContext * @param needCost the plan from def sql should calc cost or not - * @param needLock should skip lock when create mtmv cache + * @param needLock should lock when create mtmv cache + * @param currentContext current context, after create cache,should setThreadLocalInfo */ - public static MTMVCache from(String defSql, ConnectContext connectContext, boolean needCost, boolean needLock) { - StatementContext mvSqlStatementContext = new StatementContext(connectContext, + public static MTMVCache from(String defSql, + ConnectContext createCacheContext, + boolean needCost, boolean needLock, + ConnectContext currentContext) { + StatementContext mvSqlStatementContext = new StatementContext(createCacheContext, new OriginStatement(defSql, 0)); if (!needLock) { mvSqlStatementContext.setNeedLockTables(false); @@ -104,8 +108,8 @@ public static MTMVCache from(String defSql, ConnectContext connectContext, boole } LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(defSql); NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); - boolean originalRewriteFlag = connectContext.getSessionVariable().enableMaterializedViewRewrite; - connectContext.getSessionVariable().enableMaterializedViewRewrite = false; + boolean originalRewriteFlag = createCacheContext.getSessionVariable().enableMaterializedViewRewrite; + createCacheContext.getSessionVariable().enableMaterializedViewRewrite = false; try { // Can not convert to table sink, because use the same column from different table when self join // the out slot is wrong @@ -117,29 +121,39 @@ public static MTMVCache from(String defSql, ConnectContext connectContext, boole planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); } } finally { - connectContext.getSessionVariable().enableMaterializedViewRewrite = originalRewriteFlag; + createCacheContext.getSessionVariable().enableMaterializedViewRewrite = originalRewriteFlag; } - Plan originPlan = planner.getCascadesContext().getRewritePlan(); - // Eliminate result sink because sink operator is useless in query rewrite by materialized view - // and the top sort can also be removed - Plan mvPlan = originPlan.accept(new DefaultPlanRewriter() { - @Override - public Plan visitLogicalResultSink(LogicalResultSink logicalResultSink, Object context) { - return logicalResultSink.child().accept(this, context); + Plan originPlan; + Plan mvPlan; + Optional structInfoOptional; + try { + originPlan = planner.getCascadesContext().getRewritePlan(); + // Eliminate result sink because sink operator is useless in query rewrite by materialized view + // and the top sort can also be removed + mvPlan = originPlan.accept(new DefaultPlanRewriter() { + @Override + public Plan visitLogicalResultSink(LogicalResultSink logicalResultSink, + Object context) { + return logicalResultSink.child().accept(this, context); + } + }, null); + // Optimize by rules to remove top sort + CascadesContext parentCascadesContext = CascadesContext.initContext(mvSqlStatementContext, mvPlan, + PhysicalProperties.ANY); + mvPlan = MaterializedViewUtils.rewriteByRules(parentCascadesContext, childContext -> { + Rewriter.getCteChildrenRewriter(childContext, + ImmutableList.of(Rewriter.custom(RuleType.ELIMINATE_SORT, EliminateSort::new))).execute(); + return childContext.getRewritePlan(); + }, mvPlan, originPlan); + // Construct structInfo once for use later + structInfoOptional = MaterializationContext.constructStructInfo(mvPlan, originPlan, + planner.getCascadesContext(), + new BitSet()); + } finally { + if (currentContext != null) { + currentContext.setThreadLocalInfo(); } - }, null); - // Optimize by rules to remove top sort - CascadesContext parentCascadesContext = CascadesContext.initContext(mvSqlStatementContext, mvPlan, - PhysicalProperties.ANY); - mvPlan = MaterializedViewUtils.rewriteByRules(parentCascadesContext, childContext -> { - Rewriter.getCteChildrenRewriter(childContext, - ImmutableList.of(Rewriter.custom(RuleType.ELIMINATE_SORT, EliminateSort::new))).execute(); - return childContext.getRewritePlan(); - }, mvPlan, originPlan); - // Construct structInfo once for use later - Optional structInfoOptional = MaterializationContext.constructStructInfo(mvPlan, originPlan, - planner.getCascadesContext(), - new BitSet()); + } return new MTMVCache(mvPlan, originPlan, planner.getAnalyzedPlan(), needCost ? planner.getCascadesContext().getMemo().getRoot().getStatistics() : null, structInfoOptional.orElse(null)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index b6798be6d6362c..765cf620576627 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -259,7 +259,7 @@ private List createSyncMvContexts(OlapTable olapTable, } MTMVCache mtmvCache = MTMVCache.from(querySql.get(), MTMVPlanUtil.createBasicMvContext(cascadesContext.getConnectContext()), true, - false); + false, cascadesContext.getConnectContext()); contexts.add(new SyncMaterializationContext(mtmvCache.getLogicalPlan(), mtmvCache.getOriginalPlan(), olapTable, meta.getIndexId(), indexName, cascadesContext, mtmvCache.getStatistics())); diff --git a/regression-test/data/mv_p0/sum_divede_count/sum_devide_count.out b/regression-test/data/mv_p0/sum_divede_count/sum_devide_count.out index 67df77e0cc96f9..cc1909011d0cef 100644 --- a/regression-test/data/mv_p0/sum_divede_count/sum_devide_count.out +++ b/regression-test/data/mv_p0/sum_divede_count/sum_devide_count.out @@ -1,9 +1,14 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- -4 -4 -4 d +-4 -4 -4 d +1 1 1 a 1 1 1 a 2 2 2 b +2 2 2 b 3 -3 \N c +3 -3 \N c +3 2 \N c 3 2 \N c -- !select_mv -- diff --git a/regression-test/data/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.out b/regression-test/data/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.out index a958cf0b819cf5..991c3ff548c743 100644 --- a/regression-test/data/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.out +++ b/regression-test/data/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.out @@ -1,5 +1,5 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_mv -- -abc 123.0 -def 456.0 +abc 369.0 +def 1368.0 diff --git a/regression-test/data/mv_p0/test_insert_multi/test_insert_multi.out b/regression-test/data/mv_p0/test_insert_multi/test_insert_multi.out index 46af15050874a5..ca46e26a4b2311 100644 --- a/regression-test/data/mv_p0/test_insert_multi/test_insert_multi.out +++ b/regression-test/data/mv_p0/test_insert_multi/test_insert_multi.out @@ -1,9 +1,13 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- 1 1 1 2020-02-02 1 +1 1 1 2020-02-02 1 +1 1 1 2020-02-02 1 +1 2 2 2020-02-02 1 +1 2 2 2020-02-02 1 1 2 2 2020-02-02 1 -- !select_mv -- -1 1 -2 1 +1 3 +2 3 diff --git a/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy b/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy index 1784851e0470ff..239f0d09fc7b3e 100644 --- a/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy +++ b/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy @@ -34,17 +34,22 @@ suite ("sum_devide_count") { """ sql "insert into d_table select 1,1,1,'a';" + sql "insert into d_table select 1,1,1,'a';" + sql "insert into d_table select 2,2,2,'b';" sql "insert into d_table select 2,2,2,'b';" sql "insert into d_table select 3,-3,null,'c';" + sql "insert into d_table select 3,-3,null,'c';" createMV ("create materialized view kavg as select k1,k4,sum(k2),count(k2) from d_table group by k1,k4;") sql "insert into d_table select -4,-4,-4,'d';" + sql "insert into d_table select -4,-4,-4,'d';" + sql "insert into d_table select 3,2,null,'c';" sql "insert into d_table select 3,2,null,'c';" qt_select_star "select * from d_table order by k1,k2,k3,k4;" sql """analyze table d_table with sync;""" - sql """alter table d_table modify column k1 set stats ('row_count'='5');""" + sql """alter table d_table modify column k1 set stats ('row_count'='10');""" sql """set enable_stats=false;""" diff --git a/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy b/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy index 5ccdae1bd8a7c5..61c060aaa7a57c 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy @@ -35,7 +35,17 @@ suite ("test_dup_mv_repeat") { ); """ - sql "insert into db1 values('2020-01-01','abc',123),('2020-01-02','def',456);" + sql """ + insert into db1 values + ('2020-01-01','abc',123), + ('2020-01-01','abc',123), + ('2020-01-01','abc',123), + ('2020-01-02','def',456), + ('2020-01-02','def',456), + ('2020-01-02','def',456); + """ + + sql """alter table db1 modify column n set stats ('row_count'='6');""" createMV ("create materialized view dbviwe as select dt,s,sum(n) as n from db1 group by dt,s;") diff --git a/regression-test/suites/mv_p0/test_insert_multi/test_insert_multi.groovy b/regression-test/suites/mv_p0/test_insert_multi/test_insert_multi.groovy index cb68765a55d9c2..15b845f54122c5 100644 --- a/regression-test/suites/mv_p0/test_insert_multi/test_insert_multi.groovy +++ b/regression-test/suites/mv_p0/test_insert_multi/test_insert_multi.groovy @@ -27,7 +27,17 @@ suite ("test_insert_multi") { createMV ("create materialized view store_amt as select store_id, sum(sale_amt) from sales_records group by store_id;") - sql """insert into sales_records values(1,1,1,"2020-02-02",1),(1,2,2,"2020-02-02",1);""" + sql """insert into sales_records + values + (1,1,1,"2020-02-02",1), + (1,1,1,"2020-02-02",1), + (1,1,1,"2020-02-02",1), + (1,2,2,"2020-02-02",1), + (1,2,2,"2020-02-02",1), + (1,2,2,"2020-02-02",1); + """ + + sql """alter table sales_records modify column record_id set stats ('row_count'='6');""" qt_select_star "select * from sales_records order by 1,2;" From 5838ad5335c173cdcbfd372bade63c5c7fb17d66 Mon Sep 17 00:00:00 2001 From: seawinde Date: Thu, 13 Feb 2025 11:05:39 +0800 Subject: [PATCH 4/5] fix test --- .../sum_divede_count/sum_devide_count.out | 5 ++++ .../testAggQueryOnAggMV1.out | 30 +++++++++++-------- .../mv/newMv/sum_devide_count.out | 18 ++++++++++- .../sum_divede_count/sum_devide_count.groovy | 7 ++++- .../testAggQueryOnAggMV1.groovy | 8 ++++- .../mv/newMv/sum_devide_count.groovy | 18 ++++++++++- .../mv/ut/onlyGroupBy.groovy | 8 ++++- 7 files changed, 77 insertions(+), 17 deletions(-) diff --git a/regression-test/data/mv_p0/sum_divede_count/sum_devide_count.out b/regression-test/data/mv_p0/sum_divede_count/sum_devide_count.out index cc1909011d0cef..8e5dc6b3a8038c 100644 --- a/regression-test/data/mv_p0/sum_divede_count/sum_devide_count.out +++ b/regression-test/data/mv_p0/sum_divede_count/sum_devide_count.out @@ -2,12 +2,17 @@ -- !select_star -- -4 -4 -4 d -4 -4 -4 d +-4 -4 -4 d +1 1 1 a 1 1 1 a 1 1 1 a 2 2 2 b 2 2 2 b +2 2 2 b 3 -3 \N c 3 -3 \N c +3 -3 \N c +3 2 \N c 3 2 \N c 3 2 \N c diff --git a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out index f37d3ef54f31f6..b1f120c38594a9 100644 --- a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out +++ b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out @@ -2,30 +2,36 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 +2020-01-03 3 c 3 3 3 2020-01-03 3 c 3 3 3 -- !select_mv -- -2 1 -2 2 -3 3 +4 1 +6 2 +9 3 -- !select_mv -- -7 +19 -- !select_mv -- -1 2 -2 1 -3 1 +1 4 +2 3 +3 3 -- !select_mv -- -1 2 -2 2 -3 3 +1 4 +2 6 +3 9 -- !select_mv -- -1 2 +1 4 -- !select_mv -- -1 2 1 +1 4 1 diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/sum_devide_count.out b/regression-test/data/nereids_syntax_p0/mv/newMv/sum_devide_count.out index 67df77e0cc96f9..3cf1e8d95fc8b3 100644 --- a/regression-test/data/nereids_syntax_p0/mv/newMv/sum_devide_count.out +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/sum_devide_count.out @@ -1,9 +1,25 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- -4 -4 -4 d +-4 -4 -4 d +-4 -4 -4 d +-4 -4 -4 d +-4 -4 -4 d 1 1 1 a +1 1 1 a +1 1 1 a +1 1 1 a +2 2 2 b 2 2 2 b +2 2 2 b +2 2 2 b +3 -3 \N c +3 -3 \N c 3 -3 \N c +3 -3 \N c +3 2 \N c +3 2 \N c +3 2 \N c 3 2 \N c -- !select_mv -- @@ -25,5 +41,5 @@ c -0.5 d -4.0 -- !select_mv -- --0.4 +-0.5714285714285714 diff --git a/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy b/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy index 239f0d09fc7b3e..e5b6a625d2f333 100644 --- a/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy +++ b/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy @@ -35,21 +35,26 @@ suite ("sum_devide_count") { sql "insert into d_table select 1,1,1,'a';" sql "insert into d_table select 1,1,1,'a';" + sql "insert into d_table select 1,1,1,'a';" + sql "insert into d_table select 2,2,2,'b';" sql "insert into d_table select 2,2,2,'b';" sql "insert into d_table select 2,2,2,'b';" sql "insert into d_table select 3,-3,null,'c';" sql "insert into d_table select 3,-3,null,'c';" + sql "insert into d_table select 3,-3,null,'c';" createMV ("create materialized view kavg as select k1,k4,sum(k2),count(k2) from d_table group by k1,k4;") sql "insert into d_table select -4,-4,-4,'d';" sql "insert into d_table select -4,-4,-4,'d';" + sql "insert into d_table select -4,-4,-4,'d';" + sql "insert into d_table select 3,2,null,'c';" sql "insert into d_table select 3,2,null,'c';" sql "insert into d_table select 3,2,null,'c';" qt_select_star "select * from d_table order by k1,k2,k3,k4;" sql """analyze table d_table with sync;""" - sql """alter table d_table modify column k1 set stats ('row_count'='10');""" + sql """alter table d_table modify column k3 set stats ('row_count'='15');""" sql """set enable_stats=false;""" diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy index f39e8df6cabe6a..88e83ea6b6b639 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy @@ -33,10 +33,16 @@ suite ("testAggQueryOnAggMV1") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" + sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" + sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" -sql """alter table emps modify column time_col set stats ('row_count'='4');""" +sql """alter table emps modify column time_col set stats ('row_count'='9');""" createMV("create materialized view emps_mv as select deptno, sum(salary), max(commission) from emps group by deptno;") createMV("create materialized view emps_mv_count_key as select deptno, count(deptno) from emps group by deptno;") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy index 69fac070e42946..cfd1903531055b 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy @@ -31,7 +31,16 @@ suite ("sum_devide_count") { """ sql "insert into sum_devide_count select 1,1,1,'a';" + sql "insert into sum_devide_count select 1,1,1,'a';" + sql "insert into sum_devide_count select 1,1,1,'a';" + sql "insert into sum_devide_count select 1,1,1,'a';" + sql "insert into sum_devide_count select 2,2,2,'b';" sql "insert into sum_devide_count select 2,2,2,'b';" + sql "insert into sum_devide_count select 2,2,2,'b';" + sql "insert into sum_devide_count select 2,2,2,'b';" + sql "insert into sum_devide_count select 3,-3,null,'c';" + sql "insert into sum_devide_count select 3,-3,null,'c';" + sql "insert into sum_devide_count select 3,-3,null,'c';" sql "insert into sum_devide_count select 3,-3,null,'c';" sql "SET experimental_enable_nereids_planner=true" @@ -42,10 +51,17 @@ suite ("sum_devide_count") { sleep(3000) sql "insert into sum_devide_count select -4,-4,-4,'d';" + sql "insert into sum_devide_count select -4,-4,-4,'d';" + sql "insert into sum_devide_count select -4,-4,-4,'d';" + sql "insert into sum_devide_count select -4,-4,-4,'d';" + sql "insert into sum_devide_count select -4,-4,-4,'d';" + sql "insert into sum_devide_count select 3,2,null,'c';" + sql "insert into sum_devide_count select 3,2,null,'c';" + sql "insert into sum_devide_count select 3,2,null,'c';" sql "insert into sum_devide_count select 3,2,null,'c';" sql "analyze table sum_devide_count with sync;" - sql """alter table sum_devide_count modify column k1 set stats ('row_count'='5');""" + sql """alter table sum_devide_count modify column k1 set stats ('row_count'='20');""" sql """set enable_stats=false;""" diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/onlyGroupBy.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/onlyGroupBy.groovy index 227535abf6aafb..349f7ff5d896c0 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/onlyGroupBy.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/onlyGroupBy.groovy @@ -35,7 +35,13 @@ suite ("onlyGroupBy") { sql """insert into onlyGroupBy values("2020-01-01",1,"a",1,1,1);""" + sql """insert into onlyGroupBy values("2020-01-01",1,"a",1,1,1);""" + sql """insert into onlyGroupBy values("2020-01-01",1,"a",1,1,1);""" + sql """insert into onlyGroupBy values("2020-01-02",2,"b",2,2,2);""" sql """insert into onlyGroupBy values("2020-01-02",2,"b",2,2,2);""" + sql """insert into onlyGroupBy values("2020-01-02",2,"b",2,2,2);""" + sql """insert into onlyGroupBy values("2020-01-03",3,"c",3,3,3);""" + sql """insert into onlyGroupBy values("2020-01-03",3,"c",3,3,3);""" sql """insert into onlyGroupBy values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view onlyGroupBy_mv as select deptno, count(salary) from onlyGroupBy group by deptno;") @@ -43,7 +49,7 @@ suite ("onlyGroupBy") { sql """insert into onlyGroupBy values("2020-01-01",1,"a",1,1,1);""" sql "analyze table onlyGroupBy with sync;" - sql """alter table onlyGroupBy modify column time_col set stats ('row_count'='4');""" + sql """alter table onlyGroupBy modify column time_col set stats ('row_count'='9');""" sql """set enable_stats=false;""" mv_rewrite_success("select deptno from onlyGroupBy group by deptno;", "onlyGroupBy_mv") From 78091bb9a2fd310fa7c6dc31cb437b7e34f8db4e Mon Sep 17 00:00:00 2001 From: seawinde Date: Thu, 13 Feb 2025 15:03:41 +0800 Subject: [PATCH 5/5] fix test --- .../nereids_syntax_p0/mv/ut/aggOnAggMV1.out | 12 +++++++++--- .../nereids_syntax_p0/mv/ut/aggOnAggMV10.out | 17 ++++++++++------- .../nereids_syntax_p0/mv/ut/aggOnAggMV11.out | 9 ++++++--- .../nereids_syntax_p0/mv/ut/aggOnAggMV2.out | 13 +++++++++---- .../nereids_syntax_p0/mv/ut/aggOnAggMV3.out | 3 +++ .../nereids_syntax_p0/mv/ut/aggOnAggMV5.out | 3 +++ .../nereids_syntax_p0/mv/ut/aggOnAggMV6.out | 3 +++ .../nereids_syntax_p0/mv/ut/aggOnAggMV7.out | 3 +++ .../nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy | 8 +++++++- .../nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy | 5 ++++- .../nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy | 5 ++++- .../nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy | 7 ++++++- .../nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy | 5 ++++- .../nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy | 5 +++++ .../nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy | 5 ++++- .../nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy | 5 ++++- 16 files changed, 84 insertions(+), 24 deletions(-) diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV1.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV1.out index 6db240393b08dc..f8623191f75826 100644 --- a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV1.out +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV1.out @@ -2,11 +2,17 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 +2020-01-03 3 c 3 3 3 2020-01-03 3 c 3 3 3 -- !select_mv -- -2 1 -2 2 -3 3 +4 1 +6 2 +9 3 diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV10.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV10.out index 83bfded26fcd9b..cdd75b4e43d563 100644 --- a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV10.out +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV10.out @@ -2,15 +2,18 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 2020-01-02 2 b 2 2 2 2020-01-03 3 c 3 3 3 +2020-01-03 3 c 3 3 3 -- !select_mv -- -\N \N 8 -1 \N 3 -1 1 3 -2 \N 3 -2 2 3 -3 \N 4 -3 3 4 +\N \N 14 +1 \N 4 +1 1 4 +2 \N 5 +2 2 5 +3 \N 7 +3 3 7 diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV11.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV11.out index d0b1871ed2f217..f71f2fe6f05880 100644 --- a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV11.out +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV11.out @@ -2,11 +2,14 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 2020-01-02 2 b 2 2 2 2020-01-03 3 c 3 3 3 +2020-01-03 3 c 3 3 3 -- !select_mv -- -1 4 -2 2 -3 2 +1 6 +2 4 +3 4 diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV2.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV2.out index 94559632cba57e..279e2e65097066 100644 --- a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV2.out +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV2.out @@ -1,14 +1,19 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_emps_mv -- -2 9 -3 3 +2 20 +3 9 -- !select_star -- 2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 7 2 2020-01-02 2 b 2 7 2 2020-01-03 3 c 3 3 3 +2020-01-03 3 c 3 3 3 +2020-01-03 3 c 3 3 3 -- !select_mv -- -2 9 -3 3 +2 20 +3 9 diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV3.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV3.out index 791cf5fd8f70f5..035dd108ed9710 100644 --- a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV3.out +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV3.out @@ -1,8 +1,11 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 2020-01-02 2 b 2 2 2 2020-01-03 3 c 3 3 10 +2020-01-03 3 c 3 3 10 2020-01-04 4 d 21 4 4 2020-01-04 4 d 21 4 4 diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV5.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV5.out index b10e432dadb571..2b87fa7f2cf891 100644 --- a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV5.out +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV5.out @@ -2,8 +2,11 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 2020-01-02 2 b 2 2 2 2020-01-03 3 c 3 3 3 +2020-01-03 3 c 3 3 3 -- !select_mv -- diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV6.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV6.out index b10e432dadb571..2b87fa7f2cf891 100644 --- a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV6.out +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV6.out @@ -2,8 +2,11 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 2020-01-02 2 b 2 2 2 2020-01-03 3 c 3 3 3 +2020-01-03 3 c 3 3 3 -- !select_mv -- diff --git a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV7.out b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV7.out index b10e432dadb571..2b87fa7f2cf891 100644 --- a/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV7.out +++ b/regression-test/data/nereids_syntax_p0/mv/ut/aggOnAggMV7.out @@ -2,8 +2,11 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 2020-01-02 2 b 2 2 2 2020-01-03 3 c 3 3 3 +2020-01-03 3 c 3 3 3 -- !select_mv -- diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy index a8e96a8e1343c2..b91fcbd5b129a8 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy @@ -34,7 +34,13 @@ suite ("aggOnAggMV1") { """ sql """insert into aggOnAggMV1 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV1 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV1 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV1 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV1 values("2020-01-02",2,"b",2,2,2);""" + sql """insert into aggOnAggMV1 values("2020-01-02",2,"b",2,2,2);""" + sql """insert into aggOnAggMV1 values("2020-01-03",3,"c",3,3,3);""" + sql """insert into aggOnAggMV1 values("2020-01-03",3,"c",3,3,3);""" sql """insert into aggOnAggMV1 values("2020-01-03",3,"c",3,3,3);""" @@ -45,7 +51,7 @@ suite ("aggOnAggMV1") { sql """insert into aggOnAggMV1 values("2020-01-01",1,"a",1,1,1);""" sql "analyze table aggOnAggMV1 with sync;" - sql """alter table aggOnAggMV1 modify column time_col set stats ('row_count'='4');""" + sql """alter table aggOnAggMV1 modify column time_col set stats ('row_count'='9');""" sql """set enable_stats=false;""" diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy index 4af39a4cfcea4a..763c1bfa3257a1 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy @@ -34,8 +34,11 @@ suite ("aggOnAggMV10") { """ sql """insert into aggOnAggMV10 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV10 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV10 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV10 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV10 values("2020-01-03",3,"c",3,3,3);""" + sql """insert into aggOnAggMV10 values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view aggOnAggMV10_mv as select deptno, commission, sum(salary) from aggOnAggMV10 group by deptno, commission;") @@ -44,7 +47,7 @@ suite ("aggOnAggMV10") { sql """insert into aggOnAggMV10 values("2020-01-01",1,"a",1,1,1);""" sql "analyze table aggOnAggMV10 with sync;" - sql """alter table aggOnAggMV10 modify column time_col set stats ('row_count'='4');""" + sql """alter table aggOnAggMV10 modify column time_col set stats ('row_count'='7');""" sql """set enable_stats=false;""" diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy index 56a47122b5ab28..e09fb0f190e377 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy @@ -35,8 +35,11 @@ suite ("aggOnAggMV11") { sql """insert into aggOnAggMV11 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV11 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV11 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV11 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV11 values("2020-01-03",3,"c",3,3,3);""" + sql """insert into aggOnAggMV11 values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view aggOnAggMV11_mv as select deptno, count(salary) from aggOnAggMV11 group by deptno;") @@ -45,7 +48,7 @@ suite ("aggOnAggMV11") { sql """insert into aggOnAggMV11 values("2020-01-01",1,"a",1,1,1);""" sql "analyze table aggOnAggMV11 with sync;" - sql """alter table aggOnAggMV11 modify column time_col set stats ('row_count'='4');""" + sql """alter table aggOnAggMV11 modify column time_col set stats ('row_count'='7');""" sql """set enable_stats=false;""" diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy index 8393af7ebfd2b2..941dd0e04bb117 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy @@ -32,7 +32,12 @@ suite ("aggOnAggMV2") { sql """insert into aggOnAggMV2 values("2020-01-02",2,"b",2,2,2);""" + sql """insert into aggOnAggMV2 values("2020-01-02",2,"b",2,2,2);""" + sql """insert into aggOnAggMV2 values("2020-01-02",2,"b",2,2,2);""" + sql """insert into aggOnAggMV2 values("2020-01-03",3,"c",3,3,3);""" sql """insert into aggOnAggMV2 values("2020-01-03",3,"c",3,3,3);""" + sql """insert into aggOnAggMV2 values("2020-01-03",3,"c",3,3,3);""" + sql """insert into aggOnAggMV2 values("2020-01-02",2,"b",2,7,2);""" sql """insert into aggOnAggMV2 values("2020-01-02",2,"b",2,7,2);""" explain { @@ -46,7 +51,7 @@ suite ("aggOnAggMV2") { sleep(3000) sql "analyze table aggOnAggMV2 with sync;" - sql """alter table aggOnAggMV2 modify column time_col set stats ('row_count'='3');""" + sql """alter table aggOnAggMV2 modify column time_col set stats ('row_count'='8');""" sql """set enable_stats=false;""" diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy index 48e15d4b10d52d..0f449071269e61 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy @@ -34,8 +34,11 @@ suite ("aggOnAggMV3") { """ sql """insert into aggOnAggMV3 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV3 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV3 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV3 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV3 values("2020-01-03",3,"c",3,3,10);""" + sql """insert into aggOnAggMV3 values("2020-01-03",3,"c",3,3,10);""" sql """insert into aggOnAggMV3 values("2020-01-04",4,"d",21,4,4);""" sql """insert into aggOnAggMV3 values("2020-01-04",4,"d",21,4,4);""" @@ -46,7 +49,7 @@ suite ("aggOnAggMV3") { sleep(3000) sql "analyze table aggOnAggMV3 with sync;" - sql """alter table aggOnAggMV3 modify column time_col set stats ('row_count'='5');""" + sql """alter table aggOnAggMV3 modify column time_col set stats ('row_count'='8');""" sql """set enable_stats=false;""" diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy index d90057b8bf5120..7851901b04ef3d 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy @@ -36,14 +36,19 @@ suite ("aggOnAggMV5") { sql """alter table aggOnAggMV5 modify column time_col set stats ('row_count'='4');""" sql """insert into aggOnAggMV5 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV5 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV5 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV5 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV5 values("2020-01-03",3,"c",3,3,3);""" + sql """insert into aggOnAggMV5 values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view aggOnAggMV5_mv as select deptno, commission, sum(salary) from aggOnAggMV5 group by deptno, commission;") sql """insert into aggOnAggMV5 values("2020-01-01",1,"a",1,1,1);""" sql "analyze table aggOnAggMV5 with sync;" + sql """alter table aggOnAggMV5 modify column commission set stats ('row_count'='8');""" + mv_rewrite_fail("select * from aggOnAggMV5 order by empid;", "aggOnAggMV5_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy index 37029f0f089098..30c5518fa7a0fa 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy @@ -34,8 +34,11 @@ suite ("aggOnAggMV6") { """ sql """insert into aggOnAggMV6 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV6 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV6 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV6 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV6 values("2020-01-03",3,"c",3,3,3);""" + sql """insert into aggOnAggMV6 values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view aggOnAggMV6_mv as select deptno, commission, sum(salary) from aggOnAggMV6 group by deptno, commission;") @@ -44,7 +47,7 @@ suite ("aggOnAggMV6") { sql """insert into aggOnAggMV6 values("2020-01-01",1,"a",1,1,1);""" sql "analyze table aggOnAggMV6 with sync;" - sql """alter table aggOnAggMV6 modify column time_col set stats ('row_count'='4');""" + sql """alter table aggOnAggMV6 modify column time_col set stats ('row_count'='6');""" sql """set enable_stats=false;""" diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy index d4db9a05fea0a6..e1339cd122526c 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy @@ -34,8 +34,11 @@ suite ("aggOnAggMV7") { """ sql """insert into aggOnAggMV7 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV7 values("2020-01-01",1,"a",1,1,1);""" + sql """insert into aggOnAggMV7 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV7 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV7 values("2020-01-03",3,"c",3,3,3);""" + sql """insert into aggOnAggMV7 values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view aggOnAggMV7_mv as select deptno, commission, sum(salary) from aggOnAggMV7 group by deptno, commission;") @@ -44,7 +47,7 @@ suite ("aggOnAggMV7") { sql """insert into aggOnAggMV7 values("2020-01-01",1,"a",1,1,1);""" sql "analyze table aggOnAggMV7 with sync;" - sql """alter table aggOnAggMV7 modify column time_col set stats ('row_count'='4');""" + sql """alter table aggOnAggMV7 modify column time_col set stats ('row_count'='7');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from aggOnAggMV7 order by empid;", "aggOnAggMV7_mv")