From 4bd106b535e28ddaf2e3d8d93f75698ccd947d4d Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 15 Jan 2025 20:34:13 +0800 Subject: [PATCH] [test](mtmv) Fix sync mv not partition in rewrite test and some other test problems (#46546) ### What problem does this PR solve? Fix sync mv not partition in rewrite because thought mv is already built wrongly In the `createMV()` method of `Suite.groovy`, it checks whether the most recent materialized view of the current database's last table has been built successfully. If a database has two tables, the synchronization materialized view of the second table may not be completed, which could incorrectly lead to the assumption that the build is complete. To address this issue, modify the test case to ensure that there is at most one table per database. --- .../doris/nereids/stats/StatsCalculator.java | 4 +- .../unique_rewrite.out | 0 .../mv/newMv/multi_slot4.out | 14 ++- .../test_show_create_materialized_view.out | 2 +- .../doris/regression/suite/Suite.groovy | 96 +++++++++++++------ .../suites/auth_call/test_ddl_mv_auth.groovy | 4 +- .../auth_p0/test_select_column_auth.groovy | 2 +- .../test_mv_case/test_mv_case.groovy | 13 ++- .../create_view_use_mv.groovy | 4 +- .../suites/mtmv_p0/test_iceberg_mtmv.groovy | 28 +++--- .../test_paimon_olap_rewrite_mtmv.groovy | 8 ++ .../suites/mv_p0/no_await/no_await.groovy | 39 ++++---- .../unique_rewrite.groovy | 0 .../mv/date_trunc/mv_with_date_trunc.groovy | 2 +- .../mv/nested_mtmv/nested_mtmv.groovy | 6 +- .../mv/newMv/multi_slot4.groovy | 10 +- .../test_show_create_materialized_view.groovy | 7 +- .../test_dup_mv_schema_change.groovy | 4 +- .../test_uniq_mv_schema_change.groovy | 21 +--- 19 files changed, 158 insertions(+), 106 deletions(-) rename regression-test/data/mv_p0/{unique => unique_rewrite}/unique_rewrite.out (100%) rename regression-test/suites/mv_p0/{unique => unique_rewrite}/unique_rewrite.groovy (100%) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 64019818f5d1e7..5daf3bb254fc24 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -501,8 +501,8 @@ private Statistics computeOlapScan(OlapScan olapScan) { // mv is selected, return its estimated stats Optional optStats = cascadesContext.getStatementContext() .getStatistics(((Relation) olapScan).getRelationId()); - LOG.info("computeOlapScan optStats isPresent {}, tableRowCount is {}", - optStats.isPresent(), tableRowCount); + LOG.info("computeOlapScan optStats isPresent {}, tableRowCount is {}, table name is {}", + optStats.isPresent(), tableRowCount, olapTable.getQualifiedName()); if (optStats.isPresent()) { double selectedPartitionsRowCount = getSelectedPartitionRowCount(olapScan, tableRowCount); LOG.info("computeOlapScan optStats is {}, selectedPartitionsRowCount is {}", optStats.get(), diff --git a/regression-test/data/mv_p0/unique/unique_rewrite.out b/regression-test/data/mv_p0/unique_rewrite/unique_rewrite.out similarity index 100% rename from regression-test/data/mv_p0/unique/unique_rewrite.out rename to regression-test/data/mv_p0/unique_rewrite/unique_rewrite.out diff --git a/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out index 5b500067986366..264a653fd3a3d0 100644 --- a/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out +++ b/regression-test/data/nereids_syntax_p0/mv/newMv/multi_slot4.out @@ -1,14 +1,22 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- -4 -4 -4 d +-4 -4 -4 d +-4 -4 -4 d +1 1 1 a +1 1 1 a 1 1 1 a 2 2 2 b +2 2 2 b +2 2 2 b +3 -3 \N c +3 -3 \N c 3 -3 \N c 3 -3 \N c -- !select_mv -- --3 1 -2 7 -3 9 +-3 3 +2 21 +3 27 4 \N diff --git a/regression-test/data/query_p0/show/test_show_create_materialized_view.out b/regression-test/data/query_p0/show/test_show_create_materialized_view.out index 040ebf56f29fc5..1d874bb14d3c5a 100644 --- a/regression-test/data/query_p0/show/test_show_create_materialized_view.out +++ b/regression-test/data/query_p0/show/test_show_create_materialized_view.out @@ -1,4 +1,4 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !cmd -- -table_for_mv_test mv_show_create_materialized_view CREATE MATERIALIZED VIEW mv_show_create_materialized_view AS\n SELECT id, name, SUM(value) AS total_value\n FROM table_for_mv_test\n GROUP BY id, name;\n +table_for_mv_test mv_show_create_materialized_view \n CREATE MATERIALIZED VIEW mv_show_create_materialized_view \n AS \n SELECT id, name, SUM(value) AS total_value\n FROM table_for_mv_test\n GROUP BY id, name\n diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 5cbed97829c990..155cb5868bce59 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -742,10 +742,16 @@ class Suite implements GroovyInterceptable { return result } + // Should use create_sync_mv, this method only check the sync mv in current db + // If has multi sync mv in db, may make mistake + @Deprecated void createMV(String sql) { (new CreateMVAction(context, sql)).run() } + // Should use create_sync_mv, this method only check the sync mv in current db + // If has multi sync mv in db, may make mistake + @Deprecated void createMV(String sql, String expection) { (new CreateMVAction(context, sql, expection)).run() } @@ -1475,80 +1481,101 @@ class Suite implements GroovyInterceptable { return debugPoint } - void waitingMTMVTaskFinishedByMvName(String mvName) { + def waitingMTMVTaskFinishedByMvName = { mvName, dbName = context.dbName -> Thread.sleep(2000); - String showTasks = "select TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from tasks('type'='mv') where MvName = '${mvName}' order by CreateTime ASC" + String showTasks = "select TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from tasks('type'='mv') where MvDatabaseName = '${dbName}' and MvName = '${mvName}' order by CreateTime DESC LIMIT 1" String status = "NULL" List> result long startTime = System.currentTimeMillis() long timeoutTimestamp = startTime + 5 * 60 * 1000 // 5 min - do { + List toCheckTaskRow = new ArrayList<>(); + while (timeoutTimestamp > System.currentTimeMillis() && (status != "SUCCESS")) { result = sql(showTasks) - logger.info("result: " + result.toString()) - if (!result.isEmpty()) { - status = result.last().get(4) - } + logger.info("current db is " + dbName + ", showTasks is " + showTasks) + if (result.isEmpty()) { + logger.info("waitingMTMVTaskFinishedByMvName toCheckTaskRow is empty") + Thread.sleep(1000); + continue; + } + toCheckTaskRow = result.get(0); + status = toCheckTaskRow.get(4) logger.info("The state of ${showTasks} is ${status}") Thread.sleep(1000); - } while (timeoutTimestamp > System.currentTimeMillis() && (status == 'PENDING' || status == 'RUNNING' || status == 'NULL')) + } if (status != "SUCCESS") { logger.info("status is not success") } Assert.assertEquals("SUCCESS", status) def show_tables = sql """ - show tables from ${result.last().get(6)}; + show tables from ${toCheckTaskRow.get(6)}; """ - def db_id = getDbId(result.last().get(6)) - def table_id = getTableId(result.last().get(6), mvName) + def db_id = getDbId(toCheckTaskRow.get(6)) + def table_id = getTableId(toCheckTaskRow.get(6), mvName) logger.info("waitingMTMVTaskFinished analyze mv name is " + mvName - + ", db name is " + result.last().get(6) + + ", db name is " + toCheckTaskRow.get(6) + ", show_tables are " + show_tables + ", db_id is " + db_id + ", table_id " + table_id) - sql "analyze table ${result.last().get(6)}.${mvName} with sync;" + sql "analyze table ${toCheckTaskRow.get(6)}.${mvName} with sync;" } - void waitingMTMVTaskFinishedByMvNameAllowCancel(String mvName) { + def waitingMTMVTaskFinishedByMvNameAllowCancel = {mvName, dbName = context.dbName -> Thread.sleep(2000); - String showTasks = "select TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from tasks('type'='mv') where MvName = '${mvName}' order by CreateTime ASC" + String showTasks = "select TaskId,JobId,JobName,MvId,Status,MvName,MvDatabaseName,ErrorMsg from tasks('type'='mv') where MvDatabaseName = '${dbName}' and MvName = '${mvName}' order by CreateTime DESC LIMIT 1" + String status = "NULL" List> result long startTime = System.currentTimeMillis() long timeoutTimestamp = startTime + 5 * 60 * 1000 // 5 min - do { + List toCheckTaskRow = new ArrayList<>(); + while (timeoutTimestamp > System.currentTimeMillis() && (status != "SUCCESS")) { result = sql(showTasks) - logger.info("result: " + result.toString()) - if (!result.isEmpty()) { - status = result.last().get(4) - } + logger.info("current db is " + dbName + ", showTasks result: " + result.toString()) + if (result.isEmpty()) { + logger.info("waitingMTMVTaskFinishedByMvName toCheckTaskRow is empty") + Thread.sleep(1000); + continue; + } + toCheckTaskRow = result.get(0) + status = toCheckTaskRow.get(4) logger.info("The state of ${showTasks} is ${status}") Thread.sleep(1000); - } while (timeoutTimestamp > System.currentTimeMillis() && (status == 'PENDING' || status == 'RUNNING' || status == 'NULL' || status == 'CANCELED')) + } if (status != "SUCCESS") { logger.info("status is not success") assertTrue(result.toString().contains("same table")) } // Need to analyze materialized view for cbo to choose the materialized view accurately - logger.info("waitingMTMVTaskFinished analyze mv name is " + result.last().get(5)) - sql "analyze table ${result.last().get(6)}.${mvName} with sync;" + logger.info("waitingMTMVTaskFinished analyze mv name is " + toCheckTaskRow.get(5)) + sql "analyze table ${toCheckTaskRow.get(6)}.${mvName} with sync;" } - void waitingMVTaskFinishedByMvName(String dbName, String tableName) { + void waitingMVTaskFinishedByMvName(String dbName, String tableName, String indexName) { Thread.sleep(2000) - String showTasks = "SHOW ALTER TABLE MATERIALIZED VIEW from ${dbName} where TableName='${tableName}' ORDER BY CreateTime ASC" + String showTasks = "SHOW ALTER TABLE MATERIALIZED VIEW from ${dbName} where TableName='${tableName}' ORDER BY CreateTime DESC" String status = "NULL" List> result long startTime = System.currentTimeMillis() long timeoutTimestamp = startTime + 5 * 60 * 1000 // 5 min - do { + List toCheckTaskRow = new ArrayList<>(); + while (timeoutTimestamp > System.currentTimeMillis() && (status != 'FINISHED')) { result = sql(showTasks) - logger.info("result: " + result.toString()) - if (!result.isEmpty()) { - status = result.last().get(8) + logger.info("crrent db is " + dbName + ", showTasks result: " + result.toString()) + // just consider current db + for (List taskRow : result) { + if (taskRow.get(5).equals(indexName)) { + toCheckTaskRow = taskRow; + } + } + if (toCheckTaskRow.isEmpty()) { + logger.info("waitingMVTaskFinishedByMvName toCheckTaskRow is empty") + Thread.sleep(1000); + continue; } + status = toCheckTaskRow.get(8) logger.info("The state of ${showTasks} is ${status}") Thread.sleep(1000); - } while (timeoutTimestamp > System.currentTimeMillis() && (status != 'FINISHED')) + } if (status != "FINISHED") { logger.info("status is not success") } @@ -1925,6 +1952,15 @@ class Suite implements GroovyInterceptable { return isReady } + def create_sync_mv = { db, table_name, mv_name, mv_sql -> + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name} ON ${table_name};""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + AS ${mv_sql} + """ + waitingMVTaskFinishedByMvName(db, table_name, mv_name) + } + def create_async_mv = { db, mv_name, mv_sql -> sql """DROP MATERIALIZED VIEW IF EXISTS ${db}.${mv_name}""" diff --git a/regression-test/suites/auth_call/test_ddl_mv_auth.groovy b/regression-test/suites/auth_call/test_ddl_mv_auth.groovy index 4dbf54fdf0df72..0701abb8a8bdb8 100644 --- a/regression-test/suites/auth_call/test_ddl_mv_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_mv_auth.groovy @@ -80,9 +80,9 @@ suite("test_ddl_mv_auth","p0,auth_call") { connect(user, "${pwd}", context.config.jdbcUrl) { sql """use ${dbName}""" sql """create materialized view ${mvName} as select username from ${dbName}.${tableName};""" - waitingMVTaskFinishedByMvName(dbName, tableName) + waitingMVTaskFinishedByMvName(dbName, tableName, mvName) sql """alter table ${dbName}.${tableName} add rollup ${rollupName}(username)""" - waitingMVTaskFinishedByMvName(dbName, tableName) + waitingMVTaskFinishedByMvName(dbName, tableName, rollupName) def mv_res = sql """desc ${dbName}.${tableName} all;""" logger.info("mv_res: " + mv_res) diff --git a/regression-test/suites/auth_p0/test_select_column_auth.groovy b/regression-test/suites/auth_p0/test_select_column_auth.groovy index 36cc2a0a09cf1c..1e93981966d2bc 100644 --- a/regression-test/suites/auth_p0/test_select_column_auth.groovy +++ b/regression-test/suites/auth_p0/test_select_column_auth.groovy @@ -69,7 +69,7 @@ suite("test_select_column_auth","p0,auth") { (3, "333"); """ sql """refresh MATERIALIZED VIEW ${dbName}.${mtmv_name} auto""" - waitingMTMVTaskFinishedByMvName(mtmv_name) + waitingMTMVTaskFinishedByMvName(mtmv_name, dbName) sql """grant select_priv on regression_test to ${user}""" diff --git a/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy b/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy index 8e548eb27ea87c..eae3d1fb3591fe 100644 --- a/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy +++ b/regression-test/suites/correctness_p0/test_mv_case/test_mv_case.groovy @@ -16,6 +16,7 @@ // under the License. suite("test_mv_case") { + sql """drop table if exists test_table_aaa2;""" sql """CREATE TABLE `test_table_aaa2` ( `ordernum` varchar(65533) NOT NULL , @@ -29,7 +30,7 @@ suite("test_mv_case") { "replication_allocation" = "tag.location.default: 1" );""" sql """DROP MATERIALIZED VIEW IF EXISTS ods_zn_dnt_max1 ON test_table_aaa2;""" - createMV("""create materialized view ods_zn_dnt_max1 as + create_sync_mv(context.dbName, "test_table_aaa2", "ods_zn_dnt_max1", """ select ordernum,max(dnt) as dnt from test_table_aaa2 group by ordernum ORDER BY ordernum;""") @@ -92,7 +93,7 @@ suite("test_mv_case") { ) """ sql """insert into tb1 select id,map_agg(a, b) from(select 123 id,3 a,'5' b union all select 123 id, 6 a, '8' b) aa group by id""" - createMV ("""CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`) BUCKETS 2 PROPERTIES ( + sql"""CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`) BUCKETS 2 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "min_load_replica_num" = "-1", "is_being_synced" = "false", @@ -112,8 +113,9 @@ suite("test_mv_case") { cast(a.id as bigint) info_id, map_infos from - tb1 a;""") - createMV ("""CREATE MATERIALIZED VIEW mv2 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`) BUCKETS 2 PROPERTIES ( + tb1 a;""" + waitingMTMVTaskFinishedByMvName("mv1") + sql """CREATE MATERIALIZED VIEW mv2 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 MINUTE DUPLICATE KEY(info_id) DISTRIBUTED BY HASH(`info_id`) BUCKETS 2 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "min_load_replica_num" = "-1", "is_being_synced" = "false", @@ -132,6 +134,7 @@ suite("test_mv_case") { info_id, map_infos from - mv1 a;""") + mv1 a;""" + waitingMTMVTaskFinishedByMvName("mv2") qt_select_mv """ select * from mv2 """ } diff --git a/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy b/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy index 295b195aa58954..7b6069968f4a0a 100644 --- a/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy +++ b/regression-test/suites/ddl_p0/create_view_nereids/create_view_use_mv.groovy @@ -48,8 +48,8 @@ suite("create_view_use_mv") { (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'), (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17');""" - createMV(""" - CREATE MATERIALIZED VIEW t_mv_mv AS select + create_sync_mv(context.dbName, "orders", "t_mv_mv", """ + select o_orderkey, sum(o_totalprice) as sum_total, max(o_totalprice) as max_total, diff --git a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy index 36c0d3f120e109..8dd16ea571f1ec 100644 --- a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy @@ -107,35 +107,35 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values ('2024-10-26 01:02:03', 1), ('2024-10-27 01:02:03', 2), ('2024-10-27 21:02:03', 3)""" sql """CREATE MATERIALIZED VIEW ${mvName1} BUILD DEFERRED REFRESH AUTO ON MANUAL partition by(`ts`) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') as SELECT * FROM ${catalog_name}.${icebergDb}.${icebergTable1}""" sql """REFRESH MATERIALIZED VIEW ${mvName1} complete""" - waitingMTMVTaskFinishedByMvName(mvName1) + waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh1 "select * from ${mvName1} order by value" sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values ('2024-10-26 21:02:03', 4)""" sql """REFRESH MATERIALIZED VIEW ${mvName1} auto""" - waitingMTMVTaskFinishedByMvName(mvName1) + waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh2 """select * from ${mvName1} order by value""" sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values ('2024-10-26 01:22:03', 5), ('2024-10-27 01:12:03', 6);""" sql """REFRESH MATERIALIZED VIEW ${mvName1} partitions(p_20241026000000_20241027000000);""" - waitingMTMVTaskFinishedByMvName(mvName1) + waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh3 """select * from ${mvName1} order by value""" sql """REFRESH MATERIALIZED VIEW ${mvName1} auto""" - waitingMTMVTaskFinishedByMvName(mvName1) + waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh4 """select * from ${mvName1} order by value""" sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values ('2024-10-28 01:22:03', 7);""" sql """REFRESH MATERIALIZED VIEW ${mvName1} partitions(p_20241026000000_20241027000000);""" - waitingMTMVTaskFinishedByMvName(mvName1) + waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh5 """select * from ${mvName1} order by value""" sql """REFRESH MATERIALIZED VIEW ${mvName1} auto""" - waitingMTMVTaskFinishedByMvName(mvName1) + waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh6 """select * from ${mvName1} order by value""" sql """insert into ${catalog_name}.${icebergDb}.${icebergTable1} values (null, 8);""" sql """REFRESH MATERIALIZED VIEW ${mvName1} auto""" - waitingMTMVTaskFinishedByMvName(mvName1) + waitingMTMVTaskFinishedByMvName(mvName1, dbName) qt_test_ts_refresh_null """select * from ${mvName1} order by value""" def showPartitionsResult = sql """show partitions from ${mvName1}""" @@ -176,25 +176,25 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2} values ('2024-08-26', 1), ('2024-09-17', 2), ('2024-09-27', 3);""" sql """CREATE MATERIALIZED VIEW ${mvName2} BUILD DEFERRED REFRESH AUTO ON MANUAL partition by(`d`) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') as SELECT * FROM ${catalog_name}.${icebergDb}.${icebergTable2}""" sql """REFRESH MATERIALIZED VIEW ${mvName2} complete""" - waitingMTMVTaskFinishedByMvName(mvName2) + waitingMTMVTaskFinishedByMvName(mvName2, dbName) qt_test_d_refresh1 "select * from ${mvName2} order by value" sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2} values ('2024-09-01', 4);""" sql """REFRESH MATERIALIZED VIEW ${mvName2} auto""" - waitingMTMVTaskFinishedByMvName(mvName2) + waitingMTMVTaskFinishedByMvName(mvName2, dbName) qt_test_d_refresh2 "select * from ${mvName2} order by value" sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2} values ('2024-08-22', 5), ('2024-09-30', 6);""" sql """REFRESH MATERIALIZED VIEW ${mvName2} partitions(p_20240801_20240901);""" - waitingMTMVTaskFinishedByMvName(mvName2) + waitingMTMVTaskFinishedByMvName(mvName2, dbName) qt_test_d_refresh3 "select * from ${mvName2} order by value" sql """REFRESH MATERIALIZED VIEW ${mvName2} partitions(p_20240901_20241001);""" - waitingMTMVTaskFinishedByMvName(mvName2) + waitingMTMVTaskFinishedByMvName(mvName2, dbName) qt_test_d_refresh4 "select * from ${mvName2} order by value" sql """insert into ${catalog_name}.${icebergDb}.${icebergTable2} values ('2024-10-28', 7);""" sql """REFRESH MATERIALIZED VIEW ${mvName2} auto""" - waitingMTMVTaskFinishedByMvName(mvName2) + waitingMTMVTaskFinishedByMvName(mvName2, dbName) qt_test_d_refresh5 "select * from ${mvName2} order by value" showPartitionsResult = sql """show partitions from ${mvName2}""" @@ -240,7 +240,7 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ // refresh one partiton sql """REFRESH MATERIALIZED VIEW ${mvName} partitions(p_20240101000000_20240102000000);""" - waitingMTMVTaskFinishedByMvName(mvName) + waitingMTMVTaskFinishedByMvName(mvName, dbName) order_qt_refresh_one_partition "SELECT * FROM ${mvName} " def explainOnePartition = sql """ explain ${mvSql} """ logger.info("explainOnePartition: " + explainOnePartition.toString()) @@ -250,7 +250,7 @@ suite("test_iceberg_mtmv", "p0,external,iceberg,external_docker,external_docker_ //refresh auto sql """REFRESH MATERIALIZED VIEW ${mvName} auto""" - waitingMTMVTaskFinishedByMvName(mvName) + waitingMTMVTaskFinishedByMvName(mvName, dbName) order_qt_refresh_auto "SELECT * FROM ${mvName} " def explainAllPartition = sql """ explain ${mvSql}; """ logger.info("explainAllPartition: " + explainAllPartition.toString()) diff --git a/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy b/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy index a3ac1c048d30da..7a77cdc4590a4c 100644 --- a/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_paimon_olap_rewrite_mtmv.groovy @@ -87,6 +87,10 @@ suite("test_paimon_olap_rewrite_mtmv", "p0,external,mtmv,external_docker,externa def explainOnePartition = sql """ explain ${mvSql} """ logger.info("explainOnePartition: " + explainOnePartition.toString()) + + def explain_memo_plan = sql """ explain memo plan ${mvSql} """ + logger.info("explain_memo_plan: " + explain_memo_plan.toString()) + assertTrue(explainOnePartition.toString().contains("VUNION")) order_qt_refresh_one_partition_rewrite "${mvSql}" @@ -104,6 +108,10 @@ suite("test_paimon_olap_rewrite_mtmv", "p0,external,mtmv,external_docker,externa def explainAllPartition = sql """ explain ${mvSql}; """ logger.info("explainAllPartition: " + explainAllPartition.toString()) + + def explainMemoPlan = sql """ explain memo plan ${mvSql}; """ + logger.info("explainMemoPlan: " + explainMemoPlan.toString()) + assertTrue(explainAllPartition.toString().contains("VOlapScanNode")) order_qt_refresh_all_partition_rewrite "${mvSql}" diff --git a/regression-test/suites/mv_p0/no_await/no_await.groovy b/regression-test/suites/mv_p0/no_await/no_await.groovy index 866e4fdd5d3546..3eab03aa7e4adc 100644 --- a/regression-test/suites/mv_p0/no_await/no_await.groovy +++ b/regression-test/suites/mv_p0/no_await/no_await.groovy @@ -19,6 +19,8 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite ("no_await") { + String db = context.config.getDbNameByFile(context.file) + def tblName = "agg_have_dup_base_no_await" def waitDrop = { def try_times = 1000 @@ -58,92 +60,93 @@ suite ("no_await") { """ sql "insert into ${tblName} select e1, -4, -4, 'd' from (select 1 k1) as t lateral view explode_numbers(10000) tmp1 as e1;" // do not await - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") + sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" waitDrop() - sql "create materialized view k12s3m as select k1,sum(k2),max(k2) from ${tblName} group by k1;" + create_sync_mv(db, tblName, "k12s3m", """select k1,sum(k2),max(k2) from ${tblName} group by k1;""") sql "insert into ${tblName} select -4, -4, -4, \'d\'" qt_mv "select sum(k1) from ${tblName}" } diff --git a/regression-test/suites/mv_p0/unique/unique_rewrite.groovy b/regression-test/suites/mv_p0/unique_rewrite/unique_rewrite.groovy similarity index 100% rename from regression-test/suites/mv_p0/unique/unique_rewrite.groovy rename to regression-test/suites/mv_p0/unique_rewrite/unique_rewrite.groovy diff --git a/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy b/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy index 58b2e7bbdd8ca3..e88f4132eacfba 100644 --- a/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy @@ -1447,7 +1447,7 @@ suite("mv_with_date_trunc") { logger.info("lineitem table stats: " + result) result = sql """show index stats lineitem lineitem""" logger.info("lineitem index stats: " + result) - mv_rewrite_success(query4_0, "mv4_0") + mv_rewrite_success(query4_0, "mv4_0", true, is_partition_statistics_ready(db, ["lineitem", "mv4_0"])) order_qt_query4_0_after "${query4_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_0""" diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy index 4870ec99e659c0..1972c2d505bfab 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy @@ -742,10 +742,12 @@ suite("nested_mtmv") { mv_rewrite_any_success(sql_2, [mv_1, mv_2]) compare_res(sql_2 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") - mv_rewrite_any_success(sql_3, [mv_3, mv_4]) + // level 1 maybe use mv_1 and mv_2, this also meets expectation + mv_rewrite_any_success(sql_3, [mv_3, mv_4, mv_1, mv_2]) compare_res(sql_3 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") - mv_rewrite_any_success(sql_4, [mv_3, mv_4]) + // level 1 maybe use mv_1 and mv_2, this also meets expectation + mv_rewrite_any_success(sql_4, [mv_3, mv_4, mv_1, mv_2]) compare_res(sql_4 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") mv_rewrite_any_success(sql_5, [mv_3, mv_4, mv_5]) diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy index 7a49dbc4b37118..32e1684fdc9073 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy @@ -33,21 +33,29 @@ suite ("multi_slot4") { """ sql "insert into multi_slot4 select 1,1,1,'a';" + sql "insert into multi_slot4 select 1,1,1,'a';" + sql "insert into multi_slot4 select 1,1,1,'a';" + sql "insert into multi_slot4 select 2,2,2,'b';" + sql "insert into multi_slot4 select 2,2,2,'b';" sql "insert into multi_slot4 select 2,2,2,'b';" sql "insert into multi_slot4 select 3,-3,null,'c';" sql "insert into multi_slot4 select 3,-3,null,'c';" + sql "insert into multi_slot4 select 3,-3,null,'c';" + sql "insert into multi_slot4 select 3,-3,null,'c';" createMV ("create materialized view k1p2ap3ps as select k1+1,sum(abs(k2+2)+k3+3) from multi_slot4 group by k1+1;") sleep(3000) + sql "insert into multi_slot4 select -4,-4,-4,'d';" + sql "insert into multi_slot4 select -4,-4,-4,'d';" sql "insert into multi_slot4 select -4,-4,-4,'d';" sql "SET experimental_enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "analyze table multi_slot4 with sync;" - sql """alter table multi_slot4 modify column k1 set stats ('row_count'='5');""" + sql """alter table multi_slot4 modify column k1 set stats ('row_count'='13');""" sql """set enable_stats=false;""" diff --git a/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy b/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy index 9550a7fec3dbd2..56f5d655255049 100644 --- a/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy +++ b/regression-test/suites/query_p0/show/test_show_create_materialized_view.groovy @@ -30,12 +30,11 @@ suite("test_show_create_materialized_view", "query,arrow_flight_sql") { DISTRIBUTED BY HASH(id) BUCKETS 5 PROPERTIES ("replication_num" = "1"); """ - - createMV("""CREATE MATERIALIZED VIEW ${mvName} AS + + create_sync_mv(context.dbName, tableName, mvName, """ SELECT id, name, SUM(value) AS total_value FROM ${tableName} - GROUP BY id, name; - """) + GROUP BY id, name""") checkNereidsExecute("""SHOW CREATE MATERIALIZED VIEW ${mvName} ON ${tableName};""") qt_cmd("""SHOW CREATE MATERIALIZED VIEW ${mvName} ON ${tableName};""") diff --git a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy index 713c470436e7fb..a6ad20ec623048 100644 --- a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy @@ -70,7 +70,7 @@ suite ("test_dup_mv_schema_change") { """ //add materialized view - createMV("create materialized view mv1 as select date, user_id, city, age from ${tableName};") + create_sync_mv(context.dbName, tableName, "mv1", """select date, user_id, city, age from ${tableName}""") // alter and test light schema change if (!isCloudMode()) { @@ -78,7 +78,7 @@ suite ("test_dup_mv_schema_change") { } //add materialized view - createMV("create materialized view mv2 as select date, user_id, city, age, cost from ${tableName};") + create_sync_mv(context.dbName, tableName, "mv2", """select date, user_id, city, age, cost from ${tableName}""") sql """ INSERT INTO ${tableName} VALUES (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) diff --git a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy index eba6036c30a628..1d8fdd4d1e7192 100644 --- a/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_uniq_mv_schema_change.groovy @@ -21,20 +21,7 @@ import org.awaitility.Awaitility suite ("test_uniq_mv_schema_change") { def tableName = "schema_change_uniq_mv_regression_test" - def getMVJobState = { tbName -> - def jobStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName='${tbName}' ORDER BY CreateTime DESC LIMIT 1 """ - return jobStateResult[0][8] - } - def waitForJob = (tbName, timeout) -> { - Awaitility.await().atMost(timeout, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(() -> { - String result = getMVJobState(tbName) - if (result == "FINISHED") { - return true; - } - return false; - }); - // when timeout awaitlity will raise a exception. - } + try { String backend_id; @@ -78,8 +65,7 @@ suite ("test_uniq_mv_schema_change") { //add materialized view def mvName = "mv1" - sql "create materialized view ${mvName} as select user_id, date, city, age, sex from ${tableName};" - waitForJob(tableName, 3000) + create_sync_mv(context.dbName, tableName, mvName, """select user_id, date, city, age, sex from ${tableName}""") // alter and test light schema change if (!isCloudMode()) { @@ -88,8 +74,7 @@ suite ("test_uniq_mv_schema_change") { //add materialized view def mvName2 = "mv2" - sql "create materialized view ${mvName2} as select user_id, date, city, age, sex, cost from ${tableName};" - waitForJob(tableName, 3000) + create_sync_mv(context.dbName, tableName, mvName2, """select user_id, date, city, age, sex, cost from ${tableName};""") sql """ INSERT INTO ${tableName} VALUES (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21)