From 89d104159205ab56c57256dc6c2e484d8c9bb5d5 Mon Sep 17 00:00:00 2001 From: amorynan Date: Wed, 6 Nov 2024 18:09:16 +0800 Subject: [PATCH] add profile test --- ..._ip_cidr_search_with_inverted_index.groovy | 201 ++++++++++-------- 1 file changed, 118 insertions(+), 83 deletions(-) diff --git a/regression-test/suites/inverted_index_p0/test_ip_cidr_search_with_inverted_index.groovy b/regression-test/suites/inverted_index_p0/test_ip_cidr_search_with_inverted_index.groovy index 61751f2404f2d2..debc47fb14763e 100644 --- a/regression-test/suites/inverted_index_p0/test_ip_cidr_search_with_inverted_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_ip_cidr_search_with_inverted_index.groovy @@ -1,3 +1,5 @@ +import groovy.json.JsonSlurper + // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -15,6 +17,27 @@ // specific language governing permissions and limitations // under the License. +def getProfileList = { + def dst = 'http://' + context.config.feHttpAddress + def conn = new URL(dst + "/rest/v1/query_profile").openConnection() + conn.setRequestMethod("GET") + def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + + (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8")) + conn.setRequestProperty("Authorization", "Basic ${encoding}") + return conn.getInputStream().getText() +} + + +def getProfile = { id -> + def dst = 'http://' + context.config.feHttpAddress + def conn = new URL(dst + "/rest/v1/query_profile/$id").openConnection() + conn.setRequestMethod("GET") + def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + + (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8")) + conn.setRequestProperty("Authorization", "Basic ${encoding}") + return conn.getInputStream().getText() + } + suite("test_ip_cidr_search_with_inverted_index", "nonConcurrent"){ // prepare test table sql "DROP TABLE IF EXISTS tc_ip_cidr_search_with_inverted_index" @@ -40,95 +63,107 @@ suite("test_ip_cidr_search_with_inverted_index", "nonConcurrent"){ "enable_single_replica_compaction" = "false" ); """ - sql """ set enable_profile = true;""" - - sql """ insert into tc_ip_cidr_search_with_inverted_index values(1, '59.50.185.152', '2a02:e980:83:5b09:ecb8:c669:b336:650e', '127.0.0.0/8'),(3, '119.36.22.147', '2001:4888:1f:e891:161:26::', '127.0.0.0/8'),(2, '42.117.228.166', '2001:16a0:2:200a::2', null); """ - sql """ insert into tc_ip_cidr_search_with_inverted_index values(4, '.', '2001:1b70:a1:610::b102:2', null); """ - sql """ insert into tc_ip_cidr_search_with_inverted_index values(5, '255.255.255.255', 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffffg', null) """ - - qt_sql """ select count() from tc_ip_cidr_search_with_inverted_index""" - // without inverted index query - sql """ set enable_common_expr_pushdown = false; """ - sql """ set enable_inverted_index_query=false; """ - // select ipv6 in ipv4 cidr - qt_sql_without_ii_0 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '255.255.255.255/12') order by id; """ - // select ipv6 in ipv6 cidr - qt_sql_without_ii_1 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """ - qt_sql_without_ii_2 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') order by id; """ - qt_sql_without_ii_3 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') order by id; """ - - // select ipv4 in ipv6 cidr - qt_sql_without_ii_4 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """ - // select ipv4 in ipv4 cidr - qt_sql_without_ii_5 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '255.255.255.255/12') order by id; """ - qt_sql_without_ii_6 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '127.0.0.0/8') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '127.0.0.0/8') order by id; """ - qt_sql_without_ii_7 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '192.168.100.0/24') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '192.168.100.0/24') order by id; """ - - // select in null cidr - qt_sql_without_ii_8 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, null) order by id; """ - qt_sql_without_ii_9 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, null) order by id; """ - - // with inverted index query - // If we use common expr pass to inverted index , we should set enable_common_expr_pushdown = true - sql """ set enable_common_expr_pushdown = true; """ - sql """ set enable_inverted_index_query=true; """ - sql """ set inverted_index_skip_threshold = 0; """ // set skip threshold to 0 - - // select ipv6 in ipv4 cidr - qt_sql_with_ii_0 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '255.255.255.255/12') order by id; """ - // select ipv6 in ipv6 cidr - qt_sql_with_ii_1 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """ - qt_sql_with_ii_2 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') order by id; """ - qt_sql_with_ii_3 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') order by id; """ - - // select ipv4 in ipv6 cidr - qt_sql_with_ii_4 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """ - // select ipv4 in ipv4 cidr - qt_sql_with_ii_5 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '255.255.255.255/12') order by id; """ - qt_sql_with_ii_6 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '127.0.0.0/8') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '127.0.0.0/8') order by id; """ - qt_sql_with_ii_7 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '192.168.100.0/24') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '192.168.100.0/24') order by id; """ - - // select in null cidr - qt_sql_with_ii_8 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, null) order by id; """ - qt_sql_with_ii_9 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, null) order by id; """ - - - - def create_sql = { - List list = new ArrayList<>() + + try { + sql """ set enable_profile = true;""" + + sql """ insert into tc_ip_cidr_search_with_inverted_index values(1, '59.50.185.152', '2a02:e980:83:5b09:ecb8:c669:b336:650e', '127.0.0.0/8'),(3, '119.36.22.147', '2001:4888:1f:e891:161:26::', '127.0.0.0/8'),(2, '42.117.228.166', '2001:16a0:2:200a::2', null); """ + sql """ insert into tc_ip_cidr_search_with_inverted_index values(4, '.', '2001:1b70:a1:610::b102:2', null); """ + sql """ insert into tc_ip_cidr_search_with_inverted_index values(5, '255.255.255.255', 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffffg', null) """ + + qt_sql """ select count() from tc_ip_cidr_search_with_inverted_index""" + // without inverted index query + sql """ set enable_common_expr_pushdown = false; """ + sql """ set enable_inverted_index_query=false; """ // select ipv6 in ipv4 cidr - list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '255.255.255.255/12') order by id;") + qt_sql_without_ii_0 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '255.255.255.255/12') order by id; """ // select ipv6 in ipv6 cidr - list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id;") - list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') order by id;") - list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') order by id;") + qt_sql_without_ii_1 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """ + qt_sql_without_ii_2 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') order by id; """ + qt_sql_without_ii_3 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') order by id; """ + // select ipv4 in ipv6 cidr - list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id;") + qt_sql_without_ii_4 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """ // select ipv4 in ipv4 cidr - list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '255.255.255.255/12') order by id;") - list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '127.0.0.0/8') order by id;") - list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '192.168.100.0/24') order by id;") + qt_sql_without_ii_5 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '255.255.255.255/12') order by id; """ + qt_sql_without_ii_6 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '127.0.0.0/8') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '127.0.0.0/8') order by id; """ + qt_sql_without_ii_7 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '192.168.100.0/24') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '192.168.100.0/24') order by id; """ + // select in null cidr - list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, null) order by id;") - list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, null) order by id;") - return list; - } + qt_sql_without_ii_8 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, null) order by id; """ + qt_sql_without_ii_9 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, null) order by id; """ - def checkpoints_name = "ip.inverted_index_filtered" - def execute_sql = { sqlList -> - def i = 0 - for (sqlStr in sqlList) { - try { - log.info("execute sql: i") - GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [req_id: i]) - order_qt_sql """ ${sqlStr} """ - } finally { - GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name) - } - ++i + // with inverted index query + // If we use common expr pass to inverted index , we should set enable_common_expr_pushdown = true + sql """ set enable_common_expr_pushdown = true; """ + sql """ set enable_inverted_index_query=true; """ + sql """ set inverted_index_skip_threshold = 0; """ // set skip threshold to 0 + + // select ipv6 in ipv4 cidr + qt_sql_with_ii_0 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '255.255.255.255/12') order by id; """ + // select ipv6 in ipv6 cidr + qt_sql_with_ii_1 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """ + qt_sql_with_ii_2 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') order by id; """ + qt_sql_with_ii_3 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') order by id; """ + + // select ipv4 in ipv6 cidr + qt_sql_with_ii_4 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """ + // select ipv4 in ipv4 cidr + qt_sql_with_ii_5 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '255.255.255.255/12') order by id; """ + qt_sql_with_ii_6 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '127.0.0.0/8') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '127.0.0.0/8') order by id; """ + qt_sql_with_ii_7 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '192.168.100.0/24') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '192.168.100.0/24') order by id; """ + + // select in null cidr + qt_sql_with_ii_8 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, null) order by id; """ + qt_sql_with_ii_9 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, null) order by id; """ + + + def create_sql = { + List list = new ArrayList<>() + // select ipv6 in ipv4 cidr + list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '255.255.255.255/12') order by id;") + // select ipv6 in ipv6 cidr + list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id;") + list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') order by id;") + list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') order by id;") + // select ipv4 in ipv6 cidr + list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id;") + // select ipv4 in ipv4 cidr + list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '255.255.255.255/12') order by id;") + list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '127.0.0.0/8') order by id;") + list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '192.168.100.0/24') order by id;") + // select in null cidr + list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, null) order by id;") + list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, null) order by id;") + return list; } - } - execute_sql.call(create_sql.call()) + def checkpoints_name = "ip.inverted_index_filtered" + def execute_sql = { sqlList -> + def i = 0 + for (sqlStr in sqlList) { + try { + log.info("execute sql: i") + GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [req_id: i]) + order_qt_sql """ ${sqlStr} """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name) + } + ++i + } + } -} \ No newline at end of file + execute_sql.call(create_sql.call()) + } catch (Exception e) { + logger.error("exception:{}", e) + // get last profile + def id = sql """select last_query_id();""" + logger.info("last_query_id:{}", id[0][0]) + def query_id = id[0][0] + def str = getProfile(query_id) + logger.info("profile string:{}", str) + def json = new JsonSlurper().parseText(str) + logger.info("json_profile:{}", json) + } finally { + } +}