Skip to content

Commit

Permalink
add profile test
Browse files Browse the repository at this point in the history
  • Loading branch information
@amorynan
amorynan committed Nov 6, 2024
1 parent 4527cce commit 89d1041
Showing 1 changed file with 118 additions and 83 deletions.
201 changes: 118 additions & 83 deletions regression-test/suites/inverted_index_p0/test_ip_cidr_search_with_inverted_index.groovy
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import groovy.json.JsonSlurper

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
Expand All @@ -15,6 +17,27 @@
// specific language governing permissions and limitations
// under the License.

def getProfileList = {
def dst = 'http://' + context.config.feHttpAddress
def conn = new URL(dst + "/rest/v1/query_profile").openConnection()
conn.setRequestMethod("GET")
def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
(context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
conn.setRequestProperty("Authorization", "Basic ${encoding}")
return conn.getInputStream().getText()
}


def getProfile = { id ->
def dst = 'http://' + context.config.feHttpAddress
def conn = new URL(dst + "/rest/v1/query_profile/$id").openConnection()
conn.setRequestMethod("GET")
def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
(context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
conn.setRequestProperty("Authorization", "Basic ${encoding}")
return conn.getInputStream().getText()
}

suite("test_ip_cidr_search_with_inverted_index", "nonConcurrent"){
// prepare test table
sql "DROP TABLE IF EXISTS tc_ip_cidr_search_with_inverted_index"
Expand All @@ -40,95 +63,107 @@ suite("test_ip_cidr_search_with_inverted_index", "nonConcurrent"){
"enable_single_replica_compaction" = "false"
);
"""
sql """ set enable_profile = true;"""

sql """ insert into tc_ip_cidr_search_with_inverted_index values(1, '59.50.185.152', '2a02:e980:83:5b09:ecb8:c669:b336:650e', '127.0.0.0/8'),(3, '119.36.22.147', '2001:4888:1f:e891:161:26::', '127.0.0.0/8'),(2, '42.117.228.166', '2001:16a0:2:200a::2', null); """
sql """ insert into tc_ip_cidr_search_with_inverted_index values(4, '.', '2001:1b70:a1:610::b102:2', null); """
sql """ insert into tc_ip_cidr_search_with_inverted_index values(5, '255.255.255.255', 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffffg', null) """

qt_sql """ select count() from tc_ip_cidr_search_with_inverted_index"""
// without inverted index query
sql """ set enable_common_expr_pushdown = false; """
sql """ set enable_inverted_index_query=false; """
// select ipv6 in ipv4 cidr
qt_sql_without_ii_0 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '255.255.255.255/12') order by id; """
// select ipv6 in ipv6 cidr
qt_sql_without_ii_1 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """
qt_sql_without_ii_2 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') order by id; """
qt_sql_without_ii_3 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') order by id; """

// select ipv4 in ipv6 cidr
qt_sql_without_ii_4 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """
// select ipv4 in ipv4 cidr
qt_sql_without_ii_5 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '255.255.255.255/12') order by id; """
qt_sql_without_ii_6 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '127.0.0.0/8') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '127.0.0.0/8') order by id; """
qt_sql_without_ii_7 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '192.168.100.0/24') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '192.168.100.0/24') order by id; """

// select in null cidr
qt_sql_without_ii_8 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, null) order by id; """
qt_sql_without_ii_9 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, null) order by id; """

// with inverted index query
// If we use common expr pass to inverted index , we should set enable_common_expr_pushdown = true
sql """ set enable_common_expr_pushdown = true; """
sql """ set enable_inverted_index_query=true; """
sql """ set inverted_index_skip_threshold = 0; """ // set skip threshold to 0

// select ipv6 in ipv4 cidr
qt_sql_with_ii_0 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '255.255.255.255/12') order by id; """
// select ipv6 in ipv6 cidr
qt_sql_with_ii_1 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """
qt_sql_with_ii_2 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') order by id; """
qt_sql_with_ii_3 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') order by id; """

// select ipv4 in ipv6 cidr
qt_sql_with_ii_4 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """
// select ipv4 in ipv4 cidr
qt_sql_with_ii_5 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '255.255.255.255/12') order by id; """
qt_sql_with_ii_6 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '127.0.0.0/8') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '127.0.0.0/8') order by id; """
qt_sql_with_ii_7 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '192.168.100.0/24') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '192.168.100.0/24') order by id; """

// select in null cidr
qt_sql_with_ii_8 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, null) order by id; """
qt_sql_with_ii_9 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, null) order by id; """



def create_sql = {
List<String> list = new ArrayList<>()

try {
sql """ set enable_profile = true;"""

sql """ insert into tc_ip_cidr_search_with_inverted_index values(1, '59.50.185.152', '2a02:e980:83:5b09:ecb8:c669:b336:650e', '127.0.0.0/8'),(3, '119.36.22.147', '2001:4888:1f:e891:161:26::', '127.0.0.0/8'),(2, '42.117.228.166', '2001:16a0:2:200a::2', null); """
sql """ insert into tc_ip_cidr_search_with_inverted_index values(4, '.', '2001:1b70:a1:610::b102:2', null); """
sql """ insert into tc_ip_cidr_search_with_inverted_index values(5, '255.255.255.255', 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffffg', null) """

qt_sql """ select count() from tc_ip_cidr_search_with_inverted_index"""
// without inverted index query
sql """ set enable_common_expr_pushdown = false; """
sql """ set enable_inverted_index_query=false; """
// select ipv6 in ipv4 cidr
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '255.255.255.255/12') order by id;")
qt_sql_without_ii_0 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '255.255.255.255/12') order by id; """
// select ipv6 in ipv6 cidr
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id;")
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') order by id;")
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') order by id;")
qt_sql_without_ii_1 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """
qt_sql_without_ii_2 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') order by id; """
qt_sql_without_ii_3 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') order by id; """

// select ipv4 in ipv6 cidr
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id;")
qt_sql_without_ii_4 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """
// select ipv4 in ipv4 cidr
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '255.255.255.255/12') order by id;")
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '127.0.0.0/8') order by id;")
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '192.168.100.0/24') order by id;")
qt_sql_without_ii_5 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '255.255.255.255/12') order by id; """
qt_sql_without_ii_6 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '127.0.0.0/8') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '127.0.0.0/8') order by id; """
qt_sql_without_ii_7 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '192.168.100.0/24') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '192.168.100.0/24') order by id; """

// select in null cidr
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, null) order by id;")
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, null) order by id;")
return list;
}
qt_sql_without_ii_8 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, null) order by id; """
qt_sql_without_ii_9 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, null) order by id; """

def checkpoints_name = "ip.inverted_index_filtered"
def execute_sql = { sqlList ->
def i = 0
for (sqlStr in sqlList) {
try {
log.info("execute sql: i")
GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [req_id: i])
order_qt_sql """ ${sqlStr} """
} finally {
GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
}
++i
// with inverted index query
// If we use common expr pass to inverted index , we should set enable_common_expr_pushdown = true
sql """ set enable_common_expr_pushdown = true; """
sql """ set enable_inverted_index_query=true; """
sql """ set inverted_index_skip_threshold = 0; """ // set skip threshold to 0

// select ipv6 in ipv4 cidr
qt_sql_with_ii_0 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '255.255.255.255/12') order by id; """
// select ipv6 in ipv6 cidr
qt_sql_with_ii_1 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """
qt_sql_with_ii_2 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') order by id; """
qt_sql_with_ii_3 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') order by id; """

// select ipv4 in ipv6 cidr
qt_sql_with_ii_4 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id; """
// select ipv4 in ipv4 cidr
qt_sql_with_ii_5 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, '255.255.255.255/12') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '255.255.255.255/12') order by id; """
qt_sql_with_ii_6 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '127.0.0.0/8') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '127.0.0.0/8') order by id; """
qt_sql_with_ii_7 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, '192.168.100.0/24') from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '192.168.100.0/24') order by id; """

// select in null cidr
qt_sql_with_ii_8 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv4, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, null) order by id; """
qt_sql_with_ii_9 """ select id, ipv4, ipv6, is_ip_address_in_range(ipv6, null) from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, null) order by id; """


def create_sql = {
List<String> list = new ArrayList<>()
// select ipv6 in ipv4 cidr
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '255.255.255.255/12') order by id;")
// select ipv6 in ipv6 cidr
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id;")
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '::ffff:192.168.0.4/128') order by id;")
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, '2001:16a0:2:200a::2/64') order by id;")
// select ipv4 in ipv6 cidr
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/12') order by id;")
// select ipv4 in ipv4 cidr
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '255.255.255.255/12') order by id;")
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '127.0.0.0/8') order by id;")
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, '192.168.100.0/24') order by id;")
// select in null cidr
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv4, null) order by id;")
list.add("select id, ipv4, ipv6 from tc_ip_cidr_search_with_inverted_index where is_ip_address_in_range(ipv6, null) order by id;")
return list;
}
}

execute_sql.call(create_sql.call())
def checkpoints_name = "ip.inverted_index_filtered"
def execute_sql = { sqlList ->
def i = 0
for (sqlStr in sqlList) {
try {
log.info("execute sql: i")
GetDebugPoint().enableDebugPointForAllBEs(checkpoints_name, [req_id: i])
order_qt_sql """ ${sqlStr} """
} finally {
GetDebugPoint().disableDebugPointForAllBEs(checkpoints_name)
}
++i
}
}

}
execute_sql.call(create_sql.call())
} catch (Exception e) {
logger.error("exception:{}", e)
// get last profile
def id = sql """select last_query_id();"""
logger.info("last_query_id:{}", id[0][0])
def query_id = id[0][0]
def str = getProfile(query_id)
logger.info("profile string:{}", str)
def json = new JsonSlurper().parseText(str)
logger.info("json_profile:{}", json)
} finally {
}
}

0 comments on commit 89d1041

Please sign in to comment.