Skip to content

Commit

Permalink
update regress test and element at out data
Browse files Browse the repository at this point in the history
  • Loading branch information
@amorynan
amorynan committed Nov 16, 2023
1 parent dd4d5ae commit ab4d370
Show file tree
Hide file tree
Showing 4 changed files with 4,851 additions and 1,403 deletions.
698 changes: 349 additions & 349 deletions ...ssion-test/data/datatype_p0/nested_types/base_cases/one_level_nestedtypes_with_s3data.out
View file

Large diffs are not rendered by default.

5,395 changes: 4,409 additions & 986 deletions ...ssion-test/data/datatype_p0/nested_types/base_cases/two_level_nestedtypes_with_s3data.out
View file

Large diffs are not rendered by default.

69 changes: 39 additions & 30 deletions ...-test/suites/datatype_p0/nested_types/base_cases/one_level_nestedtypes_with_s3data.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ suite("one_level_nestedtypes_with_s3data") {
String bucket = context.config.otherConfigs.get("s3BucketName");


// def dataFilePath = "https://"+"${bucket}"+"."+"${s3_endpoint}"+"/regression/datalake"
def dataFilePath = "/mnt/disk1/wangqiannan/export/ol"
def dataFilePath = "https://"+"${bucket}"+"."+"${s3_endpoint}"+"/regression/datalake"
// def dataFilePath = "/mnt/disk1/wangqiannan/export/ol"
def table_names = ["test_array_one_level", "test_map_one_level", "test_struct_one_level"]

def colNameArr = ["c_bool", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_largeint", "c_float",
Expand Down Expand Up @@ -90,31 +90,37 @@ suite("one_level_nestedtypes_with_s3data") {
}
def load_from_s3 = {table_name, uri_file, format ->
if (format == "csv") {
order_qt_sql_tvf """select * from local(
"file_path" = "${uri_file}",
"backend_id" = "${be_id}",
"column_separator"="|",
"format" = "${format}") order by c1 limit 10; """
order_qt_sql_s3 """select * from s3(
"uri" = "${uri_file}",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "${format}",
"column_separator"="|",
"read_json_by_line"="true") order by c1 limit 10; """

sql """
insert into ${table_name} select * from local(
"file_path" = "${uri_file}",
"backend_id" = "${be_id}",
"column_separator"="|",
"format" = "${format}"); """
insert into ${table_name} select * from s3(
"uri" = "${uri_file}",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "${format}",
"column_separator"="|",
"read_json_by_line"="true"); """
} else {
order_qt_sql_tvf """select * from local(
"file_path" = "${uri_file}",
"backend_id" = "${be_id}",
"column_separator"="|",
"format" = "${format}") order by k1 limit 10; """
order_qt_sql_s3 """select * from s3(
"uri" = "${uri_file}",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "${format}",
"read_json_by_line"="true") order by k1 limit 10; """

sql """
insert into ${table_name} select * from local(
"file_path" = "${uri_file}",
"backend_id" = "${be_id}",
"column_separator"="|",
"format" = "${format}"); """
insert into ${table_name} select * from s3(
"uri" = "${uri_file}",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "${format}",
"read_json_by_line"="true"); """
}
// where to filter different format data
qt_select_doris """ select * from ${table_name} where k1 IS NOT NULL order by k1 limit 10; """
Expand Down Expand Up @@ -151,7 +157,8 @@ suite("one_level_nestedtypes_with_s3data") {
int fi = 0
for (String f : array_files) {
sql "truncate table ${table_names[2]};"
load_from_tvf(table_names[0], f, format_order[fi])
// load_from_tvf(table_names[0], f, format_order[fi])
load_from_s3(table_names[0], f, format_order[fi])
++ fi
}
// select element_at(column)
Expand All @@ -161,9 +168,9 @@ suite("one_level_nestedtypes_with_s3data") {
// last
order_qt_select_arr "select ${col}[-1] from ${table_names[0]} where k1 IS NOT NULL order by k1 limit 10;"
// null
order_qt_select_arr "select ${col}[0] from ${table_names[0]} where k1 IS NOT NULL order by k1 limit 10;"
order_qt_select_arr_null "select ${col}[0] from ${table_names[0]} where k1 IS NOT NULL order by k1 limit 10;"
// null
order_qt_select_arr "select ${col}[1000] from ${table_names[0]} where k1 IS NOT NULL order by k1 limit 10;"
order_qt_select_arr_null "select ${col}[1000] from ${table_names[0]} where k1 IS NOT NULL order by k1 limit 10;"
}
// select * from table where element_at(column) with equal expr
for (String col : colNameArr) {
Expand Down Expand Up @@ -192,7 +199,8 @@ suite("one_level_nestedtypes_with_s3data") {
fi = 0
for (String f : map_files) {
sql "truncate table ${table_names[2]};"
load_from_tvf(table_names[1], f, format_order[fi])
// load_from_tvf(table_names[1], f, format_order[fi])
load_from_s3(table_names[1], f, format_order[fi])
++ fi
}
// select element_at(column)
Expand All @@ -201,10 +209,10 @@ suite("one_level_nestedtypes_with_s3data") {
order_qt_select_map "select ${col}[map_keys(${col})[1]] from ${table_names[1]} where k1 IS NOT NULL order by k1 limit 10;"
// last
order_qt_select_map "select ${col}[map_keys(${col})[-1]] from ${table_names[1]} where k1 IS NOT NULL order by k1 limit 10;"
// null::q:q
order_qt_select_map_null "select ${col}[map_keys(${col})[0]] from ${table_names[1]} where k1 IS NOT NULL order by k1 limit 10;"
// null
order_qt_select_map "select ${col}[map_keys(${col})[0]] from ${table_names[1]} where k1 IS NOT NULL order by k1 limit 10;"
// null
order_qt_select_map "select ${col}[map_keys(${col})[1000]] from ${table_names[1]} where k1 IS NOT NULL order by k1 limit 10;"
order_qt_select_map_null "select ${col}[map_keys(${col})[1000]] from ${table_names[1]} where k1 IS NOT NULL order by k1 limit 10;"
}
// select * from table where element_at(column) with equal expr
for (String col : colNameArr) {
Expand Down Expand Up @@ -233,7 +241,8 @@ suite("one_level_nestedtypes_with_s3data") {
fi = 0
for (String f : struct_files) {
sql "truncate table ${table_names[2]};"
load_from_tvf(table_names[2], f, format_order[fi])
// load_from_tvf(table_names[2], f, format_order[fi])
load_from_s3(table_names[2], f, format_order[fi])
++ fi
}
// select element_at(column)
Expand Down
92 changes: 54 additions & 38 deletions ...-test/suites/datatype_p0/nested_types/base_cases/two_level_nestedtypes_with_s3data.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ suite("two_level_nestedtypes_with_s3data") {
String bucket = context.config.otherConfigs.get("s3BucketName");


// def dataFilePath = "https://"+"${bucket}"+"."+"${s3_endpoint}"+"/regression/datalake"
def dataFilePath = "/mnt/disk1/wangqiannan/export/tl/two_level"
def dataFilePath = "https://"+"${bucket}"+"."+"${s3_endpoint}"+"/regression/datalake"
// def dataFilePath = "/mnt/disk1/wangqiannan/export/tl/two_level"
def table_names = [
"two_level_array_array",
"two_level_array_map",
Expand Down Expand Up @@ -61,6 +61,9 @@ suite("two_level_nestedtypes_with_s3data") {
}
}

def select_nested_scala_element_at = { agg_expr, table_name ->
order_qt_select_nested "select ${agg_expr} from ${table_name} where ${agg_expr} IS NOT NULL AND k1 IS NOT NULL order by k1 limit 10;"
}
def groupby_or_orderby_element_at = {is_groupby, table_name, agg_expr ->
if (is_groupby) {
order_qt_sql "select ${agg_expr} from ${table_name} where k1 IS NOT NULL group by ${agg_expr};"
Expand Down Expand Up @@ -102,29 +105,35 @@ suite("two_level_nestedtypes_with_s3data") {
}
def load_from_s3 = {table_name, uri_file, format ->
if (format == "csv") {
order_qt_sql_tvf """select c2, c9, c10, c12, c16 from local(
"file_path" = "${uri_file}",
"backend_id" = "${be_id}",
"column_separator"="|",
"format" = "${format}") order by c1 limit 10; """
order_qt_sql_s3 """select c2, c9, c10, c12, c16 from s3(
"uri" = "${uri_file}",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "${format}",
"column_separator"="|",
"read_json_by_line"="true") order by c1 limit 10; """
sql """
insert into ${table_name} select * from local(
"file_path" = "${uri_file}",
"backend_id" = "${be_id}",
"column_separator"="|",
"format" = "${format}") order by c1; """
insert into ${table_name} select * from s3(
"uri" = "${uri_file}",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "${format}",
"column_separator"="|",
"read_json_by_line"="true") order by c1; """
} else {
order_qt_sql_tvf """select c_bool, c_double, c_decimal, c_date, c_char from local(
"file_path" = "${uri_file}",
"backend_id" = "${be_id}",
"column_separator"="|",
"format" = "${format}") order by k1 limit 10;"""
order_qt_sql_s3 """select c_bool, c_double, c_decimal, c_date, c_char from s3(
"uri" = "${uri_file}",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "${format}",
"read_json_by_line"="true") order by k1 limit 10;"""
sql """
insert into ${table_name} select * from local(
"file_path" = "${uri_file}",
"backend_id" = "${be_id}",
"column_separator"="|",
"format" = "${format}") order by k1; """
insert into ${table_name} select * from s3(
"uri" = "${uri_file}",
"s3.access_key"= "${ak}",
"s3.secret_key" = "${sk}",
"format" = "${format}",
"read_json_by_line"="true") order by k1; """
}
// where to filter different format data
qt_select_doris """ select c_bool, c_double, c_decimal, c_date, c_char from ${table_name} where k1 IS NOT NULL order by k1 limit 10; """
Expand Down Expand Up @@ -180,30 +189,30 @@ suite("two_level_nestedtypes_with_s3data") {
for (int fi = 0; fi < format_order.size(); ++fi) {
String form = format_order[fi]
sql "truncate table ${table_name};"
load_from_tvf(table_name, array_files[ffi], form)
load_from_s3(table_name, array_files[ffi], form)
++ ffi
}
qt_select_count_array """ select count() from ${table_name}; """
}



for (int i = 0; i < 2; ++i) {
String table_name = table_names[i] // array-array, array-map
for (int i = 0; i < 3; ++i) {
String table_name = table_names[i] // array-array, array-map, array-struct
// select element_at(column)
for (String col : colNameArr) {
// first
order_qt_select_arr "select ${col}[1] from ${table_name} where k1 IS NOT NULL order by k1 limit 10;"
// last
order_qt_select_arr "select ${col}[-1] from ${table_name} where k1 IS NOT NULL order by k1 limit 10;"
// null
order_qt_select_arr "select ${col}[0] from ${table_name} where k1 IS NOT NULL order by k1 limit 10;"
order_qt_select_arr_null "select ${col}[0] from ${table_name} where k1 IS NOT NULL order by k1 limit 10;"
// null
order_qt_select_arr "select ${col}[1000] from ${table_name} where k1 IS NOT NULL order by k1 limit 10;"
order_qt_select_arr_null "select ${col}[1000] from ${table_name} where k1 IS NOT NULL order by k1 limit 10;"
}
// select * from table where element_at(column) with equal expr
for (String col : colNameArr) {
order_qt_select_arr "select ${col}[1], ${col}[-1] from ${table_name} where ${col}[1][1] IS NOT NULL AND k1 IS NOT NULL order by k1 limit 10;"
order_qt_select_arr "select ${col}[1], ${col}[-1] from ${table_name} WHERE k1 IS NOT NULL order by k1 limit 10;"
}
// select * from table where groupby|orderby column will meet exception
for (String col : colNameArr) {
Expand All @@ -218,10 +227,13 @@ suite("two_level_nestedtypes_with_s3data") {
// most-nested-column
// array-array
String agg_expr = "${colNameArr[0]}[1][1]"
select_nested_scala_element_at(agg_expr, table_names[0])

groupby_or_orderby_element_at(true, table_names[0], agg_expr)
groupby_or_orderby_element_at(false, table_names[0], agg_expr)
// array-map
agg_expr = "${colNameArr[0]}[1][map_keys(${colNameArr[0]}[1])[1]]"
select_nested_scala_element_at(agg_expr, table_names[1])
groupby_or_orderby_element_at(true, table_names[1], agg_expr)
groupby_or_orderby_element_at(false, table_names[1], agg_expr)

Expand Down Expand Up @@ -264,27 +276,27 @@ suite("two_level_nestedtypes_with_s3data") {
for (int fi = 0; fi < format_order.size(); ++fi) {
String form = format_order[fi]
sql "truncate table ${table_name};"
load_from_tvf(table_name, map_files[ffi], form)
load_from_s3(table_name, map_files[ffi], form)
++ ffi
}
qt_select_count_map """ select count() from ${table_name}; """
}
for (int i = 3; i < 4; ++i ) {
String table_name = table_names[i] // map-array, map-map
for (int i = 3; i < 6; ++i ) {
String table_name = table_names[i] // map-array map-map map-struct
// select element_at(column)
for (String col : colNameArr) {
// first
order_qt_select_map "select ${col}[map_keys(${col})[1]] from ${table_name} where k1 IS NOT NULL order by k1 limit 10;"
// last
order_qt_select_map "select ${col}[map_keys(${col})[-1]] from ${table_name} where k1 IS NOT NULL order by k1 limit 10;"
// null
order_qt_select_map "select ${col}[map_keys(${col})[0]] from ${table_name} where k1 IS NOT NULL order by k1 limit 10;"
order_qt_select_map_null "select ${col}[map_keys(${col})[0]] from ${table_name} where k1 IS NOT NULL order by k1 limit 10;"
// null
order_qt_select_map "select ${col}[map_keys(${col})[1000]] from ${table_name} where k1 IS NOT NULL order by k1 limit 10;"
order_qt_select_map_null "select ${col}[map_keys(${col})[1000]] from ${table_name} where k1 IS NOT NULL order by k1 limit 10;"
}
// select * from table where element_at(column) with equal expr
for (String col : colNameArr) {
order_qt_select_map "select ${col}[map_keys(${col})[1]], ${col}[map_keys(${col})[-1]] from ${table_name} where ${col}[1][1] IS NOT NULL AND k1 IS NOT NULL order by k1 limit 10;"
order_qt_select_map "select ${col}[map_keys(${col})[1]], ${col}[map_keys(${col})[-1]] from ${table_name} where k1 IS NOT NULL order by k1 limit 10;"
}
// select * from table where groupby|orderby column will meet exception
for (String col : colNameArr) {
Expand All @@ -301,17 +313,19 @@ suite("two_level_nestedtypes_with_s3data") {
// most-nested-column
// map-array
agg_expr = "${colNameArr[0]}[map_keys(${colNameArr[0]})[1]][1]"
select_nested_scala_element_at(agg_expr, table_names[3])
groupby_or_orderby_element_at(true, table_names[3], agg_expr)
groupby_or_orderby_element_at(false, table_names[3], agg_expr)
// map-map
agg_expr = "${colNameArr[0]}[map_keys(${colNameArr[0]})[1]][map_keys(${colNameArr[0]}[map_keys(${colNameArr[0]})[1]])[1]]"
select_nested_scala_element_at(agg_expr, table_names[4])
groupby_or_orderby_element_at(true, table_names[4], agg_expr)
groupby_or_orderby_element_at(false, table_names[4], agg_expr)
// map-struct
// select element_at(column)
order_qt_select_map "select struct_element(${colNameArr[0]}[map_keys(${colNameArr[0]})[1]], 1), struct_element(${colNameArr[0]}[map_keys(${colNameArr[0]})[1]], 'col17') from ${table_names[5]} where k1 IS NOT NULL order by k1 limit 10;"
// select * from table where element_at(column) with equal expr
order_qt_select_map "select struct_element(${colNameArr[0]}[map_keys(${colNameArr[0]})[1]], 1), struct_element(${colNameArr[0]}[map_keys(${colNameArr[0]})[1]], 'col17') from ${table_names[5]} where struct_element(${colNameArr[0]}[map_keys(${colNameArr[0]})[1]], 1) IS NOT NULL AND k1 IS NOT NULL order by k1 limit 10;"
order_qt_select_map "select struct_element(${colNameArr[0]}[map_keys(${colNameArr[0]})[1]], 1), struct_element(${colNameArr[0]}[map_keys(${colNameArr[0]})[1]], 'col17') from ${table_names[5]} where k1 IS NOT NULL order by k1 limit 10;"
// select * from table where groupby|orderby column will meet exception
groupby_or_orderby_exception(true, table_names[5], colNameArr[0])
groupby_or_orderby_exception(false, table_names[5], colNameArr[0])
Expand Down Expand Up @@ -345,13 +359,13 @@ suite("two_level_nestedtypes_with_s3data") {
for (int fi = 0; fi < format_order.size(); ++fi) {
String form = format_order[fi]
sql "truncate table ${table_name};"
load_from_tvf(table_name, struct_files[ffi], form)
load_from_s3(table_name, struct_files[ffi], form)
++ ffi
}
qt_select_count_struct """ select count() from ${table_name}; """
}
for (int i = 6; i < 8; ++i ) {
String table_name = table_names[i] // struct-array, struct-map
String table_name = table_names[i] // struct-array, struct-map, struct-struct
// select element_at(column)
for (String col : colNameArr) {
order_qt_select_struct "select struct_element(${colNameArr[0]}, 1), struct_element(${colNameArr[0]}, 'col_1') from ${table_name} where k1 IS NOT NULL order by k1 limit 10;"
Expand Down Expand Up @@ -387,17 +401,19 @@ suite("two_level_nestedtypes_with_s3data") {
// most-nested-column
// struct-array
agg_expr = "struct_element(${colNameArr[0]}, 1)[1]"
select_nested_scala_element_at(agg_expr, table_names[6])
groupby_or_orderby_element_at(true, table_names[6], agg_expr)
groupby_or_orderby_element_at(false, table_names[6], agg_expr)
// struct-map
agg_expr = "struct_element(${colNameArr[0]}, 1)[map_keys(struct_element(${colNameArr[0]}, 1))[1]]"
select_nested_scala_element_at(agg_expr, table_names[7])
groupby_or_orderby_element_at(true, table_names[7], agg_expr)
groupby_or_orderby_element_at(false, table_names[7], agg_expr)
// struct-struct
// select element_at(column)
order_qt_select_struct "select struct_element(struct_element(${colNameArr[0]}, 1), 1), struct_element(struct_element(${colNameArr[0]}, 1), 'col1') from ${table_names[8]} where k1 IS NOT NULL order by k1 limit 10;"
// select * from table where element_at(column) with equal expr
order_qt_select_struct "select struct_element(struct_element(${colNameArr[0]}, 1), 1), struct_element(struct_element(${colNameArr[0]}, 1), 'col1') from ${table_names[8]} where struct_element(struct_element(${colNameArr[0]}, 1), 1) IS NOT NULL AND k1 IS NOT NULL order by k1 limit 10;"
order_qt_select_struct "select struct_element(struct_element(${colNameArr[0]}, 1), 1), struct_element(struct_element(${colNameArr[0]}, 1), 'col1') from ${table_names[8]} where k1 IS NOT NULL order by k1 limit 10;"
// select * from table where groupby|orderby column will meet exception
groupby_or_orderby_exception(true, table_names[8], colNameArr[0])
groupby_or_orderby_exception(false, table_names[8], colNameArr[0])
Expand Down

0 comments on commit ab4d370

Please sign in to comment.