From 45ebc3923e86a0d6f8ac7aeca7138d500cbcf19c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 28 Oct 2025 05:54:03 +0000 Subject: [PATCH] Update big5 ppl queries and check plans (#4668) * Update big5 ppl queries Signed-off-by: Lantao Jin * Add explain check for big5 queries Signed-off-by: Lantao Jin * update to latest code base Signed-off-by: Lantao Jin --------- Signed-off-by: Lantao Jin (cherry picked from commit 0f453825919178991f5e432941b92a1f8b901d19) Signed-off-by: github-actions[bot] --- .../sql/calcite/big5/CalcitePPLBig5IT.java | 9 +- .../sql/calcite/big5/PPLBig5IT.java | 222 ++++++++++++++---- .../org/opensearch/sql/ppl/ExplainIT.java | 14 -- .../opensearch/sql/ppl/PPLIntegTestCase.java | 14 ++ .../big5/queries/asc_sort_timestamp.ppl | 15 ++ .../asc_sort_timestamp_can_match_shortcut.ppl | 18 ++ ...c_sort_timestamp_no_can_match_shortcut.ppl | 21 ++ .../queries/asc_sort_with_after_timestamp.ppl | 21 ++ .../big5/queries/cardinality_agg_high.ppl | 22 ++ .../big5/queries/cardinality_agg_high_2.ppl | 21 ++ .../big5/queries/cardinality_agg_low.ppl | 19 ++ .../composite_date_histogram_daily.ppl | 31 +++ .../big5/queries/composite_terms.ppl | 30 ++- .../big5/queries/composite_terms_keyword.ppl | 31 ++- .../queries/date_histogram_hourly_agg.ppl | 22 ++ .../queries/date_histogram_minute_agg.ppl | 30 +++ .../test/resources/big5/queries/default.ppl | 12 + .../big5/queries/desc_sort_timestamp.ppl | 15 ++ ...desc_sort_timestamp_can_match_shortcut.ppl | 18 ++ ...c_sort_timestamp_no_can_match_shortcut.ppl | 21 ++ .../desc_sort_with_after_timestamp.ppl | 21 ++ .../big5/queries/keyword_in_range.ppl | 31 ++- .../resources/big5/queries/keyword_terms.ppl | 18 ++ .../queries/keyword_terms_low_cardinality.ppl | 20 +- .../big5/queries/multi_terms_keyword.ppl | 40 +++- .../optimized/cardinality_agg_high.ppl | 22 ++ .../optimized/cardinality_agg_high_2.ppl | 21 ++ .../queries/optimized/cardinality_agg_low.ppl | 19 ++ .../queries/optimized/composite_terms.ppl | 32 +++ .../optimized/composite_terms_keyword.ppl | 33 +++ .../big5/queries/optimized/keyword_terms.ppl | 22 ++ .../keyword_terms_low_cardinality.ppl | 22 ++ .../queries/optimized/multi_terms_keyword.ppl | 38 +++ .../optimized/range_auto_date_histo.ppl | 59 +++++ .../range_auto_date_histo_with_metrics.ppl | 67 ++++++ .../big5/queries/query_string_on_message.ppl | 16 +- .../query_string_on_message_filtered.ppl | 34 ++- ..._string_on_message_filtered_sorted_num.ppl | 43 +++- .../src/test/resources/big5/queries/range.ppl | 17 ++ .../resources/big5/queries/range_agg_1.ppl | 50 ++++ .../resources/big5/queries/range_agg_2.ppl | 40 ++++ .../big5/queries/range_auto_date_histo.ppl | 49 ++++ .../range_auto_date_histo_with_metrics.ppl | 69 +++++- ...d_conjunction_big_range_big_term_query.ppl | 32 ++- ...conjunction_small_range_big_term_query.ppl | 25 +- ...njunction_small_range_small_term_query.ppl | 31 ++- ...disjunction_big_range_small_term_query.ppl | 31 ++- .../resources/big5/queries/range_numeric.ppl | 19 +- .../big5/queries/range_with_asc_sort.ppl | 23 +- .../big5/queries/range_with_desc_sort.ppl | 23 +- .../test/resources/big5/queries/scroll.ppl | 15 ++ .../sort_keyword_can_match_shortcut.ppl | 20 +- .../sort_keyword_no_can_match_shortcut.ppl | 23 +- .../big5/queries/sort_numeric_asc.ppl | 18 ++ .../queries/sort_numeric_asc_with_match.ppl | 20 ++ .../big5/queries/sort_numeric_desc.ppl | 18 ++ .../queries/sort_numeric_desc_with_match.ppl | 20 ++ .../src/test/resources/big5/queries/term.ppl | 17 ++ .../big5/queries/terms_significant_1.ppl | 42 +++- .../big5/queries/terms_significant_2.ppl | 41 +++- .../calcite/asc_sort_timestamp.yaml | 13 + ...asc_sort_timestamp_can_match_shortcut.yaml | 14 ++ ..._sort_timestamp_no_can_match_shortcut.yaml | 14 ++ .../asc_sort_with_after_timestamp.yaml | 13 + .../calcite/cardinality_agg_high.yaml | 9 + .../calcite/cardinality_agg_high_2.yaml | 9 + .../calcite/cardinality_agg_low.yaml | 9 + .../composite_date_histogram_daily.yaml | 11 + .../calcite/composite_terms.yaml | 12 + .../calcite/composite_terms_keyword.yaml | 12 + .../calcite/date_histogram_hourly_agg.yaml | 10 + .../calcite/date_histogram_minute_agg.yaml | 11 + .../expectedOutput/calcite/default.yaml | 8 + .../calcite/desc_sort_timestamp.yaml | 13 + ...esc_sort_timestamp_can_match_shortcut.yaml | 14 ++ ..._sort_timestamp_no_can_match_shortcut.yaml | 14 ++ .../desc_sort_with_after_timestamp.yaml | 13 + .../calcite/keyword_in_range.yaml | 10 + .../expectedOutput/calcite/keyword_terms.yaml | 11 + .../keyword_terms_low_cardinality.yaml | 11 + .../calcite/multi_terms_keyword.yaml | 12 + .../calcite/query_string_on_message.yaml | 9 + .../query_string_on_message_filtered.yaml | 9 + ...string_on_message_filtered_sorted_num.yaml | 14 ++ .../expectedOutput/calcite/range.yaml | 9 + .../expectedOutput/calcite/range_agg_1.yaml | 10 + .../expectedOutput/calcite/range_agg_2.yaml | 10 + .../calcite/range_auto_date_histo.yaml | 10 + .../range_auto_date_histo_with_metrics.yaml | 10 + ..._conjunction_big_range_big_term_query.yaml | 9 + ...onjunction_small_range_big_term_query.yaml | 9 + ...junction_small_range_small_term_query.yaml | 9 + ...isjunction_big_range_small_term_query.yaml | 9 + .../expectedOutput/calcite/range_numeric.yaml | 9 + .../calcite/range_with_asc_sort.yaml | 14 ++ .../calcite/range_with_desc_sort.yaml | 14 ++ .../expectedOutput/calcite/scroll.yaml | 8 + .../sort_keyword_can_match_shortcut.yaml | 14 ++ .../sort_keyword_no_can_match_shortcut.yaml | 14 ++ .../calcite/sort_numeric_asc.yaml | 13 + .../calcite/sort_numeric_asc_with_match.yaml | 14 ++ .../calcite/sort_numeric_desc.yaml | 13 + .../calcite/sort_numeric_desc_with_match.yaml | 14 ++ .../expectedOutput/calcite/term.yaml | 9 + .../calcite/terms_significant_1.yaml | 11 + .../calcite/terms_significant_2.yaml | 11 + .../ppl/asc_sort_timestamp.yaml | 16 ++ ...asc_sort_timestamp_can_match_shortcut.yaml | 20 ++ ..._sort_timestamp_no_can_match_shortcut.yaml | 20 ++ .../ppl/asc_sort_with_after_timestamp.yaml | 16 ++ .../ppl/cardinality_agg_high.yaml | 12 + .../ppl/cardinality_agg_high_2.yaml | 12 + .../ppl/cardinality_agg_low.yaml | 12 + .../ppl/composite_date_histogram_daily.yaml | 19 ++ .../expectedOutput/ppl/composite_terms.yaml | 21 ++ .../ppl/composite_terms_keyword.yaml | 23 ++ .../ppl/date_histogram_hourly_agg.yaml | 15 ++ .../ppl/date_histogram_minute_agg.yaml | 19 ++ .../resources/expectedOutput/ppl/default.yaml | 15 ++ .../ppl/desc_sort_timestamp.yaml | 16 ++ ...esc_sort_timestamp_can_match_shortcut.yaml | 20 ++ ..._sort_timestamp_no_can_match_shortcut.yaml | 20 ++ .../ppl/desc_sort_with_after_timestamp.yaml | 16 ++ .../expectedOutput/ppl/keyword_in_range.yaml | 25 ++ .../expectedOutput/ppl/keyword_terms.yaml | 25 ++ .../ppl/keyword_terms_low_cardinality.yaml | 25 ++ .../ppl/multi_terms_keyword.yaml | 31 +++ .../ppl/query_string_on_message.yaml | 20 ++ .../ppl/query_string_on_message_filtered.yaml | 26 ++ ...string_on_message_filtered_sorted_num.yaml | 27 +++ .../resources/expectedOutput/ppl/range.yaml | 19 ++ .../expectedOutput/ppl/range_agg_1.yaml | 28 +++ .../expectedOutput/ppl/range_agg_2.yaml | 26 ++ .../ppl/range_auto_date_histo.yaml | 38 +++ .../range_auto_date_histo_with_metrics.yaml | 36 +++ ..._conjunction_big_range_big_term_query.yaml | 21 ++ ...onjunction_small_range_big_term_query.yaml | 19 ++ ...junction_small_range_small_term_query.yaml | 21 ++ ...isjunction_big_range_small_term_query.yaml | 21 ++ .../expectedOutput/ppl/range_numeric.yaml | 19 ++ .../ppl/range_with_asc_sort.yaml | 20 ++ .../ppl/range_with_desc_sort.yaml | 20 ++ .../resources/expectedOutput/ppl/scroll.yaml | 15 ++ .../ppl/sort_keyword_can_match_shortcut.yaml | 20 ++ .../sort_keyword_no_can_match_shortcut.yaml | 20 ++ .../expectedOutput/ppl/sort_numeric_asc.yaml | 16 ++ .../ppl/sort_numeric_asc_with_match.yaml | 21 ++ .../expectedOutput/ppl/sort_numeric_desc.yaml | 16 ++ .../ppl/sort_numeric_desc_with_match.yaml | 21 ++ .../resources/expectedOutput/ppl/term.yaml | 16 ++ .../ppl/terms_significant_1.yaml | 27 +++ .../ppl/terms_significant_2.yaml | 27 +++ 152 files changed, 3175 insertions(+), 108 deletions(-) create mode 100644 integ-test/src/test/resources/big5/queries/cardinality_agg_high.ppl create mode 100644 integ-test/src/test/resources/big5/queries/cardinality_agg_high_2.ppl create mode 100644 integ-test/src/test/resources/big5/queries/cardinality_agg_low.ppl create mode 100644 integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_high.ppl create mode 100644 integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_high_2.ppl create mode 100644 integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_low.ppl create mode 100644 integ-test/src/test/resources/big5/queries/optimized/composite_terms.ppl create mode 100644 integ-test/src/test/resources/big5/queries/optimized/composite_terms_keyword.ppl create mode 100644 integ-test/src/test/resources/big5/queries/optimized/keyword_terms.ppl create mode 100644 integ-test/src/test/resources/big5/queries/optimized/keyword_terms_low_cardinality.ppl create mode 100644 integ-test/src/test/resources/big5/queries/optimized/multi_terms_keyword.ppl create mode 100644 integ-test/src/test/resources/big5/queries/optimized/range_auto_date_histo.ppl create mode 100644 integ-test/src/test/resources/big5/queries/optimized/range_auto_date_histo_with_metrics.ppl create mode 100644 integ-test/src/test/resources/big5/queries/range_agg_1.ppl create mode 100644 integ-test/src/test/resources/big5/queries/range_agg_2.ppl create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp_can_match_shortcut.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp_no_can_match_shortcut.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/asc_sort_with_after_timestamp.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_high.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_high_2.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_low.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/composite_date_histogram_daily.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/composite_terms.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/composite_terms_keyword.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/date_histogram_hourly_agg.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/date_histogram_minute_agg.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/default.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp_can_match_shortcut.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp_no_can_match_shortcut.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/desc_sort_with_after_timestamp.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/keyword_in_range.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/keyword_terms.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/keyword_terms_low_cardinality.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/multi_terms_keyword.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message_filtered.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message_filtered_sorted_num.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/range.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/range_agg_1.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/range_agg_2.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/range_auto_date_histo.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/range_auto_date_histo_with_metrics.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_big_range_big_term_query.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_small_range_big_term_query.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_small_range_small_term_query.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/range_field_disjunction_big_range_small_term_query.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/range_numeric.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/range_with_asc_sort.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/range_with_desc_sort.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/scroll.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/sort_keyword_can_match_shortcut.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/sort_keyword_no_can_match_shortcut.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_asc.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_asc_with_match.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_desc.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_desc_with_match.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/term.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/terms_significant_1.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/terms_significant_2.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp_can_match_shortcut.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp_no_can_match_shortcut.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/asc_sort_with_after_timestamp.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_high.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_high_2.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_low.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/composite_date_histogram_daily.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/composite_terms.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/composite_terms_keyword.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/date_histogram_hourly_agg.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/date_histogram_minute_agg.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/default.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp_can_match_shortcut.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp_no_can_match_shortcut.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/desc_sort_with_after_timestamp.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/keyword_in_range.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/keyword_terms.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/keyword_terms_low_cardinality.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/multi_terms_keyword.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message_filtered.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message_filtered_sorted_num.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/range.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/range_agg_1.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/range_agg_2.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/range_auto_date_histo.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/range_auto_date_histo_with_metrics.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_big_range_big_term_query.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_small_range_big_term_query.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_small_range_small_term_query.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/range_field_disjunction_big_range_small_term_query.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/range_numeric.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/range_with_asc_sort.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/range_with_desc_sort.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/scroll.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/sort_keyword_can_match_shortcut.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/sort_keyword_no_can_match_shortcut.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_asc.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_asc_with_match.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_desc.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_desc_with_match.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/term.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/terms_significant_1.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/terms_significant_2.yaml diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java index cc49a571778..665b3f0a874 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java @@ -21,24 +21,25 @@ public void init() throws Exception { @Test public void bin_bins() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/bin_bins.ppl")); + String ppl = sanitize(loadExpectedQuery("bin_bins.ppl")); timing(summary, "bin_bins", ppl); } @Test public void bin_span_log() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/bin_span_log.ppl")); + String ppl = sanitize(loadExpectedQuery("bin_span_log.ppl")); timing(summary, "bin_span_log", ppl); } @Test public void bin_span_time() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/bin_span_time.ppl")); + String ppl = sanitize(loadExpectedQuery("bin_span_time.ppl")); timing(summary, "bin_span_time", ppl); } + @Test public void coalesce_nonexistent_field_fallback() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/coalesce_nonexistent_field_fallback.ppl")); + String ppl = sanitize(loadExpectedQuery("coalesce_nonexistent_field_fallback.ppl")); timing(summary, "coalesce_nonexistent_field_fallback", ppl); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java index ae1e9881173..4997d361203 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.calcite.big5; +import static org.opensearch.sql.util.MatcherUtils.assertYamlEqualsIgnoreId; + import java.io.IOException; import java.util.Locale; import java.util.Map; @@ -49,257 +51,383 @@ public static void reset() throws IOException { @Test public void asc_sort_timestamp() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/asc_sort_timestamp.ppl")); + String ppl = sanitize(loadExpectedQuery("asc_sort_timestamp.ppl")); timing(summary, "asc_sort_timestamp", ppl); + String expected = loadExpectedPlan("asc_sort_timestamp.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void asc_sort_timestamp_can_match_shortcut() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/asc_sort_timestamp_can_match_shortcut.ppl")); + String ppl = sanitize(loadExpectedQuery("asc_sort_timestamp_can_match_shortcut.ppl")); timing(summary, "asc_sort_timestamp_can_match_shortcut", ppl); + String expected = loadExpectedPlan("asc_sort_timestamp_can_match_shortcut.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void asc_sort_timestamp_no_can_match_shortcut() throws IOException { - String ppl = - sanitize(loadFromFile("big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl")); + String ppl = sanitize(loadExpectedQuery("asc_sort_timestamp_no_can_match_shortcut.ppl")); timing(summary, "asc_sort_timestamp_no_can_match_shortcut", ppl); + String expected = loadExpectedPlan("asc_sort_timestamp_no_can_match_shortcut.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void asc_sort_with_after_timestamp() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/asc_sort_with_after_timestamp.ppl")); + String ppl = sanitize(loadExpectedQuery("asc_sort_with_after_timestamp.ppl")); timing(summary, "asc_sort_with_after_timestamp", ppl); + String expected = loadExpectedPlan("asc_sort_with_after_timestamp.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void composite_date_histogram_daily() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/composite_date_histogram_daily.ppl")); + String ppl = sanitize(loadExpectedQuery("composite_date_histogram_daily.ppl")); timing(summary, "composite_date_histogram_daily", ppl); + String expected = loadExpectedPlan("composite_date_histogram_daily.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void composite_terms_keyword() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/composite_terms_keyword.ppl")); + String ppl = sanitize(loadExpectedQuery("composite_terms_keyword.ppl")); timing(summary, "composite_terms_keyword", ppl); + String expected = loadExpectedPlan("composite_terms_keyword.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void composite_terms() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/composite_terms.ppl")); + String ppl = sanitize(loadExpectedQuery("composite_terms.ppl")); timing(summary, "composite_terms", ppl); + String expected = loadExpectedPlan("composite_terms.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void date_histogram_hourly_agg() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/date_histogram_hourly_agg.ppl")); + String ppl = sanitize(loadExpectedQuery("date_histogram_hourly_agg.ppl")); timing(summary, "date_histogram_hourly_agg", ppl); + String expected = loadExpectedPlan("date_histogram_hourly_agg.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void date_histogram_minute_agg() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/date_histogram_minute_agg.ppl")); + String ppl = sanitize(loadExpectedQuery("date_histogram_minute_agg.ppl")); timing(summary, "date_histogram_minute_agg", ppl); + String expected = loadExpectedPlan("date_histogram_minute_agg.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void test_default() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/default.ppl")); + String ppl = sanitize(loadExpectedQuery("default.ppl")); timing(summary, "default", ppl); + String expected = loadExpectedPlan("default.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void desc_sort_timestamp() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/desc_sort_timestamp.ppl")); + String ppl = sanitize(loadExpectedQuery("desc_sort_timestamp.ppl")); timing(summary, "desc_sort_timestamp", ppl); + String expected = loadExpectedPlan("desc_sort_timestamp.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void desc_sort_timestamp_can_match_shortcut() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/desc_sort_timestamp_can_match_shortcut.ppl")); + String ppl = sanitize(loadExpectedQuery("desc_sort_timestamp_can_match_shortcut.ppl")); timing(summary, "desc_sort_timestamp_can_match_shortcut", ppl); + String expected = loadExpectedPlan("desc_sort_timestamp_can_match_shortcut.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void desc_sort_timestamp_no_can_match_shortcut() throws IOException { - String ppl = - sanitize(loadFromFile("big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl")); + String ppl = sanitize(loadExpectedQuery("desc_sort_timestamp_no_can_match_shortcut.ppl")); timing(summary, "desc_sort_timestamp_no_can_match_shortcut", ppl); + String expected = loadExpectedPlan("desc_sort_timestamp_no_can_match_shortcut.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void desc_sort_with_after_timestamp() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/desc_sort_with_after_timestamp.ppl")); + String ppl = sanitize(loadExpectedQuery("desc_sort_with_after_timestamp.ppl")); timing(summary, "desc_sort_with_after_timestamp", ppl); + String expected = loadExpectedPlan("desc_sort_with_after_timestamp.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void keyword_in_range() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/keyword_in_range.ppl")); + String ppl = sanitize(loadExpectedQuery("keyword_in_range.ppl")); timing(summary, "keyword_in_range", ppl); + String expected = loadExpectedPlan("keyword_in_range.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void keyword_terms() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/keyword_terms.ppl")); + String ppl = sanitize(loadExpectedQuery("keyword_terms.ppl")); timing(summary, "keyword_terms", ppl); + String expected = loadExpectedPlan("keyword_terms.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void keyword_terms_low_cardinality() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/keyword_terms_low_cardinality.ppl")); + String ppl = sanitize(loadExpectedQuery("keyword_terms_low_cardinality.ppl")); timing(summary, "keyword_terms_low_cardinality", ppl); + String expected = loadExpectedPlan("keyword_terms_low_cardinality.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void multi_terms_keyword() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/multi_terms_keyword.ppl")); + String ppl = sanitize(loadExpectedQuery("multi_terms_keyword.ppl")); timing(summary, "multi_terms_keyword", ppl); + String expected = loadExpectedPlan("multi_terms_keyword.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void query_string_on_message() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/query_string_on_message.ppl")); + String ppl = sanitize(loadExpectedQuery("query_string_on_message.ppl")); timing(summary, "query_string_on_message", ppl); + String expected = loadExpectedPlan("query_string_on_message.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void query_string_on_message_filtered() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/query_string_on_message_filtered.ppl")); + String ppl = sanitize(loadExpectedQuery("query_string_on_message_filtered.ppl")); timing(summary, "query_string_on_message_filtered", ppl); + String expected = loadExpectedPlan("query_string_on_message_filtered.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void query_string_on_message_filtered_sorted_num() throws IOException { - String ppl = - sanitize(loadFromFile("big5/queries/query_string_on_message_filtered_sorted_num.ppl")); + String ppl = sanitize(loadExpectedQuery("query_string_on_message_filtered_sorted_num.ppl")); timing(summary, "query_string_on_message_filtered_sorted_num", ppl); + String expected = loadExpectedPlan("query_string_on_message_filtered_sorted_num.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/range.ppl")); + String ppl = sanitize(loadExpectedQuery("range.ppl")); timing(summary, "range", ppl); + String expected = loadExpectedPlan("range.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_auto_date_histo() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/range_auto_date_histo.ppl")); + String ppl = sanitize(loadExpectedQuery("range_auto_date_histo.ppl")); timing(summary, "range_auto_date_histo", ppl); + String expected = loadExpectedPlan("range_auto_date_histo.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_auto_date_histo_with_metrics() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/range_auto_date_histo_with_metrics.ppl")); + String ppl = sanitize(loadExpectedQuery("range_auto_date_histo_with_metrics.ppl")); timing(summary, "range_auto_date_histo_with_metrics", ppl); + String expected = loadExpectedPlan("range_auto_date_histo_with_metrics.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_numeric() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/range_numeric.ppl")); + String ppl = sanitize(loadExpectedQuery("range_numeric.ppl")); timing(summary, "range_numeric", ppl); + String expected = loadExpectedPlan("range_numeric.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_field_conjunction_big_range_big_term_query() throws IOException { String ppl = - sanitize(loadFromFile("big5/queries/range_field_conjunction_big_range_big_term_query.ppl")); + sanitize(loadExpectedQuery("range_field_conjunction_big_range_big_term_query.ppl")); timing(summary, "range_field_conjunction_big_range_big_term_query", ppl); + String expected = loadExpectedPlan("range_field_conjunction_big_range_big_term_query.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_field_conjunction_small_range_big_term_query() throws IOException { String ppl = - sanitize( - loadFromFile("big5/queries/range_field_conjunction_small_range_big_term_query.ppl")); + sanitize(loadExpectedQuery("range_field_conjunction_small_range_big_term_query.ppl")); timing(summary, "range_field_conjunction_small_range_big_term_query", ppl); + String expected = loadExpectedPlan("range_field_conjunction_small_range_big_term_query.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_field_conjunction_small_range_small_term_query() throws IOException { String ppl = - sanitize( - loadFromFile("big5/queries/range_field_conjunction_small_range_small_term_query.ppl")); + sanitize(loadExpectedQuery("range_field_conjunction_small_range_small_term_query.ppl")); timing(summary, "range_field_conjunction_small_range_small_term_query", ppl); + String expected = loadExpectedPlan("range_field_conjunction_small_range_small_term_query.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_field_disjunction_big_range_small_term_query() throws IOException { String ppl = - sanitize( - loadFromFile("big5/queries/range_field_disjunction_big_range_small_term_query.ppl")); + sanitize(loadExpectedQuery("range_field_disjunction_big_range_small_term_query.ppl")); timing(summary, "range_field_disjunction_big_range_small_term_query", ppl); + String expected = loadExpectedPlan("range_field_disjunction_big_range_small_term_query.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_with_asc_sort() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/range_with_asc_sort.ppl")); + String ppl = sanitize(loadExpectedQuery("range_with_asc_sort.ppl")); timing(summary, "range_with_asc_sort", ppl); + String expected = loadExpectedPlan("range_with_asc_sort.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void range_with_desc_sort() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/range_with_desc_sort.ppl")); + String ppl = sanitize(loadExpectedQuery("range_with_desc_sort.ppl")); timing(summary, "range_with_desc_sort", ppl); + String expected = loadExpectedPlan("range_with_desc_sort.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void scroll() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/scroll.ppl")); + String ppl = sanitize(loadExpectedQuery("scroll.ppl")); timing(summary, "scroll", ppl); + String expected = loadExpectedPlan("scroll.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void sort_keyword_can_match_shortcut() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/sort_keyword_can_match_shortcut.ppl")); + String ppl = sanitize(loadExpectedQuery("sort_keyword_can_match_shortcut.ppl")); timing(summary, "sort_keyword_can_match_shortcut", ppl); + String expected = loadExpectedPlan("sort_keyword_can_match_shortcut.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void sort_keyword_no_can_match_shortcut() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/sort_keyword_no_can_match_shortcut.ppl")); + String ppl = sanitize(loadExpectedQuery("sort_keyword_no_can_match_shortcut.ppl")); timing(summary, "sort_keyword_no_can_match_shortcut", ppl); + String expected = loadExpectedPlan("sort_keyword_no_can_match_shortcut.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void sort_numeric_asc() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_asc.ppl")); + String ppl = sanitize(loadExpectedQuery("sort_numeric_asc.ppl")); timing(summary, "sort_numeric_asc", ppl); + String expected = loadExpectedPlan("sort_numeric_asc.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void sort_numeric_asc_with_match() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_asc_with_match.ppl")); + String ppl = sanitize(loadExpectedQuery("sort_numeric_asc_with_match.ppl")); timing(summary, "sort_numeric_asc_with_match", ppl); + String expected = loadExpectedPlan("sort_numeric_asc_with_match.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void sort_numeric_desc() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_desc.ppl")); + String ppl = sanitize(loadExpectedQuery("sort_numeric_desc.ppl")); timing(summary, "sort_numeric_desc", ppl); + String expected = loadExpectedPlan("sort_numeric_desc.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void sort_numeric_desc_with_match() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_desc_with_match.ppl")); + String ppl = sanitize(loadExpectedQuery("sort_numeric_desc_with_match.ppl")); timing(summary, "sort_numeric_desc_with_match", ppl); + String expected = loadExpectedPlan("sort_numeric_desc_with_match.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void term() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/term.ppl")); + String ppl = sanitize(loadExpectedQuery("term.ppl")); timing(summary, "term", ppl); + String expected = loadExpectedPlan("term.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void terms_significant_1() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/terms_significant_1.ppl")); + String ppl = sanitize(loadExpectedQuery("terms_significant_1.ppl")); timing(summary, "terms_significant_1", ppl); + String expected = loadExpectedPlan("terms_significant_1.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); } @Test public void terms_significant_2() throws IOException { - String ppl = sanitize(loadFromFile("big5/queries/terms_significant_2.ppl")); + String ppl = sanitize(loadExpectedQuery("terms_significant_2.ppl")); timing(summary, "terms_significant_2", ppl); + String expected = loadExpectedPlan("terms_significant_2.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); + } + + @Test + public void range_agg_1() throws IOException { + String ppl = sanitize(loadExpectedQuery("range_agg_1.ppl")); + timing(summary, "range_agg_1", ppl); + String expected = loadExpectedPlan("range_agg_1.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); + } + + @Test + public void range_agg_2() throws IOException { + String ppl = sanitize(loadExpectedQuery("range_agg_2.ppl")); + timing(summary, "range_agg_2", ppl); + String expected = loadExpectedPlan("range_agg_2.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); + } + + @Test + public void cardinality_agg_high() throws IOException { + String ppl = sanitize(loadExpectedQuery("cardinality_agg_high.ppl")); + timing(summary, "cardinality_agg_high", ppl); + String expected = loadExpectedPlan("cardinality_agg_high.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); + } + + @Test + public void cardinality_agg_high_2() throws IOException { + String ppl = sanitize(loadExpectedQuery("cardinality_agg_high_2.ppl")); + timing(summary, "cardinality_agg_high_2", ppl); + String expected = loadExpectedPlan("cardinality_agg_high_2.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); + } + + @Test + public void cardinality_agg_low() throws IOException { + String ppl = sanitize(loadExpectedQuery("cardinality_agg_low.ppl")); + timing(summary, "cardinality_agg_low", ppl); + String expected = loadExpectedPlan("cardinality_agg_low.yaml"); + assertYamlEqualsIgnoreId(expected, explainQueryYaml(ppl)); + } + + protected String loadExpectedQuery(String fileName) throws IOException { + if (isCalciteEnabled()) { + try { + return loadFromFile("big5/queries/optimized/" + fileName); + } catch (Exception e) { + } + } + return loadFromFile("big5/queries/" + fileName); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index 1f231feb262..1e713a1ab5b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -743,18 +743,4 @@ public void testExplainSearchWildcardStar() throws IOException { explainQueryToString( String.format("search source=%s severityText=ERR* | fields severityText ", TEST_INDEX_OTEL_LOGS))); } - - protected String loadExpectedPlan(String fileName) throws IOException { - String prefix; - if (isCalciteEnabled()) { - if (isPushdownDisabled()) { - prefix = "expectedOutput/calcite_no_pushdown/"; - } else { - prefix = "expectedOutput/calcite/"; - } - } else { - prefix = "expectedOutput/ppl/"; - } - return loadFromFile(prefix + fileName); - } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index 162440a10c8..f06e3ba775b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -402,4 +402,18 @@ protected static String loadFromFile(String filename) { throw new RuntimeException(e); } } + + protected String loadExpectedPlan(String fileName) throws IOException { + String prefix; + if (isCalciteEnabled()) { + if (isPushdownDisabled()) { + prefix = "expectedOutput/calcite_no_pushdown/"; + } else { + prefix = "expectedOutput/calcite/"; + } + } else { + prefix = "expectedOutput/ppl/"; + } + return loadFromFile(prefix + fileName); + } } diff --git a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp.ppl b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp.ppl index 7582d40d69a..f66d590a44d 100644 --- a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp.ppl +++ b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp.ppl @@ -1,3 +1,18 @@ +/* +{ + "name": "asc_sort_timestamp", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "match_all": {} + }, + "sort" : [ + {"@timestamp" : "asc"} + ] + } +} + */ source = big5 | sort + `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_can_match_shortcut.ppl index aab85fb7c1b..de766526c63 100644 --- a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_can_match_shortcut.ppl +++ b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_can_match_shortcut.ppl @@ -1,3 +1,21 @@ +/* +{ + "name": "asc_sort_timestamp_can_match_shortcut", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match": { + "process.name": "kernel" + } + }, + "sort" : [ + {"@timestamp" : "asc"} + ] + } +} +*/ source = big5 process.name=kernel | sort + `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl index aab85fb7c1b..4957fcd485f 100644 --- a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl +++ b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl @@ -1,3 +1,24 @@ +/* +{ + "name": "asc_sort_timestamp_no_can_match_shortcut", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-params" : { + "pre_filter_shard_size" : 100000 + }, + "body": { + "track_total_hits": false, + "query": { + "match": { + "process.name": "kernel" + } + }, + "sort" : [ + {"@timestamp" : "asc"} + ] + } +} +*/ source = big5 process.name=kernel | sort + `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/asc_sort_with_after_timestamp.ppl b/integ-test/src/test/resources/big5/queries/asc_sort_with_after_timestamp.ppl index 7582d40d69a..02e96b4f731 100644 --- a/integ-test/src/test/resources/big5/queries/asc_sort_with_after_timestamp.ppl +++ b/integ-test/src/test/resources/big5/queries/asc_sort_with_after_timestamp.ppl @@ -1,3 +1,24 @@ +/* +{ + "name": "asc_sort_with_after_timestamp", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match_all": {} + }, + "sort" : [ + {"@timestamp" : "asc"} + ], +{% if distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list or distribution_version.split('.') | map('int') | list >= "7.0.0".split('.') | map('int') | list %} + "search_after": ["2023-01-01T23:59:58.000Z"] +{% else %} + "search_after": [1673049598] +{% endif %} + } +} +*/ source = big5 | sort + `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/cardinality_agg_high.ppl b/integ-test/src/test/resources/big5/queries/cardinality_agg_high.ppl new file mode 100644 index 00000000000..0e8c3dec630 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/cardinality_agg_high.ppl @@ -0,0 +1,22 @@ +/* +{ + "name": "cardinality-agg-high", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "agent": { + "cardinality": { + "field": "agent.name" + {% if distribution_version.split('.') | map('int') | list >= "2.19.1".split('.') | map('int') | list and distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list %} + , "execution_hint": "ordinals" + {% endif %} + } + } + } + } +} +*/ +source = big5 +| stats dc(`agent.name`) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/cardinality_agg_high_2.ppl b/integ-test/src/test/resources/big5/queries/cardinality_agg_high_2.ppl new file mode 100644 index 00000000000..93130873f90 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/cardinality_agg_high_2.ppl @@ -0,0 +1,21 @@ +/* +{ + "name": "cardinality-agg-high-2", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 1800, + "body": { + "size": 0, + "aggs": { + "agent": { + "cardinality": { + "field": "event.id", + "execution_hint":"ordinals" + } + } + } + } +} +*/ +source = big5 +| stats dc(`event.id`) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/cardinality_agg_low.ppl b/integ-test/src/test/resources/big5/queries/cardinality_agg_low.ppl new file mode 100644 index 00000000000..ca6c8b214c3 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/cardinality_agg_low.ppl @@ -0,0 +1,19 @@ +/* +{ + "name": "cardinality-agg-low", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "region": { + "cardinality": { + "field": "cloud.region" + } + } + } + } +} +*/ +source = big5 +| stats dc(`cloud.region`) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl b/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl index caa27c86fba..656289b0603 100644 --- a/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl +++ b/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl @@ -1,3 +1,34 @@ +/* +{ + "name": "composite-date_histogram-daily", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2022-12-30T00:00:00", + "lt": "2023-01-07T12:00:00" + } + } + }, + "aggs": { + "logs": { + "composite": { + "sources": [ + {% if distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list or distribution_version.split('.') | map('int') | list >= "7.0.0".split('.') | map('int') | list %} + { "date": { "date_histogram": { "field": "@timestamp", "calendar_interval": "day" } } } + {% else %} + { "date": { "date_histogram": { "field": "@timestamp", "interval": "day" } } } + {% endif %} + ] + } + } + } + } +} +*/ source = big5 | where `@timestamp` >= '2022-12-30 00:00:00' and `@timestamp` < '2023-01-07 12:00:00' | stats count() by span(`@timestamp`, 1d) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/composite_terms.ppl b/integ-test/src/test/resources/big5/queries/composite_terms.ppl index 859e3c87e54..07edca09e69 100644 --- a/integ-test/src/test/resources/big5/queries/composite_terms.ppl +++ b/integ-test/src/test/resources/big5/queries/composite_terms.ppl @@ -1,4 +1,32 @@ +/* +{ + "name": "composite-terms", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-02T00:00:00", + "lt": "2023-01-02T10:00:00" + } + } + }, + "aggs": { + "logs": { + "composite": { + "sources": [ + { "process_name": { "terms": { "field": "process.name", "order": "desc" }}}, + { "cloud_region": { "terms": { "field": "cloud.region", "order": "asc" }}} + ] + } + } + } + } +} +*/ source = big5 -| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' +| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' | stats count() by `process.name`, `cloud.region` | sort - `process.name`, + `cloud.region` \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl b/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl index 5eb03e5fe2a..42b8c9585a4 100644 --- a/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl +++ b/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl @@ -1,4 +1,33 @@ +/* +{ + "name": "composite_terms-keyword", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-02T00:00:00", + "lt": "2023-01-02T10:00:00" + } + } + }, + "aggs": { + "logs": { + "composite": { + "sources": [ + { "process_name": { "terms": { "field": "process.name", "order": "desc" }}}, + { "cloud_region": { "terms": { "field": "cloud.region", "order": "asc" }}}, + { "cloudstream": { "terms": { "field": "aws.cloudwatch.log_stream", "order": "asc" }}} + ] + } + } + } + } +} +*/ source = big5 -| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' +| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' | stats count() by `process.name`, `cloud.region`, `aws.cloudwatch.log_stream` | sort - `process.name`, + `cloud.region`, + `aws.cloudwatch.log_stream` \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/date_histogram_hourly_agg.ppl b/integ-test/src/test/resources/big5/queries/date_histogram_hourly_agg.ppl index 054b915b335..4a340cf04d2 100644 --- a/integ-test/src/test/resources/big5/queries/date_histogram_hourly_agg.ppl +++ b/integ-test/src/test/resources/big5/queries/date_histogram_hourly_agg.ppl @@ -1,2 +1,24 @@ +/* +{ + "name": "date_histogram_hourly_agg", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "by_hour": { + "date_histogram": { + "field": "@timestamp", + {% if distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list or distribution_version.split('.') | map('int') | list >= "7.0.0".split('.') | map('int') | list %} + "calendar_interval": "hour" + {% else %} + "interval": "hour" + {% endif %} + } + } + } + } +} +*/ source = big5 | stats count() by span(`@timestamp`, 1h) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/date_histogram_minute_agg.ppl b/integ-test/src/test/resources/big5/queries/date_histogram_minute_agg.ppl index b9fd72abfb5..e7c647f213e 100644 --- a/integ-test/src/test/resources/big5/queries/date_histogram_minute_agg.ppl +++ b/integ-test/src/test/resources/big5/queries/date_histogram_minute_agg.ppl @@ -1,3 +1,33 @@ +/* +{ + "name": "date_histogram_minute_agg", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-01T00:00:00", + "lt": "2023-01-03T00:00:00" + } + } + }, + "aggs": { + "by_hour": { + "date_histogram": { + "field": "@timestamp", + {% if distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list or distribution_version.split('.') | map('int') | list >= "7.0.0".split('.') | map('int') | list %} + "calendar_interval": "minute" + {% else %} + "interval": "minute" + {% endif %} + } + } + } + } +} +*/ source = big5 | where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' | stats count() by span(`@timestamp`, 1m) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/default.ppl b/integ-test/src/test/resources/big5/queries/default.ppl index 6b63c414ac0..2aed0d33141 100644 --- a/integ-test/src/test/resources/big5/queries/default.ppl +++ b/integ-test/src/test/resources/big5/queries/default.ppl @@ -1,2 +1,14 @@ +/* +{ + "name": "match-all", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "match_all": {} + } + } +} +*/ source = big5 | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp.ppl b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp.ppl index af3445efdff..91ff8c5ad01 100644 --- a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp.ppl +++ b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp.ppl @@ -1,3 +1,18 @@ +/* +{ + "name": "desc_sort_timestamp", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "match_all": {} + }, + "sort" : [ + {"@timestamp" : "desc"} + ] + } +} +*/ source = big5 | sort - `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_can_match_shortcut.ppl index 84205ef61fc..844784cbde3 100644 --- a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_can_match_shortcut.ppl +++ b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_can_match_shortcut.ppl @@ -1,3 +1,21 @@ +/* +{ + "name": "desc_sort_timestamp_can_match_shortcut", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match": { + "process.name": "kernel" + } + }, + "sort" : [ + {"@timestamp" : "desc"} + ] + } +} +*/ source = big5 process.name=kernel | sort - `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl index 84205ef61fc..b8516925f18 100644 --- a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl +++ b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl @@ -1,3 +1,24 @@ +/* +{ + "name": "desc_sort_timestamp_no_can_match_shortcut", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-params" : { + "pre_filter_shard_size" : 100000 + }, + "body": { + "track_total_hits": false, + "query": { + "match": { + "process.name": "kernel" + } + }, + "sort" : [ + {"@timestamp" : "desc"} + ] + } +} +*/ source = big5 process.name=kernel | sort - `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/desc_sort_with_after_timestamp.ppl b/integ-test/src/test/resources/big5/queries/desc_sort_with_after_timestamp.ppl index af3445efdff..869514e9f8f 100644 --- a/integ-test/src/test/resources/big5/queries/desc_sort_with_after_timestamp.ppl +++ b/integ-test/src/test/resources/big5/queries/desc_sort_with_after_timestamp.ppl @@ -1,3 +1,24 @@ +/* +{ + "name": "desc_sort_with_after_timestamp", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match_all": {} + }, + "sort" : [ + {"@timestamp" : "desc"} + ], +{% if distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list or distribution_version.split('.') | map('int') | list >= "7.0.0".split('.') | map('int') | list %} + "search_after": ["2023-01-01T23:59:58.000Z"] +{% else %} + "search_after": [1673049598] +{% endif %} + } +} +*/ source = big5 | sort - `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/keyword_in_range.ppl b/integ-test/src/test/resources/big5/queries/keyword_in_range.ppl index 1c717e8472c..331a2212d9e 100644 --- a/integ-test/src/test/resources/big5/queries/keyword_in_range.ppl +++ b/integ-test/src/test/resources/big5/queries/keyword_in_range.ppl @@ -1,4 +1,31 @@ +/* +{ + "name": "keyword-in-range", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "bool": { + "must": [ + { + "range": { + "@timestamp": { + "gte": "2023-01-01T00:00:00", + "lt": "2023-01-03T00:00:00" + } + } + }, + { + "match": { + "process.name": "kernel" + } + } + ] + } + } + } +} +*/ source = big5 process.name=kernel -| where `@timestamp` >= '2023-01-01 00:00:00' - and `@timestamp` < '2023-01-03 00:00:00' +| where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/keyword_terms.ppl b/integ-test/src/test/resources/big5/queries/keyword_terms.ppl index 99353b5299f..1329aaf6570 100644 --- a/integ-test/src/test/resources/big5/queries/keyword_terms.ppl +++ b/integ-test/src/test/resources/big5/queries/keyword_terms.ppl @@ -1,3 +1,21 @@ +/* +{ + "name": "keyword-terms", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "station": { + "terms": { + "field": "aws.cloudwatch.log_stream", + "size": 500 + } + } + } + } +} +*/ source = big5 | stats count() as station by `aws.cloudwatch.log_stream` | sort - station diff --git a/integ-test/src/test/resources/big5/queries/keyword_terms_low_cardinality.ppl b/integ-test/src/test/resources/big5/queries/keyword_terms_low_cardinality.ppl index 02e335723d3..11adb833804 100644 --- a/integ-test/src/test/resources/big5/queries/keyword_terms_low_cardinality.ppl +++ b/integ-test/src/test/resources/big5/queries/keyword_terms_low_cardinality.ppl @@ -1,4 +1,22 @@ +/* +{ + "name": "keyword-terms-low-cardinality", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "country": { + "terms": { + "field": "aws.cloudwatch.log_stream", + "size": 50 + } + } + } + } +} +*/ source = big5 | stats count() as country by `aws.cloudwatch.log_stream` | sort - country -| head 100 \ No newline at end of file +| head 50 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/multi_terms_keyword.ppl b/integ-test/src/test/resources/big5/queries/multi_terms_keyword.ppl index a148a8bbc90..d88f2cf7ce3 100644 --- a/integ-test/src/test/resources/big5/queries/multi_terms_keyword.ppl +++ b/integ-test/src/test/resources/big5/queries/multi_terms_keyword.ppl @@ -1,4 +1,38 @@ +/* +{ + "name": "multi_terms-keyword", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 7200, + "body":{ + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-05T00:00:00", + "lt": "2023-01-05T05:00:00" + } + } + }, + "aggs": { + "important_terms": { + "multi_terms": { + "terms": [ + { + "field": "process.name" + }, + { + "field": "cloud.region" + } + ] + } + } + } + } +} +*/ source = big5 -| where `@timestamp` >= '2022-12-30 00:00:00' and `@timestamp` < '2023-01-01 03:00:00' -| stats count() by `process.name`, `event.id`, `cloud.region` -| sort - `count()` \ No newline at end of file +| where `@timestamp` >= '2023-01-05 00:00:00' and `@timestamp` < '2023-01-05 05:00:00' +| stats count() by `process.name`, `cloud.region` +| sort - `count()` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_high.ppl b/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_high.ppl new file mode 100644 index 00000000000..7d291cd766e --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_high.ppl @@ -0,0 +1,22 @@ +/* +{ + "name": "cardinality-agg-high", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "agent": { + "cardinality": { + "field": "agent.name" + {% if distribution_version.split('.') | map('int') | list >= "2.19.1".split('.') | map('int') | list and distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list %} + , "execution_hint": "ordinals" + {% endif %} + } + } + } + } +} +*/ +source = big5 +| stats bucket_nullable = false dc(`agent.name`) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_high_2.ppl b/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_high_2.ppl new file mode 100644 index 00000000000..7ff01915a8c --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_high_2.ppl @@ -0,0 +1,21 @@ +/* +{ + "name": "cardinality-agg-high-2", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 1800, + "body": { + "size": 0, + "aggs": { + "agent": { + "cardinality": { + "field": "event.id", + "execution_hint":"ordinals" + } + } + } + } +} +*/ +source = big5 +| stats bucket_nullable = false dc(`event.id`) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_low.ppl b/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_low.ppl new file mode 100644 index 00000000000..f763f61a77d --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/cardinality_agg_low.ppl @@ -0,0 +1,19 @@ +/* +{ + "name": "cardinality-agg-low", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "region": { + "cardinality": { + "field": "cloud.region" + } + } + } + } +} +*/ +source = big5 +| stats bucket_nullable = false dc(`cloud.region`) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/composite_terms.ppl b/integ-test/src/test/resources/big5/queries/optimized/composite_terms.ppl new file mode 100644 index 00000000000..97897e227de --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/composite_terms.ppl @@ -0,0 +1,32 @@ +/* +{ + "name": "composite-terms", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-02T00:00:00", + "lt": "2023-01-02T10:00:00" + } + } + }, + "aggs": { + "logs": { + "composite": { + "sources": [ + { "process_name": { "terms": { "field": "process.name", "order": "desc" }}}, + { "cloud_region": { "terms": { "field": "cloud.region", "order": "asc" }}} + ] + } + } + } + } +} +*/ +source = big5 +| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' +| stats bucket_nullable = false count() by `process.name`, `cloud.region` +| sort - `process.name`, + `cloud.region` \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/composite_terms_keyword.ppl b/integ-test/src/test/resources/big5/queries/optimized/composite_terms_keyword.ppl new file mode 100644 index 00000000000..04d12b4fb0e --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/composite_terms_keyword.ppl @@ -0,0 +1,33 @@ +/* +{ + "name": "composite_terms-keyword", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-02T00:00:00", + "lt": "2023-01-02T10:00:00" + } + } + }, + "aggs": { + "logs": { + "composite": { + "sources": [ + { "process_name": { "terms": { "field": "process.name", "order": "desc" }}}, + { "cloud_region": { "terms": { "field": "cloud.region", "order": "asc" }}}, + { "cloudstream": { "terms": { "field": "aws.cloudwatch.log_stream", "order": "asc" }}} + ] + } + } + } + } +} +*/ +source = big5 +| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' +| stats bucket_nullable = false count() by `process.name`, `cloud.region`, `aws.cloudwatch.log_stream` +| sort - `process.name`, + `cloud.region`, + `aws.cloudwatch.log_stream` \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/keyword_terms.ppl b/integ-test/src/test/resources/big5/queries/optimized/keyword_terms.ppl new file mode 100644 index 00000000000..062eea752bb --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/keyword_terms.ppl @@ -0,0 +1,22 @@ +/* +{ + "name": "keyword-terms", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "station": { + "terms": { + "field": "aws.cloudwatch.log_stream", + "size": 500 + } + } + } + } +} +*/ +source = big5 +| stats bucket_nullable = false count() as station by `aws.cloudwatch.log_stream` +| sort - station +| head 500 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/keyword_terms_low_cardinality.ppl b/integ-test/src/test/resources/big5/queries/optimized/keyword_terms_low_cardinality.ppl new file mode 100644 index 00000000000..71812820cab --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/keyword_terms_low_cardinality.ppl @@ -0,0 +1,22 @@ +/* +{ + "name": "keyword-terms-low-cardinality", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "country": { + "terms": { + "field": "aws.cloudwatch.log_stream", + "size": 50 + } + } + } + } +} +*/ +source = big5 +| stats bucket_nullable = false count() as country by `aws.cloudwatch.log_stream` +| sort - country +| head 50 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/multi_terms_keyword.ppl b/integ-test/src/test/resources/big5/queries/optimized/multi_terms_keyword.ppl new file mode 100644 index 00000000000..9221e728e09 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/multi_terms_keyword.ppl @@ -0,0 +1,38 @@ +/* +{ + "name": "multi_terms-keyword", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 7200, + "body":{ + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-05T00:00:00", + "lt": "2023-01-05T05:00:00" + } + } + }, + "aggs": { + "important_terms": { + "multi_terms": { + "terms": [ + { + "field": "process.name" + }, + { + "field": "cloud.region" + } + ] + } + } + } + } +} +*/ +source = big5 +| where `@timestamp` >= '2023-01-05 00:00:00' and `@timestamp` < '2023-01-05 05:00:00' +| stats bucket_nullable = false count() by `process.name`, `cloud.region` +| sort - `count()` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/range_auto_date_histo.ppl b/integ-test/src/test/resources/big5/queries/optimized/range_auto_date_histo.ppl new file mode 100644 index 00000000000..6711ce74d58 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/range_auto_date_histo.ppl @@ -0,0 +1,59 @@ +/* +{ + "name": "range-auto-date-histo", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "tmax": { + "range": { + "field": "metrics.size", + "ranges": [ + { + "to": -10 + }, + { + "from": -10, + "to": 10 + }, + { + "from": 10, + "to": 100 + }, + { + "from": 100, + "to": 1000 + }, + { + "from": 1000, + "to": 2000 + }, + { + "from": 2000 + } + ] + }, + "aggs": { + "date": { + "auto_date_histogram": { + "field": "@timestamp", + "buckets": 20 + } + } + } + } + } + } +} +*/ +source = big5 +| eval range_bucket = case( + `metrics.size` < -10, 'range_1', + `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', + `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', + `metrics.size` >= 2000, 'range_6') +| bin @timestamp bins=20 +| stats count() by range_bucket, @timestamp \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/range_auto_date_histo_with_metrics.ppl b/integ-test/src/test/resources/big5/queries/optimized/range_auto_date_histo_with_metrics.ppl new file mode 100644 index 00000000000..09e83a63ebe --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/optimized/range_auto_date_histo_with_metrics.ppl @@ -0,0 +1,67 @@ +/* +{ + "name": "range-auto-date-histo-with-metrics", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 7200, + "body": { + "size": 0, + "aggs": { + "tmax": { + "range": { + "field": "metrics.size", + "ranges": [ + { + "to": 100 + }, + { + "from": 100, + "to": 1000 + }, + { + "from": 1000, + "to": 2000 + }, + { + "from": 2000 + } + ] + }, + "aggs": { + "date": { + "auto_date_histogram": { + "field": "@timestamp", + "buckets": 10 + }, + "aggs": { + "tmin": { + "min": { + "field": "metrics.tmin" + } + }, + "tavg": { + "avg": { + "field": "metrics.size" + } + }, + "tmax": { + "max": { + "field": "metrics.size" + } + } + } + } + } + } + } + } +} +*/ +source = big5 +| eval range_bucket = case( + `metrics.size` < 100, 'range_1', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_2', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_3', + `metrics.size` >= 2000, 'range_4') +| bin @timestamp bins=10 +| stats min(`metrics.tmin`) as tmin, avg(`metrics.size`) as tavg, max(`metrics.size`) as tmax by range_bucket, @timestamp \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/query_string_on_message.ppl b/integ-test/src/test/resources/big5/queries/query_string_on_message.ppl index 2f0d33a6a3b..6a11bd84867 100644 --- a/integ-test/src/test/resources/big5/queries/query_string_on_message.ppl +++ b/integ-test/src/test/resources/big5/queries/query_string_on_message.ppl @@ -1,2 +1,16 @@ -source = big5 | where query_string(['message'], 'shield AND carp AND shark') +/* +{ + "name": "query-string-on-message", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "query_string": { + "query": "message: monkey jackal bear" + } + } + } +} +*/ +source = big5 message=monkey OR message=jackal OR message=bear | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered.ppl b/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered.ppl index 3abd5cea089..cb8be286a45 100644 --- a/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered.ppl +++ b/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered.ppl @@ -1,4 +1,32 @@ -source = big5 message=shield message=carp message=shark -| where `@timestamp` >= '2023-01-01 00:00:00' - and `@timestamp` < '2023-01-03 00:00:00' +/* +{ + "name": "query-string-on-message-filtered", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "bool": { + "must": [ + { + "range": { + "@timestamp": { + "gte": "2023-01-03T00:00:00", + "lt": "2023-01-03T10:00:00" + } + } + }, + { + "query_string": { + "query": "message: monkey jackal bear" + } + } + ] + } + } + } +} +*/ +source = big5 +| where `@timestamp` >= '2023-01-03 00:00:00' and `@timestamp` < '2023-01-03 10:00:00' + AND query_string(['message'], 'monkey jackal bear') | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered_sorted_num.ppl b/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered_sorted_num.ppl index 8baf49c987c..732c57c83d6 100644 --- a/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered_sorted_num.ppl +++ b/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered_sorted_num.ppl @@ -1,5 +1,40 @@ -source = big5 | where query_string(['message'], 'shield AND carp AND shark') -| where `@timestamp` >= '2023-01-01 00:00:00' - and `@timestamp` < '2023-01-03 00:00:00' -| sort - `metrics.size` +/* +{ + "name": "query-string-on-message-filtered-sorted-num", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "bool": { + "must": [ + { + "range": { + "@timestamp": { + "gte": "2023-01-03T00:00:00", + "lt": "2023-01-03T10:00:00" + } + } + }, + { + "query_string": { + "query": "message: monkey jackal bear" + } + } + ] + } + }, + "sort": [ + { + "@timestamp": { + "order": "asc" + } + } + ] + } +} +*/ +source = big5 +| where `@timestamp` >= '2023-01-03 00:00:00' and `@timestamp` < '2023-01-03 10:00:00' + AND query_string(['message'], 'monkey jackal bear') +| sort `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range.ppl b/integ-test/src/test/resources/big5/queries/range.ppl index 74eae492541..b5480c6ba51 100644 --- a/integ-test/src/test/resources/big5/queries/range.ppl +++ b/integ-test/src/test/resources/big5/queries/range.ppl @@ -1,3 +1,20 @@ +/* +{ + "name": "range", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-01T00:00:00", + "lt": "2023-01-03T00:00:00" + } + } + } + } +} +*/ source = big5 | where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_agg_1.ppl b/integ-test/src/test/resources/big5/queries/range_agg_1.ppl new file mode 100644 index 00000000000..95280e697bb --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/range_agg_1.ppl @@ -0,0 +1,50 @@ +/* +{ + "name": "range-agg-1", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "tmax": { + "range": { + "field": "metrics.size", + "ranges": [ + { + "to": -10 + }, + { + "from": -10, + "to": 10 + }, + { + "from": 10, + "to": 100 + }, + { + "from": 100, + "to": 1000 + }, + { + "from": 1000, + "to": 2000 + }, + { + "from": 2000 + } + ] + } + } + } + } +} +*/ +source = big5 +| eval range_bucket = case( + `metrics.size` < -10, 'range_1', + `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', + `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', + `metrics.size` >= 2000, 'range_6') +| stats count() by range_bucket \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_agg_2.ppl b/integ-test/src/test/resources/big5/queries/range_agg_2.ppl new file mode 100644 index 00000000000..0988d688054 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/range_agg_2.ppl @@ -0,0 +1,40 @@ +/* +{ + "name": "range-agg-2", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "tmax": { + "range": { + "field": "metrics.size", + "ranges": [ + { + "to": 100 + }, + { + "from": 100, + "to": 1000 + }, + { + "from": 1000, + "to": 2000 + }, + { + "from": 2000 + } + ] + } + } + } + } +} +*/ +source = big5 +| eval range_bucket = case( + `metrics.size` < 100, 'range_1', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_2', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_3', + `metrics.size` >= 2000, 'range_4') +| stats count() by range_bucket \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl b/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl index 52a51cf7419..6126623a8b1 100644 --- a/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl +++ b/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl @@ -1,3 +1,52 @@ +/* +{ + "name": "range-auto-date-histo", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "size": 0, + "aggs": { + "tmax": { + "range": { + "field": "metrics.size", + "ranges": [ + { + "to": -10 + }, + { + "from": -10, + "to": 10 + }, + { + "from": 10, + "to": 100 + }, + { + "from": 100, + "to": 1000 + }, + { + "from": 1000, + "to": 2000 + }, + { + "from": 2000 + } + ] + }, + "aggs": { + "date": { + "auto_date_histogram": { + "field": "@timestamp", + "buckets": 20 + } + } + } + } + } + } +} +*/ source = big5 | eval range_bucket = case( `metrics.size` < -10, 'range_1', diff --git a/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl b/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl index 506978ace5b..d08c67823fa 100644 --- a/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl +++ b/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl @@ -1,10 +1,67 @@ +/* +{ + "name": "range-auto-date-histo-with-metrics", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 7200, + "body": { + "size": 0, + "aggs": { + "tmax": { + "range": { + "field": "metrics.size", + "ranges": [ + { + "to": 100 + }, + { + "from": 100, + "to": 1000 + }, + { + "from": 1000, + "to": 2000 + }, + { + "from": 2000 + } + ] + }, + "aggs": { + "date": { + "auto_date_histogram": { + "field": "@timestamp", + "buckets": 10 + }, + "aggs": { + "tmin": { + "min": { + "field": "metrics.tmin" + } + }, + "tavg": { + "avg": { + "field": "metrics.size" + } + }, + "tmax": { + "max": { + "field": "metrics.size" + } + } + } + } + } + } + } + } +} +*/ source = big5 | eval range_bucket = case( - `metrics.size` < -10, 'range_1', - `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', - `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', - `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', - `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', - `metrics.size` >= 2000, 'range_6') + `metrics.size` < 100, 'range_1', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_2', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_3', + `metrics.size` >= 2000, 'range_4') | stats min(`metrics.tmin`) as tmin, avg(`metrics.size`) as tavg, max(`metrics.size`) as tmax by range_bucket, span(`@timestamp`, 1h) as auto_span | sort + range_bucket, + auto_span \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_field_conjunction_big_range_big_term_query.ppl b/integ-test/src/test/resources/big5/queries/range_field_conjunction_big_range_big_term_query.ppl index e6390a38cbf..905ad7f3dcc 100644 --- a/integ-test/src/test/resources/big5/queries/range_field_conjunction_big_range_big_term_query.ppl +++ b/integ-test/src/test/resources/big5/queries/range_field_conjunction_big_range_big_term_query.ppl @@ -1,5 +1,31 @@ +/* +{ + "name": "range_field_conjunction_big_range_big_term_query", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "bool": { + "must": [ + { + "term": { + "process.name": "systemd" + } + }, + { + "range": { + "metrics.size": { + "gte": 1, + "lte": 100 + } + } + } + ] + } + } + } +} +*/ source = big5 -| where `process.name` = 'systemd' - and `metrics.size` >= 1 - and `metrics.size` <= 1000 +| where `process.name` = 'systemd' and `metrics.size` >= 1 and `metrics.size` <= 100 | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_big_term_query.ppl b/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_big_term_query.ppl index f762da83896..1451f9a382f 100644 --- a/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_big_term_query.ppl +++ b/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_big_term_query.ppl @@ -1,3 +1,26 @@ +/* +{ + "name": "range_field_conjunction_small_range_big_term_query", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "bool": { + "must": [ + { + "range": { + "metrics.size": { + "gte": 20, + "lte": 30 + } + } + } + ] + } + } + } +} +*/ source = big5 -| where `metrics.size` >= 1 and `metrics.size` <= 42 +| where `metrics.size` >= 20 and `metrics.size` <= 30 | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_small_term_query.ppl b/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_small_term_query.ppl index 9d0742e122f..a6af46b8d08 100644 --- a/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_small_term_query.ppl +++ b/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_small_term_query.ppl @@ -1,4 +1,31 @@ +/* +{ + "name": "range_field_conjunction_small_range_small_term_query", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "bool": { + "should": [ + { + "term": { + "aws.cloudwatch.log_stream": "indigodagger" + } + }, + { + "range": { + "metrics.size": { + "gte": 10, + "lte": 20 + } + } + } + ] + } + } + } +} +*/ source = big5 -| where `aws.cloudwatch.log_stream` = 'indigodagger' - or (`metrics.size` >= 1 and `metrics.size` <= 30) +| where `aws.cloudwatch.log_stream` = 'indigodagger' or (`metrics.size` >= 10 and `metrics.size` <= 20) | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_field_disjunction_big_range_small_term_query.ppl b/integ-test/src/test/resources/big5/queries/range_field_disjunction_big_range_small_term_query.ppl index 4ea1dcfc518..59ac3769159 100644 --- a/integ-test/src/test/resources/big5/queries/range_field_disjunction_big_range_small_term_query.ppl +++ b/integ-test/src/test/resources/big5/queries/range_field_disjunction_big_range_small_term_query.ppl @@ -1,4 +1,31 @@ +/* +{ + "name": "range_field_disjunction_big_range_small_term_query", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "bool": { + "should": [ + { + "term": { + "aws.cloudwatch.log_stream": "indigodagger" + } + }, + { + "range": { + "metrics.size": { + "gte": 1, + "lte": 100 + } + } + } + ] + } + } + } +} +*/ source = big5 -| where `aws.cloudwatch.log_stream` = 'indigodagger' - or (`metrics.size` >= 1 and `metrics.size` <= 1000) +| where `aws.cloudwatch.log_stream` = 'indigodagger' or (`metrics.size` >= 1 and `metrics.size` <= 100) | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_numeric.ppl b/integ-test/src/test/resources/big5/queries/range_numeric.ppl index 5b5b50b7c35..dfdabbf0877 100644 --- a/integ-test/src/test/resources/big5/queries/range_numeric.ppl +++ b/integ-test/src/test/resources/big5/queries/range_numeric.ppl @@ -1,3 +1,20 @@ +/* +{ + "name": "range-numeric", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "query": { + "range": { + "metrics.size": { + "gte": 20, + "lte": 200 + } + } + } + } +} +*/ source = big5 -| where `metrics.size` >= 1 and `metrics.size` <= 1000 +| where `metrics.size` >= 20 and `metrics.size` <= 200 | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_with_asc_sort.ppl b/integ-test/src/test/resources/big5/queries/range_with_asc_sort.ppl index a3325df54ed..f9ef584a340 100644 --- a/integ-test/src/test/resources/big5/queries/range_with_asc_sort.ppl +++ b/integ-test/src/test/resources/big5/queries/range_with_asc_sort.ppl @@ -1,5 +1,24 @@ +/* +{ + "name": "range_with_asc_sort", + "operation-type": "search", + "index": "{{ index_name | default('big5') }}", + "body": { + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-01T00:00:00", + "lte": "2023-01-13T00:00:00" + } + } + }, + "sort": [ + { "@timestamp": "asc" } + ] + } +} +*/ source = big5 -| where `@timestamp` >= '2023-01-01 00:00:00' - and `@timestamp` <= '2023-01-13 00:00:00' +| where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` <= '2023-01-13 00:00:00' | sort + `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_with_desc_sort.ppl b/integ-test/src/test/resources/big5/queries/range_with_desc_sort.ppl index ba3a042d511..e98fc75acc0 100644 --- a/integ-test/src/test/resources/big5/queries/range_with_desc_sort.ppl +++ b/integ-test/src/test/resources/big5/queries/range_with_desc_sort.ppl @@ -1,5 +1,24 @@ +/* +{ + "name": "range_with_desc_sort", + "operation-type": "search", + "index": "{{ index_name | default('big5') }}", + "body": { + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-01T00:00:00", + "lte": "2023-01-13T00:00:00" + } + } + }, + "sort": [ + { "@timestamp": "desc" } + ] + } +} +*/ source = big5 -| where `@timestamp` >= '2023-01-01 00:00:00' - and `@timestamp` <= '2023-01-13 00:00:00' +| where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` <= '2023-01-13 00:00:00' | sort - `@timestamp` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/scroll.ppl b/integ-test/src/test/resources/big5/queries/scroll.ppl index 6b63c414ac0..9bb35aaf830 100644 --- a/integ-test/src/test/resources/big5/queries/scroll.ppl +++ b/integ-test/src/test/resources/big5/queries/scroll.ppl @@ -1,2 +1,17 @@ +/* +{ + "name": "scroll", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "pages": 25, + "results-per-page": 1000, + "body": { + "query": { + "match_all": {} + } + } +} +*/ +/* scroll is unsupported in PPL */ source = big5 | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_keyword_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/sort_keyword_can_match_shortcut.ppl index aab85fb7c1b..c2f2d89be18 100644 --- a/integ-test/src/test/resources/big5/queries/sort_keyword_can_match_shortcut.ppl +++ b/integ-test/src/test/resources/big5/queries/sort_keyword_can_match_shortcut.ppl @@ -1,3 +1,21 @@ +/* +{ + "name": "sort_keyword_can_match_shortcut", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match": { + "process.name": "kernel" + } + }, + "sort" : [ + {"meta.file" : "asc"} + ] + } +} +*/ source = big5 process.name=kernel -| sort + `@timestamp` +| sort + `meta.file` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_keyword_no_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/sort_keyword_no_can_match_shortcut.ppl index aab85fb7c1b..de375ef4517 100644 --- a/integ-test/src/test/resources/big5/queries/sort_keyword_no_can_match_shortcut.ppl +++ b/integ-test/src/test/resources/big5/queries/sort_keyword_no_can_match_shortcut.ppl @@ -1,3 +1,24 @@ +/* +{ + "name": "sort_keyword_no_can_match_shortcut", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-params" : { + "pre_filter_shard_size" : 100000 + }, + "body": { + "track_total_hits": false, + "query": { + "match": { + "process.name": "kernel" + } + }, + "sort" : [ + {"meta.file" : "asc"} + ] + } +} +*/ source = big5 process.name=kernel -| sort + `@timestamp` +| sort + `meta.file` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_numeric_asc.ppl b/integ-test/src/test/resources/big5/queries/sort_numeric_asc.ppl index eb96d2b4bab..6340dfa854e 100644 --- a/integ-test/src/test/resources/big5/queries/sort_numeric_asc.ppl +++ b/integ-test/src/test/resources/big5/queries/sort_numeric_asc.ppl @@ -1,3 +1,21 @@ +/* +{ + "name": "sort_numeric_asc", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match_all": {} + }, + "sort": [ + { + "metrics.size": "asc" + } + ] + } +} +*/ source = big5 | sort + `metrics.size` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_numeric_asc_with_match.ppl b/integ-test/src/test/resources/big5/queries/sort_numeric_asc_with_match.ppl index 198667db866..417bc4dcf46 100644 --- a/integ-test/src/test/resources/big5/queries/sort_numeric_asc_with_match.ppl +++ b/integ-test/src/test/resources/big5/queries/sort_numeric_asc_with_match.ppl @@ -1,3 +1,23 @@ +/* +{ + "name": "sort_numeric_asc_with_match", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match": { + "log.file.path": "/var/log/messages/solarshark" + } + }, + "sort": [ + { + "metrics.size": "asc" + } + ] + } +} +*/ source = big5 log.file.path=\"/var/log/messages/solarshark\" | sort + `metrics.size` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_numeric_desc.ppl b/integ-test/src/test/resources/big5/queries/sort_numeric_desc.ppl index f4a4165fbfc..b55d15a135e 100644 --- a/integ-test/src/test/resources/big5/queries/sort_numeric_desc.ppl +++ b/integ-test/src/test/resources/big5/queries/sort_numeric_desc.ppl @@ -1,3 +1,21 @@ +/* +{ + "name": "sort_numeric_desc", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match_all": {} + }, + "sort": [ + { + "metrics.size": "desc" + } + ] + } +} +*/ source = big5 | sort - `metrics.size` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_numeric_desc_with_match.ppl b/integ-test/src/test/resources/big5/queries/sort_numeric_desc_with_match.ppl index f282e9ae67d..53ece3b052b 100644 --- a/integ-test/src/test/resources/big5/queries/sort_numeric_desc_with_match.ppl +++ b/integ-test/src/test/resources/big5/queries/sort_numeric_desc_with_match.ppl @@ -1,3 +1,23 @@ +/* +{ + "name": "sort_numeric_desc_with_match", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "query": { + "match": { + "log.file.path": "/var/log/messages/solarshark" + } + }, + "sort": [ + { + "metrics.size": "desc" + } + ] + } +} +*/ source = big5 log.file.path=\"/var/log/messages/solarshark\" | sort - `metrics.size` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/term.ppl b/integ-test/src/test/resources/big5/queries/term.ppl index 2cbfae69eba..20799833371 100644 --- a/integ-test/src/test/resources/big5/queries/term.ppl +++ b/integ-test/src/test/resources/big5/queries/term.ppl @@ -1,3 +1,20 @@ +/* +{ + "name": "term", + "operation-type": "search", + "index": "{{index_name | default('big5')}}", + "request-timeout": 7200, + "body": { + "query": { + "term": { + "log.file.path": { + "value": "/var/log/messages/birdknight" + } + } + } + } +} +*/ source = big5 | where `log.file.path` = '/var/log/messages/birdknight' | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/terms_significant_1.ppl b/integ-test/src/test/resources/big5/queries/terms_significant_1.ppl index b33048f82a1..452e0f23ae5 100644 --- a/integ-test/src/test/resources/big5/queries/terms_significant_1.ppl +++ b/integ-test/src/test/resources/big5/queries/terms_significant_1.ppl @@ -1,5 +1,41 @@ +/* +{ + "name": "terms-significant-1", + "operation-type": "search", + "request-timeout": 7200, + "index": "{{index_name | default('big5')}}", + "body": + { + "track_total_hits": false, + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-01T00:00:00", + "lt": "2023-01-03T00:00:00" + } + } + }, + "aggs": { + "terms": { + "terms": { + "field": "aws.cloudwatch.log_stream", + "size": 10 + }, + "aggs": { + "significant_ips": { + "significant_terms": { + "field": "process.name" + } + } + } + } + } + } +} +*/ +/* significant_terms is unsupported in PPL */ source = big5 -| where `@timestamp` >= '2023-01-01 00:00:00' - and `@timestamp` < '2023-01-03 00:00:00' -| stats count() by `aws.cloudwatch.log_stream` +| where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' +| stats count() by `aws.cloudwatch.log_stream`, `process.name` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/terms_significant_2.ppl b/integ-test/src/test/resources/big5/queries/terms_significant_2.ppl index 994914e3bbe..954cf194e4d 100644 --- a/integ-test/src/test/resources/big5/queries/terms_significant_2.ppl +++ b/integ-test/src/test/resources/big5/queries/terms_significant_2.ppl @@ -1,5 +1,40 @@ +/* +{ + "name": "terms-significant-2", + "operation-type": "search", + "request-timeout": 7200, + "index": "{{index_name | default('big5')}}", + "body": { + "track_total_hits": false, + "size": 0, + "query": { + "range": { + "@timestamp": { + "gte": "2023-01-01T00:00:00", + "lt": "2023-01-03T00:00:00" + } + } + }, + "aggs": { + "terms": { + "terms": { + "field": "process.name", + "size": 10 + }, + "aggs": { + "significant_ips": { + "significant_terms": { + "field": "aws.cloudwatch.log_stream" + } + } + } + } + } + } +} +*/ +/* significant_terms is unsupported in PPL */ source = big5 -| where `@timestamp` >= '2023-01-01 00:00:00' - and `@timestamp` < '2023-01-03 00:00:00' -| stats count() by `process.name` +| where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' +| stats count() by `process.name`, `aws.cloudwatch.log_stream` | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp.yaml new file mode 100644 index 00000000000..81138f6fe80 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], SORT->[{ + "@timestamp" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp_can_match_shortcut.yaml new file mode 100644 index 00000000000..ce84d53f479 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp_can_match_shortcut.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + "@timestamp" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp_no_can_match_shortcut.yaml new file mode 100644 index 00000000000..ce84d53f479 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_timestamp_no_can_match_shortcut.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + "@timestamp" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_with_after_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_with_after_timestamp.yaml new file mode 100644 index 00000000000..81138f6fe80 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/asc_sort_with_after_timestamp.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], SORT->[{ + "@timestamp" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_high.yaml b/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_high.yaml new file mode 100644 index 00000000000..bd3889e2926 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_high.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], dc(`agent.name`)=[COUNT(DISTINCT $0)]) + LogicalProject(agent.name=[$3]) + LogicalFilter(condition=[IS NOT NULL($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->IS NOT NULL($3), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},dc(`agent.name`)=COUNT(DISTINCT $0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"agent.name","boost":1.0}},"aggregations":{"dc(`agent.name`)":{"cardinality":{"field":"agent.name"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_high_2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_high_2.yaml new file mode 100644 index 00000000000..6d5c2b7448e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_high_2.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], dc(`event.id`)=[COUNT(DISTINCT $0)]) + LogicalProject(event.id=[$37]) + LogicalFilter(condition=[IS NOT NULL($37)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->IS NOT NULL($37), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},dc(`event.id`)=COUNT(DISTINCT $0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"event.id","boost":1.0}},"aggregations":{"dc(`event.id`)":{"cardinality":{"field":"event.id"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_low.yaml b/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_low.yaml new file mode 100644 index 00000000000..dec25a78628 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/cardinality_agg_low.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], dc(`cloud.region`)=[COUNT(DISTINCT $0)]) + LogicalProject(cloud.region=[$14]) + LogicalFilter(condition=[IS NOT NULL($14)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[FILTER->IS NOT NULL($14), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},dc(`cloud.region`)=COUNT(DISTINCT $0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"cloud.region","boost":1.0}},"aggregations":{"dc(`cloud.region`)":{"cardinality":{"field":"cloud.region"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/composite_date_histogram_daily.yaml b/integ-test/src/test/resources/expectedOutput/calcite/composite_date_histogram_daily.yaml new file mode 100644 index 00000000000..9b69c67b74c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/composite_date_histogram_daily.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(`@timestamp`,1d)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(`@timestamp`,1d)=[SPAN($17, 1, 'd')]) + LogicalFilter(condition=[IS NOT NULL($17)]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2022-12-30 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-07 12:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[@timestamp], FILTER->SEARCH($0, Sarg[['2022-12-30 00:00:00':VARCHAR..'2023-01-07 12:00:00':VARCHAR); NULL AS FALSE]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2022-12-30T00:00:00.000Z","to":"2023-01-07T12:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"@timestamp","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1d)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/composite_terms.yaml b/integ-test/src/test/resources/expectedOutput/calcite/composite_terms.yaml new file mode 100644 index 00000000000..8720f023f80 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/composite_terms.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], sort1=[$2], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], sort1=[$2], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first]) + LogicalProject(count()=[$2], process.name=[$0], cloud.region=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) + LogicalProject(process.name=[$7], cloud.region=[$14]) + LogicalFilter(condition=[AND(IS NOT NULL($7), IS NOT NULL($14))]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-02 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-02 10:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, cloud.region, @timestamp], FILTER->SEARCH($2, Sarg[['2023-01-02 00:00:00':VARCHAR..'2023-01-02 10:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), process.name, cloud.region], SORT->[1 DESC LAST, 2 ASC FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","cloud.region","@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/composite_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/calcite/composite_terms_keyword.yaml new file mode 100644 index 00000000000..ac251d900f0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/composite_terms_keyword.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], sort1=[$2], sort2=[$3], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], sort1=[$2], sort2=[$3], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first]) + LogicalProject(count()=[$3], process.name=[$0], cloud.region=[$1], aws.cloudwatch.log_stream=[$2]) + LogicalAggregate(group=[{0, 1, 2}], count()=[COUNT()]) + LogicalProject(process.name=[$7], cloud.region=[$14], aws.cloudwatch.log_stream=[$34]) + LogicalFilter(condition=[AND(IS NOT NULL($7), IS NOT NULL($14), IS NOT NULL($34))]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-02 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-02 10:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, cloud.region, @timestamp, aws.cloudwatch.log_stream], FILTER->SEARCH($2, Sarg[['2023-01-02 00:00:00':VARCHAR..'2023-01-02 10:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},count()=COUNT()), PROJECT->[count(), process.name, cloud.region, aws.cloudwatch.log_stream], SORT->[1 DESC LAST, 2 ASC FIRST, 3 ASC FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","cloud.region","@timestamp","aws.cloudwatch.log_stream"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/date_histogram_hourly_agg.yaml b/integ-test/src/test/resources/expectedOutput/calcite/date_histogram_hourly_agg.yaml new file mode 100644 index 00000000000..06361ea27e8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/date_histogram_hourly_agg.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(`@timestamp`,1h)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(`@timestamp`,1h)=[SPAN($17, 1, 'h')]) + LogicalFilter(condition=[IS NOT NULL($17)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[@timestamp], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1h)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"@timestamp","boost":1.0}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1h)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1h"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/date_histogram_minute_agg.yaml b/integ-test/src/test/resources/expectedOutput/calcite/date_histogram_minute_agg.yaml new file mode 100644 index 00000000000..c715c2c2a42 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/date_histogram_minute_agg.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(`@timestamp`,1m)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(`@timestamp`,1m)=[SPAN($17, 1, 'm')]) + LogicalFilter(condition=[IS NOT NULL($17)]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[@timestamp], FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR); NULL AS FALSE]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"@timestamp","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1m)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/default.yaml b/integ-test/src/test/resources/expectedOutput/calcite/default.yaml new file mode 100644 index 00000000000..59e68e48769 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/default.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp.yaml new file mode 100644 index 00000000000..7e14abeeef2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], SORT->[{ + "@timestamp" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp_can_match_shortcut.yaml new file mode 100644 index 00000000000..13239b869cc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp_can_match_shortcut.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + "@timestamp" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp_no_can_match_shortcut.yaml new file mode 100644 index 00000000000..13239b869cc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_timestamp_no_can_match_shortcut.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + "@timestamp" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_with_after_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_with_after_timestamp.yaml new file mode 100644 index 00000000000..7e14abeeef2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/desc_sort_with_after_timestamp.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], SORT->[{ + "@timestamp" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/keyword_in_range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/keyword_in_range.yaml new file mode 100644 index 00000000000..85c08cf100c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/keyword_in_range.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->AND(query_string(MAP('query', 'process.name:kernel':VARCHAR)), SEARCH($7, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR)), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/keyword_terms.yaml b/integ-test/src/test/resources/expectedOutput/calcite/keyword_terms.yaml new file mode 100644 index 00000000000..da777dc2784 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/keyword_terms.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[500]) + LogicalProject(station=[$1], aws.cloudwatch.log_stream=[$0]) + LogicalAggregate(group=[{0}], station=[COUNT()]) + LogicalProject(aws.cloudwatch.log_stream=[$34]) + LogicalFilter(condition=[IS NOT NULL($34)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},station=COUNT()), PROJECT->[station, aws.cloudwatch.log_stream], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->500, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","size":500,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"station":"desc"},{"_key":"asc"}]},"aggregations":{"station":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/keyword_terms_low_cardinality.yaml b/integ-test/src/test/resources/expectedOutput/calcite/keyword_terms_low_cardinality.yaml new file mode 100644 index 00000000000..fd4f1b547e3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/keyword_terms_low_cardinality.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[50]) + LogicalProject(country=[$1], aws.cloudwatch.log_stream=[$0]) + LogicalAggregate(group=[{0}], country=[COUNT()]) + LogicalProject(aws.cloudwatch.log_stream=[$34]) + LogicalFilter(condition=[IS NOT NULL($34)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},country=COUNT()), PROJECT->[country, aws.cloudwatch.log_stream], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->50, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","size":50,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"country":"desc"},{"_key":"asc"}]},"aggregations":{"country":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/multi_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/calcite/multi_terms_keyword.yaml new file mode 100644 index 00000000000..511664f319f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/multi_terms_keyword.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) + LogicalProject(count()=[$2], process.name=[$0], cloud.region=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) + LogicalProject(process.name=[$7], cloud.region=[$14]) + LogicalFilter(condition=[AND(IS NOT NULL($7), IS NOT NULL($14))]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-05 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-05 05:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, cloud.region, @timestamp], FILTER->SEARCH($2, Sarg[['2023-01-05 00:00:00':VARCHAR..'2023-01-05 05:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), SORT_AGG_METRICS->[2 DESC LAST], PROJECT->[count(), process.name, cloud.region], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-05T00:00:00.000Z","to":"2023-01-05T05:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","cloud.region","@timestamp"],"excludes":[]},"aggregations":{"multi_terms_buckets":{"multi_terms":{"terms":[{"field":"process.name"},{"field":"cloud.region"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message.yaml b/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message.yaml new file mode 100644 index 00000000000..31cbb3b8d70 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', '((message:monkey OR message:jackal) OR message:bear)':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->query_string(MAP('query', '((message:monkey OR message:jackal) OR message:bear)':VARCHAR)), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"((message:monkey OR message:jackal) OR message:bear)","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message_filtered.yaml b/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message_filtered.yaml new file mode 100644 index 00000000000..e1471d87a4e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message_filtered.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 10:00:00':VARCHAR)), query_string(MAP('fields', MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query', 'monkey jackal bear':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->AND(SEARCH($7, Sarg[['2023-01-03 00:00:00':VARCHAR..'2023-01-03 10:00:00':VARCHAR)]:VARCHAR), query_string(MAP('fields', MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query', 'monkey jackal bear':VARCHAR))), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-03T00:00:00.000Z","to":"2023-01-03T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"query_string":{"query":"monkey jackal bear","fields":["message^1.0"],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message_filtered_sorted_num.yaml b/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message_filtered_sorted_num.yaml new file mode 100644 index 00000000000..27a43886bc4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/query_string_on_message_filtered_sorted_num.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 10:00:00':VARCHAR)), query_string(MAP('fields', MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query', 'monkey jackal bear':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->AND(SEARCH($7, Sarg[['2023-01-03 00:00:00':VARCHAR..'2023-01-03 10:00:00':VARCHAR)]:VARCHAR), query_string(MAP('fields', MAP('message':VARCHAR, 1.0E0:DOUBLE)), MAP('query', 'monkey jackal bear':VARCHAR))), SORT->[{ + "@timestamp" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-03T00:00:00.000Z","to":"2023-01-03T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"query_string":{"query":"monkey jackal bear","fields":["message^1.0"],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range.yaml new file mode 100644 index 00000000000..56c63c5c406 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->SEARCH($7, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_agg_1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_agg_1.yaml new file mode 100644 index 00000000000..86c1551609c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_agg_1.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], range_bucket=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(range_bucket=[CASE(<($28, -10), 'range_1':VARCHAR, SEARCH($28, Sarg[[-10..10)]), 'range_2':VARCHAR, SEARCH($28, Sarg[[10..100)]), 'range_3':VARCHAR, SEARCH($28, Sarg[[100..1000)]), 'range_4':VARCHAR, SEARCH($28, Sarg[[1000..2000)]), 'range_5':VARCHAR, >=($28, 2000), 'range_6':VARCHAR, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), range_bucket]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"range_bucket":{"range":{"field":"metrics.size","ranges":[{"key":"range_1","to":-10.0},{"key":"range_2","from":-10.0,"to":10.0},{"key":"range_3","from":10.0,"to":100.0},{"key":"range_4","from":100.0,"to":1000.0},{"key":"range_5","from":1000.0,"to":2000.0},{"key":"range_6","from":2000.0}],"keyed":true}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_agg_2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_agg_2.yaml new file mode 100644 index 00000000000..daae2d2fc97 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_agg_2.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], range_bucket=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(range_bucket=[CASE(<($28, 100), 'range_1':VARCHAR, SEARCH($28, Sarg[[100..1000)]), 'range_2':VARCHAR, SEARCH($28, Sarg[[1000..2000)]), 'range_3':VARCHAR, >=($28, 2000), 'range_4':VARCHAR, null:NULL)]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), range_bucket]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"range_bucket":{"range":{"field":"metrics.size","ranges":[{"key":"range_1","to":100.0},{"key":"range_2","from":100.0,"to":1000.0},{"key":"range_3","from":1000.0,"to":2000.0},{"key":"range_4","from":2000.0}],"keyed":true}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_auto_date_histo.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_auto_date_histo.yaml new file mode 100644 index 00000000000..21c20b5a523 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_auto_date_histo.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$2], range_bucket=[$0], @timestamp=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) + LogicalProject(range_bucket=[CASE(<($28, -10), 'range_1':VARCHAR, SEARCH($28, Sarg[[-10..10)]), 'range_2':VARCHAR, SEARCH($28, Sarg[[10..100)]), 'range_3':VARCHAR, SEARCH($28, Sarg[[100..1000)]), 'range_4':VARCHAR, SEARCH($28, Sarg[[1000..2000)]), 'range_5':VARCHAR, >=($28, 2000), 'range_6':VARCHAR, null:NULL)], @timestamp=[WIDTH_BUCKET($17, 20, -(MAX($17) OVER (), MIN($17) OVER ()), MAX($17) OVER ())]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), range_bucket, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"range_bucket":{"range":{"field":"metrics.size","ranges":[{"key":"range_1","to":-10.0},{"key":"range_2","from":-10.0,"to":10.0},{"key":"range_3","from":10.0,"to":100.0},{"key":"range_4","from":100.0,"to":1000.0},{"key":"range_5","from":1000.0,"to":2000.0},{"key":"range_6","from":2000.0}],"keyed":true},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":20,"minimum_interval":null}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_auto_date_histo_with_metrics.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_auto_date_histo_with_metrics.yaml new file mode 100644 index 00000000000..b29b215e612 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_auto_date_histo_with_metrics.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(tmin=[$2], tavg=[$3], tmax=[$4], range_bucket=[$0], @timestamp=[$1]) + LogicalAggregate(group=[{0, 1}], tmin=[MIN($2)], tavg=[AVG($3)], tmax=[MAX($3)]) + LogicalProject(range_bucket=[CASE(<($28, 100), 'range_1':VARCHAR, SEARCH($28, Sarg[[100..1000)]), 'range_2':VARCHAR, SEARCH($28, Sarg[[1000..2000)]), 'range_3':VARCHAR, >=($28, 2000), 'range_4':VARCHAR, null:NULL)], @timestamp=[WIDTH_BUCKET($17, 10, -(MAX($17) OVER (), MIN($17) OVER ()), MAX($17) OVER ())], metrics.tmin=[$29], metrics.size=[$28]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},tmin=MIN($2),tavg=AVG($3),tmax=MAX($3)), PROJECT->[tmin, tavg, tmax, range_bucket, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"range_bucket":{"range":{"field":"metrics.size","ranges":[{"key":"range_1","to":100.0},{"key":"range_2","from":100.0,"to":1000.0},{"key":"range_3","from":1000.0,"to":2000.0},{"key":"range_4","from":2000.0}],"keyed":true},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":10,"minimum_interval":null},"aggregations":{"tmin":{"min":{"field":"metrics.tmin"}},"tavg":{"avg":{"field":"metrics.size"}},"tmax":{"max":{"field":"metrics.size"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_big_range_big_term_query.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_big_range_big_term_query.yaml new file mode 100644 index 00000000000..ba8b035ab51 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_big_range_big_term_query.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[AND(=($7, 'systemd'), SEARCH($28, Sarg[[1..100]]))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, process.name, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->AND(=($2, 'systemd'), SEARCH($14, Sarg[[1..100]])), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"must":[{"term":{"process.name":{"value":"systemd","boost":1.0}}},{"range":{"metrics.size":{"from":1.0,"to":100.0,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_small_range_big_term_query.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_small_range_big_term_query.yaml new file mode 100644 index 00000000000..69dddd2ef14 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_small_range_big_term_query.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[SEARCH($28, Sarg[[20..30]])]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->SEARCH($13, Sarg[[20..30]]), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"metrics.size":{"from":20.0,"to":30.0,"include_lower":true,"include_upper":true,"boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_small_range_small_term_query.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_small_range_small_term_query.yaml new file mode 100644 index 00000000000..612e412b307 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_field_conjunction_small_range_small_term_query.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[OR(=($34, 'indigodagger'), SEARCH($28, Sarg[[10..20]]))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, aws.cloudwatch.log_stream, event], FILTER->OR(=($15, 'indigodagger'), SEARCH($13, Sarg[[10..20]])), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"should":[{"term":{"aws.cloudwatch.log_stream":{"value":"indigodagger","boost":1.0}}},{"range":{"metrics.size":{"from":10.0,"to":20.0,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_field_disjunction_big_range_small_term_query.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_field_disjunction_big_range_small_term_query.yaml new file mode 100644 index 00000000000..24cabf88754 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_field_disjunction_big_range_small_term_query.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[OR(=($34, 'indigodagger'), SEARCH($28, Sarg[[1..100]]))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, aws.cloudwatch.log_stream, event], FILTER->OR(=($15, 'indigodagger'), SEARCH($13, Sarg[[1..100]])), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"bool":{"should":[{"term":{"aws.cloudwatch.log_stream":{"value":"indigodagger","boost":1.0}}},{"range":{"metrics.size":{"from":1.0,"to":100.0,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_numeric.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_numeric.yaml new file mode 100644 index 00000000000..cdf19c603a0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_numeric.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[SEARCH($28, Sarg[[20..200]])]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->SEARCH($13, Sarg[[20..200]]), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"metrics.size":{"from":20.0,"to":200.0,"include_lower":true,"include_upper":true,"boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_with_asc_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_with_asc_sort.yaml new file mode 100644 index 00000000000..e0b91168f1f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_with_asc_sort.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <=($17, TIMESTAMP('2023-01-13 00:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->SEARCH($7, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-13 00:00:00':VARCHAR]]:VARCHAR), SORT->[{ + "@timestamp" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-13T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/range_with_desc_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/range_with_desc_sort.yaml new file mode 100644 index 00000000000..8af1fc7058d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/range_with_desc_sort.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$7], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$17], dir0=[DESC-nulls-last], fetch=[10]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <=($17, TIMESTAMP('2023-01-13 00:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->SEARCH($7, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-13 00:00:00':VARCHAR]]:VARCHAR), SORT->[{ + "@timestamp" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-13T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/scroll.yaml b/integ-test/src/test/resources/expectedOutput/calcite/scroll.yaml new file mode 100644 index 00000000000..59e68e48769 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/scroll.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/sort_keyword_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/sort_keyword_can_match_shortcut.yaml new file mode 100644 index 00000000000..501c35a492a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/sort_keyword_can_match_shortcut.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$25], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, meta.file, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + "meta.file" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"meta.file":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/sort_keyword_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/calcite/sort_keyword_no_can_match_shortcut.yaml new file mode 100644 index 00000000000..501c35a492a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/sort_keyword_no_can_match_shortcut.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$25], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'process.name:kernel':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, meta.file, host, metrics, aws, event], FILTER->query_string(MAP('query', 'process.name:kernel':VARCHAR)), SORT->[{ + "meta.file" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"process.name:kernel","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"meta.file":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_asc.yaml b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_asc.yaml new file mode 100644 index 00000000000..cbbc5106ec6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_asc.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$28], dir0=[ASC-nulls-first], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], SORT->[{ + "metrics.size" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_asc_with_match.yaml b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_asc_with_match.yaml new file mode 100644 index 00000000000..9aa906cc6ca --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_asc_with_match.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$28], dir0=[ASC-nulls-first], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->query_string(MAP('query', 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR)), SORT->[{ + "metrics.size" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"log.file.path:\\/var\\/log\\/messages\\/solarshark","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_desc.yaml b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_desc.yaml new file mode 100644 index 00000000000..3f059c7519f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_desc.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$28], dir0=[DESC-nulls-last], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], SORT->[{ + "metrics.size" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_desc_with_match.yaml b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_desc_with_match.yaml new file mode 100644 index 00000000000..b52bb433722 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/sort_numeric_desc_with_match.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(sort0=[$28], dir0=[DESC-nulls-last], fetch=[10]) + LogicalFilter(condition=[query_string(MAP('query', 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, metrics.size, aws, event], FILTER->query_string(MAP('query', 'log.file.path:\/var\/log\/messages\/solarshark':VARCHAR)), SORT->[{ + "metrics.size" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"query_string":{"query":"log.file.path:\\/var\\/log\\/messages\\/solarshark","fields":[],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1.0}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]},"sort":[{"metrics.size":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/term.yaml b/integ-test/src/test/resources/expectedOutput/calcite/term.yaml new file mode 100644 index 00000000000..21c0d2d0e5d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/term.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(agent=[$0], process=[$6], log=[$8], message=[$11], tags=[$12], cloud=[$13], input=[$15], @timestamp=[$17], ecs=[$18], data_stream=[$20], meta=[$24], host=[$26], metrics=[$27], aws=[$30], event=[$35]) + LogicalSort(fetch=[10]) + LogicalFilter(condition=[=($10, '/var/log/messages/birdknight')]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[agent, process, log, log.file.path, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], FILTER->=($3, '/var/log/messages/birdknight'), LIMIT->10, PROJECT->[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream, meta, host, metrics, aws, event], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"term":{"log.file.path":{"value":"/var/log/messages/birdknight","boost":1.0}}},"_source":{"includes":["agent","process","log","message","tags","cloud","input","@timestamp","ecs","data_stream","meta","host","metrics","aws","event"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/terms_significant_1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/terms_significant_1.yaml new file mode 100644 index 00000000000..2f3aab7b147 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/terms_significant_1.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[10]) + LogicalProject(count()=[$2], aws.cloudwatch.log_stream=[$0], process.name=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) + LogicalProject(aws.cloudwatch.log_stream=[$34], process.name=[$7]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, @timestamp, aws.cloudwatch.log_stream], FILTER->SEARCH($1, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), aws.cloudwatch.log_stream, process.name], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","@timestamp","aws.cloudwatch.log_stream"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"process.name":{"terms":{"field":"process.name","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/terms_significant_2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/terms_significant_2.yaml new file mode 100644 index 00000000000..cf04d9b8695 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/terms_significant_2.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[10]) + LogicalProject(count()=[$2], process.name=[$0], aws.cloudwatch.log_stream=[$1]) + LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) + LogicalProject(process.name=[$7], aws.cloudwatch.log_stream=[$34]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, @timestamp, aws.cloudwatch.log_stream], FILTER->SEARCH($1, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), process.name, aws.cloudwatch.log_stream], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","@timestamp","aws.cloudwatch.log_stream"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp.yaml new file mode 100644 index 00000000000..e57913c2f62 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\",\"missing\":\"_first\"\ + }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp_can_match_shortcut.yaml new file mode 100644 index 00000000000..b0a8deed278 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp_can_match_shortcut.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"process.name:kernel\"\ + ,\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"\ + asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp_no_can_match_shortcut.yaml new file mode 100644 index 00000000000..b0a8deed278 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_timestamp_no_can_match_shortcut.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"process.name:kernel\"\ + ,\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"\ + asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_with_after_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_with_after_timestamp.yaml new file mode 100644 index 00000000000..e57913c2f62 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/asc_sort_with_after_timestamp.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\",\"missing\":\"_first\"\ + }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_high.yaml b/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_high.yaml new file mode 100644 index 00000000000..ffe939e5a52 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_high.yaml @@ -0,0 +1,12 @@ +root: + name: ProjectOperator + description: + fields: "[dc(`agent.name`)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"dc(`agent.name`)\":{\"cardinality\"\ + :{\"field\":\"agent.name\"}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_high_2.yaml b/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_high_2.yaml new file mode 100644 index 00000000000..0c147949642 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_high_2.yaml @@ -0,0 +1,12 @@ +root: + name: ProjectOperator + description: + fields: "[dc(`event.id`)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"dc(`event.id`)\":{\"cardinality\"\ + :{\"field\":\"event.id\"}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_low.yaml b/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_low.yaml new file mode 100644 index 00000000000..f064201008e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/cardinality_agg_low.yaml @@ -0,0 +1,12 @@ +root: + name: ProjectOperator + description: + fields: "[dc(`cloud.region`)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"dc(`cloud.region`)\":{\"\ + cardinality\":{\"field\":\"cloud.region\"}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/composite_date_histogram_daily.yaml b/integ-test/src/test/resources/expectedOutput/ppl/composite_date_histogram_daily.yaml new file mode 100644 index 00000000000..9a0882dc49a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/composite_date_histogram_daily.yaml @@ -0,0 +1,19 @@ +root: + name: ProjectOperator + description: + fields: "[count(), span(`@timestamp`,1d)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"@timestamp\":{\"from\":1672358400000,\"to\":null,\"include_lower\":true,\"\ + include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ + :null,\"to\":1673092800000,\"include_lower\":true,\"include_upper\":false,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\"\ + :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(`@timestamp`,1d)\"\ + :{\"date_histogram\":{\"field\":\"@timestamp\",\"missing_bucket\":false,\"\ + order\":\"asc\",\"fixed_interval\":\"1d\"}}}]},\"aggregations\":{\"count()\"\ + :{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/composite_terms.yaml b/integ-test/src/test/resources/expectedOutput/ppl/composite_terms.yaml new file mode 100644 index 00000000000..481d9cdd423 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/composite_terms.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[count(), process.name, cloud.region]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"@timestamp\":{\"from\":1672617600000,\"to\":null,\"include_lower\":true,\"\ + include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ + :null,\"to\":1672653600000,\"include_lower\":true,\"include_upper\":false,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\"\ + :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"process.name\"\ + :{\"terms\":{\"field\":\"process.name\",\"missing_bucket\":true,\"missing_order\"\ + :\"last\",\"order\":\"desc\"}}},{\"cloud.region\":{\"terms\":{\"field\"\ + :\"cloud.region\",\"missing_bucket\":true,\"missing_order\":\"first\",\"\ + order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"\ + field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/composite_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/ppl/composite_terms_keyword.yaml new file mode 100644 index 00000000000..a7f12407647 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/composite_terms_keyword.yaml @@ -0,0 +1,23 @@ +root: + name: ProjectOperator + description: + fields: "[count(), process.name, cloud.region, aws.cloudwatch.log_stream]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"@timestamp\":{\"from\":1672617600000,\"to\":null,\"include_lower\":true,\"\ + include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ + :null,\"to\":1672653600000,\"include_lower\":true,\"include_upper\":false,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\"\ + :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"process.name\"\ + :{\"terms\":{\"field\":\"process.name\",\"missing_bucket\":true,\"missing_order\"\ + :\"last\",\"order\":\"desc\"}}},{\"cloud.region\":{\"terms\":{\"field\"\ + :\"cloud.region\",\"missing_bucket\":true,\"missing_order\":\"first\",\"\ + order\":\"asc\"}}},{\"aws.cloudwatch.log_stream\":{\"terms\":{\"field\"\ + :\"aws.cloudwatch.log_stream\",\"missing_bucket\":true,\"missing_order\"\ + :\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\"\ + :{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/date_histogram_hourly_agg.yaml b/integ-test/src/test/resources/expectedOutput/ppl/date_histogram_hourly_agg.yaml new file mode 100644 index 00000000000..72549142297 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/date_histogram_hourly_agg.yaml @@ -0,0 +1,15 @@ +root: + name: ProjectOperator + description: + fields: "[count(), span(`@timestamp`,1h)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"\ + composite\":{\"size\":1000,\"sources\":[{\"span(`@timestamp`,1h)\":{\"date_histogram\"\ + :{\"field\":\"@timestamp\",\"missing_bucket\":false,\"order\":\"asc\",\"\ + fixed_interval\":\"1h\"}}}]},\"aggregations\":{\"count()\":{\"value_count\"\ + :{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/date_histogram_minute_agg.yaml b/integ-test/src/test/resources/expectedOutput/ppl/date_histogram_minute_agg.yaml new file mode 100644 index 00000000000..be30d2a0801 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/date_histogram_minute_agg.yaml @@ -0,0 +1,19 @@ +root: + name: ProjectOperator + description: + fields: "[count(), span(`@timestamp`,1m)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"@timestamp\":{\"from\":1672531200000,\"to\":null,\"include_lower\":true,\"\ + include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ + :null,\"to\":1672704000000,\"include_lower\":true,\"include_upper\":false,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\"\ + :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(`@timestamp`,1m)\"\ + :{\"date_histogram\":{\"field\":\"@timestamp\",\"missing_bucket\":false,\"\ + order\":\"asc\",\"fixed_interval\":\"1m\"}}}]},\"aggregations\":{\"count()\"\ + :{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/default.yaml b/integ-test/src/test/resources/expectedOutput/ppl/default.yaml new file mode 100644 index 00000000000..23ca821adf6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/default.yaml @@ -0,0 +1,15 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp.yaml new file mode 100644 index 00000000000..ed13e6905cb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]},\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"missing\":\"_last\"\ + }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp_can_match_shortcut.yaml new file mode 100644 index 00000000000..e2fc446cdd7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp_can_match_shortcut.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"process.name:kernel\"\ + ,\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"\ + desc\",\"missing\":\"_last\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp_no_can_match_shortcut.yaml new file mode 100644 index 00000000000..e2fc446cdd7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_timestamp_no_can_match_shortcut.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"process.name:kernel\"\ + ,\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"\ + desc\",\"missing\":\"_last\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_with_after_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_with_after_timestamp.yaml new file mode 100644 index 00000000000..ed13e6905cb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/desc_sort_with_after_timestamp.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]},\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"missing\":\"_last\"\ + }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/keyword_in_range.yaml b/integ-test/src/test/resources/expectedOutput/ppl/keyword_in_range.yaml new file mode 100644 index 00000000000..2a85a0971ce --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/keyword_in_range.yaml @@ -0,0 +1,25 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\"\ + :{\"filter\":[{\"range\":{\"@timestamp\":{\"from\":1672531200000,\"to\"\ + :null,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},{\"\ + range\":{\"@timestamp\":{\"from\":null,\"to\":1672704000000,\"include_lower\"\ + :true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"query_string\":{\"query\":\"process.name:kernel\"\ + ,\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"agent\",\"process\",\"\ + log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\",\"\ + data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/keyword_terms.yaml b/integ-test/src/test/resources/expectedOutput/ppl/keyword_terms.yaml new file mode 100644 index 00000000000..3031899608b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/keyword_terms.yaml @@ -0,0 +1,25 @@ +root: + name: ProjectOperator + description: + fields: "[station, aws.cloudwatch.log_stream]" + children: + - name: TakeOrderedOperator + description: + limit: 500 + offset: 0 + sortList: + station: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"aws.cloudwatch.log_stream\"\ + :{\"terms\":{\"field\":\"aws.cloudwatch.log_stream\",\"missing_bucket\"\ + :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ + :{\"station\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/keyword_terms_low_cardinality.yaml b/integ-test/src/test/resources/expectedOutput/ppl/keyword_terms_low_cardinality.yaml new file mode 100644 index 00000000000..2a05fec4f3e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/keyword_terms_low_cardinality.yaml @@ -0,0 +1,25 @@ +root: + name: ProjectOperator + description: + fields: "[country, aws.cloudwatch.log_stream]" + children: + - name: TakeOrderedOperator + description: + limit: 50 + offset: 0 + sortList: + country: + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"aws.cloudwatch.log_stream\"\ + :{\"terms\":{\"field\":\"aws.cloudwatch.log_stream\",\"missing_bucket\"\ + :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ + :{\"country\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/multi_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/ppl/multi_terms_keyword.yaml new file mode 100644 index 00000000000..80709a3e0f5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/multi_terms_keyword.yaml @@ -0,0 +1,31 @@ +root: + name: ProjectOperator + description: + fields: "[count(), process.name, cloud.region]" + children: + - name: TakeOrderedOperator + description: + limit: 10 + offset: 0 + sortList: + count(): + sortOrder: DESC + nullOrder: NULL_LAST + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + range\":{\"@timestamp\":{\"from\":1672876800000,\"to\":null,\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\"\ + :{\"from\":null,\"to\":1672894800000,\"include_lower\":true,\"include_upper\"\ + :false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"process.name\":{\"terms\":{\"field\":\"process.name\"\ + ,\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"\ + }}},{\"cloud.region\":{\"terms\":{\"field\":\"cloud.region\",\"missing_bucket\"\ + :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ + :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message.yaml b/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message.yaml new file mode 100644 index 00000000000..20f024800e5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"((message:monkey\ + \ OR message:jackal) OR message:bear)\",\"fields\":[],\"type\":\"best_fields\"\ + ,\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\"\ + :true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\"\ + :50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message_filtered.yaml b/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message_filtered.yaml new file mode 100644 index 00000000000..fdd6d08721b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message_filtered.yaml @@ -0,0 +1,26 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\"\ + :{\"filter\":[{\"range\":{\"@timestamp\":{\"from\":1672704000000,\"to\"\ + :null,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},{\"\ + range\":{\"@timestamp\":{\"from\":null,\"to\":1672740000000,\"include_lower\"\ + :true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"query_string\":{\"query\":\"monkey jackal bear\"\ + ,\"fields\":[\"message^1.0\"],\"type\":\"best_fields\",\"default_operator\"\ + :\"or\",\"max_determinized_states\":10000,\"enable_position_increments\"\ + :true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\"\ + :50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"agent\",\"process\",\"\ + log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\",\"\ + data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message_filtered_sorted_num.yaml b/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message_filtered_sorted_num.yaml new file mode 100644 index 00000000000..4cc79a8db95 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/query_string_on_message_filtered_sorted_num.yaml @@ -0,0 +1,27 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\"\ + :{\"filter\":[{\"range\":{\"@timestamp\":{\"from\":1672704000000,\"to\"\ + :null,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},{\"\ + range\":{\"@timestamp\":{\"from\":null,\"to\":1672740000000,\"include_lower\"\ + :true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"query_string\":{\"query\":\"monkey jackal bear\"\ + ,\"fields\":[\"message^1.0\"],\"type\":\"best_fields\",\"default_operator\"\ + :\"or\",\"max_determinized_states\":10000,\"enable_position_increments\"\ + :true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\"\ + :50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"agent\",\"process\",\"\ + log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\",\"\ + data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\",\"missing\":\"_first\"\ + }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range.yaml new file mode 100644 index 00000000000..f9d406b2906 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range.yaml @@ -0,0 +1,19 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"@timestamp\":{\"from\":1672531200000,\"to\":null,\"include_lower\":true,\"\ + include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ + :null,\"to\":1672704000000,\"include_lower\":true,\"include_upper\":false,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\"\ + :{\"includes\":[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\"\ + ,\"input\",\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\"\ + ,\"aws\",\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_agg_1.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_agg_1.yaml new file mode 100644 index 00000000000..3f99d51799a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_agg_1.yaml @@ -0,0 +1,28 @@ +root: + name: ProjectOperator + description: + fields: "[count(), range_bucket]" + children: + - name: AggregationOperator + description: + aggregators: "[count()]" + groupBy: "[range_bucket]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + range_bucket: "CaseClause(whenClauses=[WhenClause(condition=<(metrics.size,\ + \ -10), result=\"range_1\"), WhenClause(condition=and(>=(metrics.size,\ + \ -10), <(metrics.size, 10)), result=\"range_2\"), WhenClause(condition=and(>=(metrics.size,\ + \ 10), <(metrics.size, 100)), result=\"range_3\"), WhenClause(condition=and(>=(metrics.size,\ + \ 100), <(metrics.size, 1000)), result=\"range_4\"), WhenClause(condition=and(>=(metrics.size,\ + \ 1000), <(metrics.size, 2000)), result=\"range_5\"), WhenClause(condition=>=(metrics.size,\ + \ 2000), result=\"range_6\")], defaultResult=null)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true, searchDone=false,\ + \ pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_agg_2.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_agg_2.yaml new file mode 100644 index 00000000000..400198d6635 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_agg_2.yaml @@ -0,0 +1,26 @@ +root: + name: ProjectOperator + description: + fields: "[count(), range_bucket]" + children: + - name: AggregationOperator + description: + aggregators: "[count()]" + groupBy: "[range_bucket]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + range_bucket: "CaseClause(whenClauses=[WhenClause(condition=<(metrics.size,\ + \ 100), result=\"range_1\"), WhenClause(condition=and(>=(metrics.size,\ + \ 100), <(metrics.size, 1000)), result=\"range_2\"), WhenClause(condition=and(>=(metrics.size,\ + \ 1000), <(metrics.size, 2000)), result=\"range_3\"), WhenClause(condition=>=(metrics.size,\ + \ 2000), result=\"range_4\")], defaultResult=null)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true, searchDone=false,\ + \ pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_auto_date_histo.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_auto_date_histo.yaml new file mode 100644 index 00000000000..159c9be49a1 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_auto_date_histo.yaml @@ -0,0 +1,38 @@ +root: + name: ProjectOperator + description: + fields: "[count(), auto_span, range_bucket]" + children: + - name: SortOperator + description: + sortList: + range_bucket: + sortOrder: ASC + nullOrder: NULL_FIRST + auto_span: + sortOrder: ASC + nullOrder: NULL_FIRST + children: + - name: AggregationOperator + description: + aggregators: "[count()]" + groupBy: "[auto_span, range_bucket]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + range_bucket: "CaseClause(whenClauses=[WhenClause(condition=<(metrics.size,\ + \ -10), result=\"range_1\"), WhenClause(condition=and(>=(metrics.size,\ + \ -10), <(metrics.size, 10)), result=\"range_2\"), WhenClause(condition=and(>=(metrics.size,\ + \ 10), <(metrics.size, 100)), result=\"range_3\"), WhenClause(condition=and(>=(metrics.size,\ + \ 100), <(metrics.size, 1000)), result=\"range_4\"), WhenClause(condition=and(>=(metrics.size,\ + \ 1000), <(metrics.size, 2000)), result=\"range_5\"), WhenClause(condition=>=(metrics.size,\ + \ 2000), result=\"range_6\")], defaultResult=null)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true,\ + \ searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_auto_date_histo_with_metrics.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_auto_date_histo_with_metrics.yaml new file mode 100644 index 00000000000..4f3004af106 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_auto_date_histo_with_metrics.yaml @@ -0,0 +1,36 @@ +root: + name: ProjectOperator + description: + fields: "[tmin, tavg, tmax, auto_span, range_bucket]" + children: + - name: SortOperator + description: + sortList: + range_bucket: + sortOrder: ASC + nullOrder: NULL_FIRST + auto_span: + sortOrder: ASC + nullOrder: NULL_FIRST + children: + - name: AggregationOperator + description: + aggregators: "[tmin, tavg, tmax]" + groupBy: "[auto_span, range_bucket]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + range_bucket: "CaseClause(whenClauses=[WhenClause(condition=<(metrics.size,\ + \ 100), result=\"range_1\"), WhenClause(condition=and(>=(metrics.size,\ + \ 100), <(metrics.size, 1000)), result=\"range_2\"), WhenClause(condition=and(>=(metrics.size,\ + \ 1000), <(metrics.size, 2000)), result=\"range_3\"), WhenClause(condition=>=(metrics.size,\ + \ 2000), result=\"range_4\")], defaultResult=null)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true,\ + \ searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_big_range_big_term_query.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_big_range_big_term_query.yaml new file mode 100644 index 00000000000..2663492036c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_big_range_big_term_query.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\"\ + :{\"filter\":[{\"term\":{\"process.name\":{\"value\":\"systemd\",\"boost\"\ + :1.0}}},{\"range\":{\"metrics.size\":{\"from\":1,\"to\":null,\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"range\":{\"metrics.size\":{\"from\":null,\"to\"\ + :100,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"\ + agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"\ + @timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\"\ + ,\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_small_range_big_term_query.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_small_range_big_term_query.yaml new file mode 100644 index 00000000000..3365a5b0813 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_small_range_big_term_query.yaml @@ -0,0 +1,19 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"metrics.size\":{\"from\":20,\"to\":null,\"include_lower\":true,\"include_upper\"\ + :true,\"boost\":1.0}}},{\"range\":{\"metrics.size\":{\"from\":null,\"to\"\ + :30,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"agent\",\"process\",\"\ + log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\",\"\ + data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_small_range_small_term_query.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_small_range_small_term_query.yaml new file mode 100644 index 00000000000..664d6428c7b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_field_conjunction_small_range_small_term_query.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"should\":[{\"term\"\ + :{\"aws.cloudwatch.log_stream\":{\"value\":\"indigodagger\",\"boost\":1.0}}},{\"\ + bool\":{\"filter\":[{\"range\":{\"metrics.size\":{\"from\":10,\"to\":null,\"\ + include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},{\"range\"\ + :{\"metrics.size\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\"\ + :true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"\ + agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"\ + @timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\"\ + ,\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_field_disjunction_big_range_small_term_query.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_field_disjunction_big_range_small_term_query.yaml new file mode 100644 index 00000000000..641befc2867 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_field_disjunction_big_range_small_term_query.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"should\":[{\"term\"\ + :{\"aws.cloudwatch.log_stream\":{\"value\":\"indigodagger\",\"boost\":1.0}}},{\"\ + bool\":{\"filter\":[{\"range\":{\"metrics.size\":{\"from\":1,\"to\":null,\"\ + include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},{\"range\"\ + :{\"metrics.size\":{\"from\":null,\"to\":100,\"include_lower\":true,\"include_upper\"\ + :true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"\ + agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"\ + @timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\"\ + ,\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_numeric.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_numeric.yaml new file mode 100644 index 00000000000..156f9ced9fe --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_numeric.yaml @@ -0,0 +1,19 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"metrics.size\":{\"from\":20,\"to\":null,\"include_lower\":true,\"include_upper\"\ + :true,\"boost\":1.0}}},{\"range\":{\"metrics.size\":{\"from\":null,\"to\"\ + :200,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}}],\"\ + adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"\ + agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"\ + @timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\"\ + ,\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_with_asc_sort.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_with_asc_sort.yaml new file mode 100644 index 00000000000..05a16cc76cf --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_with_asc_sort.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"@timestamp\":{\"from\":1672531200000,\"to\":null,\"include_lower\":true,\"\ + include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ + :null,\"to\":1673568000000,\"include_lower\":true,\"include_upper\":true,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\"\ + :{\"includes\":[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\"\ + ,\"input\",\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\"\ + ,\"aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\"\ + :\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/range_with_desc_sort.yaml b/integ-test/src/test/resources/expectedOutput/ppl/range_with_desc_sort.yaml new file mode 100644 index 00000000000..e7322cb282e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/range_with_desc_sort.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ + :{\"@timestamp\":{\"from\":1672531200000,\"to\":null,\"include_lower\":true,\"\ + include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ + :null,\"to\":1673568000000,\"include_lower\":true,\"include_upper\":true,\"\ + boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\"\ + :{\"includes\":[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\"\ + ,\"input\",\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\"\ + ,\"aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\"\ + :\"desc\",\"missing\":\"_last\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/scroll.yaml b/integ-test/src/test/resources/expectedOutput/ppl/scroll.yaml new file mode 100644 index 00000000000..23ca821adf6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/scroll.yaml @@ -0,0 +1,15 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/sort_keyword_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/sort_keyword_can_match_shortcut.yaml new file mode 100644 index 00000000000..20b18df0256 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/sort_keyword_can_match_shortcut.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"process.name:kernel\"\ + ,\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"meta.file\":{\"order\":\"\ + asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/sort_keyword_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/sort_keyword_no_can_match_shortcut.yaml new file mode 100644 index 00000000000..20b18df0256 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/sort_keyword_no_can_match_shortcut.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"process.name:kernel\"\ + ,\"fields\":[],\"type\":\"best_fields\",\"default_operator\":\"or\",\"max_determinized_states\"\ + :10000,\"enable_position_increments\":true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\"\ + :0,\"fuzzy_max_expansions\":50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"meta.file\":{\"order\":\"\ + asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_asc.yaml b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_asc.yaml new file mode 100644 index 00000000000..9d0f5c0ab0c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_asc.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]},\"sort\":[{\"metrics.size\":{\"order\":\"asc\",\"missing\":\"_first\"\ + }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_asc_with_match.yaml b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_asc_with_match.yaml new file mode 100644 index 00000000000..3718496568c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_asc_with_match.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"log.file.path:\\\ + \\/var\\\\/log\\\\/messages\\\\/solarshark\",\"fields\":[],\"type\":\"best_fields\"\ + ,\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\"\ + :true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\"\ + :50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"metrics.size\":{\"order\"\ + :\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_desc.yaml b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_desc.yaml new file mode 100644 index 00000000000..27126b931f0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_desc.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ + ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ + ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ + :[]},\"sort\":[{\"metrics.size\":{\"order\":\"desc\",\"missing\":\"_last\"\ + }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_desc_with_match.yaml b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_desc_with_match.yaml new file mode 100644 index 00000000000..a146d0531d5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/sort_numeric_desc_with_match.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"query_string\":{\"query\":\"log.file.path:\\\ + \\/var\\\\/log\\\\/messages\\\\/solarshark\",\"fields\":[],\"type\":\"best_fields\"\ + ,\"default_operator\":\"or\",\"max_determinized_states\":10000,\"enable_position_increments\"\ + :true,\"fuzziness\":\"AUTO\",\"fuzzy_prefix_length\":0,\"fuzzy_max_expansions\"\ + :50,\"phrase_slop\":0,\"escape\":false,\"auto_generate_synonyms_phrase_query\"\ + :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ + :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ + ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ + aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"metrics.size\":{\"order\"\ + :\"desc\",\"missing\":\"_last\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/term.yaml b/integ-test/src/test/resources/expectedOutput/ppl/term.yaml new file mode 100644 index 00000000000..ea9331ffa08 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/term.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[agent, process, log, message, tags, cloud, input, @timestamp, ecs, data_stream,\ + \ meta, host, metrics, aws, event]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ + size\":10,\"timeout\":\"1m\",\"query\":{\"term\":{\"log.file.path\":{\"\ + value\":\"/var/log/messages/birdknight\",\"boost\":1.0}}},\"_source\":{\"\ + includes\":[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\"\ + ,\"input\",\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\"\ + ,\"aws\",\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/terms_significant_1.yaml b/integ-test/src/test/resources/expectedOutput/ppl/terms_significant_1.yaml new file mode 100644 index 00000000000..09bd047cfb9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/terms_significant_1.yaml @@ -0,0 +1,27 @@ +root: + name: ProjectOperator + description: + fields: "[count(), aws.cloudwatch.log_stream, process.name]" + children: + - name: LimitOperator + description: + limit: 10 + offset: 0 + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + range\":{\"@timestamp\":{\"from\":1672531200000,\"to\":null,\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\"\ + :{\"from\":null,\"to\":1672704000000,\"include_lower\":true,\"include_upper\"\ + :false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"aws.cloudwatch.log_stream\":{\"terms\":{\"field\":\"aws.cloudwatch.log_stream\"\ + ,\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"\ + }}},{\"process.name\":{\"terms\":{\"field\":\"process.name\",\"missing_bucket\"\ + :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ + :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/terms_significant_2.yaml b/integ-test/src/test/resources/expectedOutput/ppl/terms_significant_2.yaml new file mode 100644 index 00000000000..908ab96f04b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/terms_significant_2.yaml @@ -0,0 +1,27 @@ +root: + name: ProjectOperator + description: + fields: "[count(), process.name, aws.cloudwatch.log_stream]" + children: + - name: LimitOperator + description: + limit: 10 + offset: 0 + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + range\":{\"@timestamp\":{\"from\":1672531200000,\"to\":null,\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\"\ + :{\"from\":null,\"to\":1672704000000,\"include_lower\":true,\"include_upper\"\ + :false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"process.name\":{\"terms\":{\"field\":\"process.name\"\ + ,\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"\ + }}},{\"aws.cloudwatch.log_stream\":{\"terms\":{\"field\":\"aws.cloudwatch.log_stream\"\ + ,\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"\ + }}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"\ + }}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file