Update big5 ppl queries and check plans

integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java

@@ -21,24 +21,25 @@ public void init() throws Exception {
 
   @Test
   public void bin_bins() throws IOException {
-    String ppl = sanitize(loadFromFile("big5/queries/bin_bins.ppl"));
+    String ppl = sanitize(loadExpectedQuery("bin_bins.ppl"));
     timing(summary, "bin_bins", ppl);
   }
 
   @Test
   public void bin_span_log() throws IOException {
-    String ppl = sanitize(loadFromFile("big5/queries/bin_span_log.ppl"));
+    String ppl = sanitize(loadExpectedQuery("bin_span_log.ppl"));
     timing(summary, "bin_span_log", ppl);
   }
 
   @Test
   public void bin_span_time() throws IOException {
-    String ppl = sanitize(loadFromFile("big5/queries/bin_span_time.ppl"));
+    String ppl = sanitize(loadExpectedQuery("bin_span_time.ppl"));
     timing(summary, "bin_span_time", ppl);
   }
 
+  @Test
   public void coalesce_nonexistent_field_fallback() throws IOException {
-    String ppl = sanitize(loadFromFile("big5/queries/coalesce_nonexistent_field_fallback.ppl"));
+    String ppl = sanitize(loadExpectedQuery("coalesce_nonexistent_field_fallback.ppl"));
     timing(summary, "coalesce_nonexistent_field_fallback", ppl);
   }
 }

integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java

@@ -710,18 +710,4 @@ public void testExplainSearchWildcardStar() throws IOException {
         explainQueryToString(
             String.format("search source=%s severityText=ERR*", TEST_INDEX_OTEL_LOGS)));
   }
-
-  protected String loadExpectedPlan(String fileName) throws IOException {
-    String prefix;
-    if (isCalciteEnabled()) {
-      if (isPushdownDisabled()) {
-        prefix = "expectedOutput/calcite_no_pushdown/";
-      } else {
-        prefix = "expectedOutput/calcite/";
-      }
-    } else {
-      prefix = "expectedOutput/ppl/";
-    }
-    return loadFromFile(prefix + fileName);
-  }
 }

integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java

@@ -392,4 +392,18 @@ protected static String loadFromFile(String filename) {
       throw new RuntimeException(e);
     }
   }
+
+  protected String loadExpectedPlan(String fileName) throws IOException {
+    String prefix;
+    if (isCalciteEnabled()) {
+      if (isPushdownDisabled()) {
+        prefix = "expectedOutput/calcite_no_pushdown/";
+      } else {
+        prefix = "expectedOutput/calcite/";
+      }
+    } else {
+      prefix = "expectedOutput/ppl/";
+    }
+    return loadFromFile(prefix + fileName);
+  }
 }

integ-test/src/test/resources/big5/queries/asc_sort_timestamp.ppl

@@ -1,3 +1,18 @@
+/*
+{
+  "name": "asc_sort_timestamp",
+  "operation-type": "search",
+  "index": "{{index_name | default('big5')}}",
+  "body": {
+    "query": {
+      "match_all": {}
+    },
+    "sort" : [
+      {"@timestamp" : "asc"}
+    ]
+  }
+}
+ */
 source = big5
 | sort + `@timestamp`
 | head 10

integ-test/src/test/resources/big5/queries/asc_sort_timestamp_can_match_shortcut.ppl

@@ -1,3 +1,21 @@
+/*
+{
+  "name": "asc_sort_timestamp_can_match_shortcut",
+  "operation-type": "search",
+  "index": "{{index_name | default('big5')}}",
+  "body": {
+    "track_total_hits": false,
+    "query": {
+      "match": {
+        "process.name": "kernel"
+      }
+    },
+    "sort" : [
+      {"@timestamp" : "asc"}
+    ]
+  }
+}
+*/
 source = big5 process.name=kernel
 | sort + `@timestamp`
 | head 10

integ-test/src/test/resources/big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl

@@ -1,3 +1,24 @@
+/*
+{
+  "name": "asc_sort_timestamp_no_can_match_shortcut",
+  "operation-type": "search",
+  "index": "{{index_name | default('big5')}}",
+  "request-params" : {
+      "pre_filter_shard_size" : 100000
+  },
+  "body": {
+    "track_total_hits": false,
+    "query": {
+      "match": {
+        "process.name": "kernel"
+      }
+    },
+    "sort" : [
+      {"@timestamp" : "asc"}
+    ]
+  }
+}
+*/
 source = big5 process.name=kernel
 | sort + `@timestamp`
 | head 10

integ-test/src/test/resources/big5/queries/asc_sort_with_after_timestamp.ppl

@@ -1,3 +1,24 @@
+/*
+{
+  "name": "asc_sort_with_after_timestamp",
+  "operation-type": "search",
+  "index": "{{index_name | default('big5')}}",
+  "body": {
+    "track_total_hits": false,
+    "query": {
+      "match_all": {}
+    },
+    "sort" : [
+      {"@timestamp" : "asc"}
+    ],
+{% if distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list or distribution_version.split('.') | map('int') | list >= "7.0.0".split('.') | map('int') | list %}
+      "search_after": ["2023-01-01T23:59:58.000Z"]
+{% else %}
+      "search_after": [1673049598]
+{% endif %}
+  }
+}
+*/
 source = big5
 | sort + `@timestamp`
 | head 10

integ-test/src/test/resources/big5/queries/cardinality_agg_high.ppl

@@ -0,0 +1,22 @@
+/*
+{
+  "name": "cardinality-agg-high",
+  "operation-type": "search",
+  "index": "{{index_name | default('big5')}}",
+  "body": {
+    "size": 0,
+    "aggs": {
+      "agent": {
+        "cardinality": {
+         "field": "agent.name"
+          {% if distribution_version.split('.') | map('int') | list  >= "2.19.1".split('.') | map('int') | list and distribution_version.split('.') | map('int') | list  < "6.0.0".split('.') | map('int') | list %}
+            , "execution_hint": "ordinals"
+          {% endif %}
+        }
+      }
+    }
+  }
+}
+*/
+source = big5
+| stats dc(`agent.name`)

integ-test/src/test/resources/big5/queries/cardinality_agg_high_2.ppl

@@ -0,0 +1,21 @@
+/*
+{
+  "name": "cardinality-agg-high-2",
+  "operation-type": "search",
+  "index": "{{index_name | default('big5')}}",
+  "request-timeout": 1800,
+  "body": {
+    "size": 0,
+    "aggs": {
+      "agent": {
+        "cardinality": {
+         "field": "event.id",
+         "execution_hint":"ordinals"
+        }
+      }
+    }
+  }
+}
+*/
+source = big5
+| stats dc(`event.id`)

integ-test/src/test/resources/big5/queries/cardinality_agg_low.ppl

@@ -0,0 +1,19 @@
+/*
+{
+  "name": "cardinality-agg-low",
+  "operation-type": "search",
+  "index": "{{index_name | default('big5')}}",
+  "body": {
+    "size": 0,
+    "aggs": {
+      "region": {
+        "cardinality": {
+          "field": "cloud.region"
+        }
+      }
+    }
+  }
+}
+*/
+source = big5
+| stats dc(`cloud.region`)

integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl

@@ -1,3 +1,34 @@
+/*
+{
+  "name": "composite-date_histogram-daily",
+  "operation-type": "search",
+  "index": "{{index_name | default('big5')}}",
+  "body": {
+    "size": 0,
+    "query": {
+      "range": {
+        "@timestamp": {
+          "gte": "2022-12-30T00:00:00",
+          "lt": "2023-01-07T12:00:00"
+        }
+      }
+    },
+    "aggs": {
+      "logs": {
+        "composite": {
+          "sources": [
+    {% if distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list or distribution_version.split('.') | map('int') | list >= "7.0.0".split('.') | map('int') | list %}
+              { "date": { "date_histogram": { "field": "@timestamp", "calendar_interval": "day" } } }
+    {% else %}
+              { "date": { "date_histogram": { "field": "@timestamp", "interval": "day" } } }
+    {% endif %}
+          ]
+        }
+      }
+    }
+  }
+}
+*/
 source = big5
 | where `@timestamp` >= '2022-12-30 00:00:00' and `@timestamp` < '2023-01-07 12:00:00'
 | stats count() by span(`@timestamp`, 1d)

integ-test/src/test/resources/big5/queries/composite_terms.ppl

@@ -1,4 +1,32 @@
+/*
+{
+  "name": "composite-terms",
+  "operation-type": "search",
+  "index": "{{index_name | default('big5')}}",
+  "body": {
+    "size": 0,
+    "query": {
+      "range": {
+        "@timestamp": {
+          "gte": "2023-01-02T00:00:00",
+          "lt": "2023-01-02T10:00:00"
+        }
+      }
+    },
+    "aggs": {
+      "logs": {
+        "composite": {
+          "sources": [
+            { "process_name": { "terms": { "field": "process.name", "order": "desc" }}},
+            { "cloud_region": { "terms": { "field": "cloud.region", "order": "asc" }}}
+          ]
+        }
+      }
+    }
+  }
+}
+*/
 source = big5
-| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-03 00:00:00'
+| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00'
 | stats count() by `process.name`, `cloud.region`
 | sort - `process.name`, + `cloud.region`

integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl

@@ -1,4 +1,33 @@
+/*
+{
+  "name": "composite_terms-keyword",
+  "operation-type": "search",
+  "index": "{{index_name | default('big5')}}",
+  "body": {
+    "size": 0,
+    "query": {
+      "range": {
+        "@timestamp": {
+          "gte": "2023-01-02T00:00:00",
+          "lt": "2023-01-02T10:00:00"
+        }
+      }
+    },
+    "aggs": {
+      "logs": {
+        "composite": {
+          "sources": [
+            { "process_name": { "terms": { "field": "process.name", "order": "desc" }}},
+            { "cloud_region": { "terms": { "field": "cloud.region", "order": "asc" }}},
+            { "cloudstream": { "terms": { "field": "aws.cloudwatch.log_stream", "order": "asc" }}}
+          ]
+        }
+      }
+    }
+  }
+}
+*/
 source = big5
-| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-03 00:00:00'
+| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00'
 | stats count() by `process.name`, `cloud.region`, `aws.cloudwatch.log_stream`
 | sort - `process.name`, + `cloud.region`, + `aws.cloudwatch.log_stream`

integ-test/src/test/resources/big5/queries/date_histogram_hourly_agg.ppl

@@ -1,2 +1,24 @@
+/*
+{
+  "name": "date_histogram_hourly_agg",
+  "operation-type": "search",
+  "index": "{{index_name | default('big5')}}",
+  "body": {
+    "size": 0,
+    "aggs": {
+      "by_hour": {
+        "date_histogram": {
+          "field": "@timestamp",
+          {% if distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list or distribution_version.split('.') | map('int') | list >= "7.0.0".split('.') | map('int') | list %}
+            "calendar_interval": "hour"
+          {% else %}
+            "interval": "hour"
+          {% endif %}
+        }
+      }
+    }
+  }
+}
+*/
 source = big5
 | stats count() by span(`@timestamp`, 1h)

integ-test/src/test/resources/big5/queries/date_histogram_minute_agg.ppl

@@ -1,3 +1,33 @@
+/*
+{
+  "name": "date_histogram_minute_agg",
+  "operation-type": "search",
+  "index": "{{index_name | default('big5')}}",
+  "body": {
+    "size": 0,
+    "query": {
+      "range": {
+        "@timestamp": {
+          "gte": "2023-01-01T00:00:00",
+          "lt": "2023-01-03T00:00:00"
+        }
+      }
+    },
+    "aggs": {
+      "by_hour": {
+        "date_histogram": {
+          "field": "@timestamp",
+          {% if distribution_version.split('.') | map('int') | list < "6.0.0".split('.') | map('int') | list or distribution_version.split('.') | map('int') | list >= "7.0.0".split('.') | map('int') | list %}
+            "calendar_interval": "minute"
+          {% else %}
+            "interval": "minute"
+          {% endif %}
+        }
+      }
+    }
+  }
+}
+*/
 source = big5
 | where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00'
 | stats count() by span(`@timestamp`, 1m)