Merge branch 'default-main'

* default-main: (34 commits)
  Enhance dynamic source clause to support only metadata filters (#4554)
  Make nested alias type support referring to outer context (#4673)
  Update big5 ppl queries and check plans (#4668)
  Support push down sort after limit (#4657)
  Use table scan rowType in filter pushdown could fix rename issue (#4670)
  Fix: Support Alias Fields in MIN, MAX, FIRST, LAST, and TAKE Aggregations (#4621)
  Fix bin nested fields issue (#4606)
  Add `per_minute`, `per_hour`, `per_day` function support (#4531)
  Pushdown sort aggregate metrics (#4603)
  Followup: Change ComparableLinkedHashMap to compare Key than Value (#4648)
  Mitigate the CI failure caused by 500 Internal Server Error (#4646)
  Allow renaming group-by fields to existing field names (#4586)
  Publish internal modules separately for downstream reuse (#4484)
  Revert "Update grammar files and developer guide (#4301)" (#4643)
  Support Automatic Type Conversion for REX/SPATH/PARSE Command Extractions (#4599)
  Replace all dots in fields of table scan's PhysType (#4633)
  Return comparable LinkedHashMap in `valueForCalcite()` of ExprTupleValue (#4629)
  Refactor JsonExtractAllFunctionIT and MapConcatFunctionIT (#4623)
  Pushdown case function in aggregations as range queries (#4400)
  Update GEOIP function to support IP types as input (#4613)
  ...

Signed-off-by: Asif Bashar <[email protected]>

Author: asifabashar

.github/workflows/maven-publish-modules.yml

@@ -0,0 +1,47 @@
+name: Publish unified query modules to maven
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
+
+env:
+  SNAPSHOT_REPO_URL: https://ci.opensearch.org/ci/dbc/snapshots/maven/
+
+jobs:
+  publish-unified-query-modules:
+    strategy:
+      fail-fast: false
+    if: github.repository == 'opensearch-project/sql'
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - uses: actions/setup-java@v3
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 21
+      - uses: actions/checkout@v3
+      - name: Load secret
+        uses: 1password/load-secrets-action@v2
+        with:
+          # Export loaded secrets as environment variables
+          export-env: true
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
+      - name: publish snapshots to maven
+        run: |
+          ./gradlew publishUnifiedQueryPublicationToSnapshotsRepository

.github/workflows/maven-publish.yml

@@ -5,11 +5,8 @@ on:
   push:
     branches:
       - main
-      - 1.*
-      - 2.*
-
-env:
-  SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
 
 jobs:
   build-and-publish-snapshots:
@@ -35,9 +32,13 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
-
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
       - name: publish snapshots to maven
         run: |
-          ./gradlew publishPluginZipPublicationToSnapshotsRepository
+          ./gradlew publishPluginZipPublicationToSnapshotsRepository

.github/workflows/publish-async-query-core.yml

@@ -5,8 +5,8 @@ on:
   push:
     branches:
       - main
-      - 1.*
-      - 2.*
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
     paths:
       - 'async-query-core/**'
       - '.github/workflows/publish-async-query-core.yml'
@@ -18,7 +18,6 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
   COMMIT_MAP_FILENAME: commit-history-async-query-core.json
 
 jobs:
@@ -47,8 +46,19 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+
+      - name: Export SNAPSHOT_REPO_URL
+        run: |
+          snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }}
+          echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
 
       - name: Set commit ID
         id: set_commit
@@ -83,4 +93,4 @@ jobs:
           source ./.github/maven-publish-utils.sh
           
           # Call the main function for async-query-core
-          publish_async_query_core "${{ steps.extract_version.outputs.VERSION }}" "${{ steps.set_commit.outputs.commit_id }}"
+          publish_async_query_core "${{ steps.extract_version.outputs.VERSION }}" "${{ steps.set_commit.outputs.commit_id }}"

.github/workflows/publish-grammar-files.yml

@@ -5,8 +5,8 @@ on:
   push:
     branches:
       - main
-      - 1.*
-      - 2.*
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
     paths:
       - 'language-grammar/src/main/antlr4/**'
       - 'language-grammar/build.gradle'
@@ -19,7 +19,6 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
   COMMIT_MAP_FILENAME: commit-history-language-grammar.json
 
 jobs:
@@ -51,8 +50,19 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+
+      - name: Export SNAPSHOT_REPO_URL
+        run: |
+          snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }}
+          echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
 
       - name: Set version
         id: set_version

DEVELOPER_GUIDE.rst

@@ -173,7 +173,6 @@ Here are other files and sub-folders that you are likely to touch:
 - ``build.gradle``: Gradle build script.
 - ``docs``: documentation for developers and reference manual for users.
 - ``doc-test``: code that run .rst docs in ``docs`` folder by Python doctest library.
-- ``language-grammar``: centralized package for ANTLR grammar files. See `Language Grammar Package`_ for details.
 
 Note that other related project code has already merged into this single repository together:
 
@@ -444,29 +443,3 @@ with an appropriate label `backport <backport-branch-name>` is merged to main wi
 PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR and make sure the
 backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR
 to the `1.x` branch.
-
-Language Grammar Package
-========================
-
-The ``language-grammar`` package serves as a centralized repository for all ANTLR grammar files used throughout the OpenSearch SQL project. This package contains the definitive versions of grammar files for:
-
-- SQL parsing (``OpenSearchSQLParser.g4``, ``OpenSearchSQLLexer.g4``)
-- PPL parsing (``OpenSearchPPLParser.g4``, ``OpenSearchPPLLexer.g4``)
-- Legacy SQL parsing (``OpenSearchLegacySqlParser.g4``, ``OpenSearchLegacySqlLexer.g4``)
-- Spark SQL extensions (``SparkSqlBase.g4``, ``FlintSparkSqlExtensions.g4``, ``SqlBaseParser.g4``, ``SqlBaseLexer.g4``)
-
-Purpose
--------
-
-The language-grammar package enables sharing of grammar files between the main SQL repository and the Spark repository, ensuring consistency and reducing duplication. Once updated, the package automatically triggers CI to upload the new version to Maven Central for consumption by other projects.
-
-Updating Grammar Files
-----------------------
-
-When grammar files are modified in their respective modules (``sql/``, ``ppl/``, ``legacy/``, ``async-query-core/``), they must be manually copied to the ``language-grammar/src/main/antlr4/`` directory.
-
-**Workflow:**
-
-1. Modify grammar files in their source locations (e.g., ``sql/src/main/antlr/``)
-2. Copy updated files to ``language-grammar/src/main/antlr4/``
-3. Commit changes to trigger automatic Maven publication via CI

api/README.md

@@ -0,0 +1,75 @@
+# Unified Query API
+
+This module provides a high-level integration layer for the Calcite-based query engine, enabling external systems such as Apache Spark or command-line tools to parse and analyze queries without exposing low-level internals.
+
+## Overview
+
+The `UnifiedQueryPlanner` serves as the primary entry point for external consumers. It accepts PPL (Piped Processing Language) queries and returns Calcite `RelNode` logical plans as intermediate representation.
+
+## Usage
+
+Use the declarative, fluent builder API to initialize the `UnifiedQueryPlanner`.
+
+```java
+UnifiedQueryPlanner planner = UnifiedQueryPlanner.builder()
+    .language(QueryType.PPL)
+    .catalog("opensearch", schema)
+    .defaultNamespace("opensearch")
+    .cacheMetadata(true)
+    .build();
+
+RelNode plan = planner.plan("source = opensearch.test");
+```
+
+## Development & Testing
+
+A set of unit tests is provided to validate planner behavior.
+
+To run tests:
+
+```
+./gradlew :api:test
+```
+
+## Integration Guide
+
+This guide walks through how to integrate unified query planner into your application.
+
+### Step 1: Add Dependency
+
+The module is currently published as a snapshot to the AWS Sonatype Snapshots repository. To include it as a dependency in your project, add the following to your `pom.xml` or `build.gradle`:
+
+```xml
+<dependency>
+  <groupId>org.opensearch.query</groupId>
+  <artifactId>unified-query-api</artifactId>
+  <version>YOUR_VERSION_HERE</version>
+</dependency>
+```
+
+### Step 2: Implement a Calcite Schema
+
+You must implement the Calcite `Schema` interface and register them using the fluent `catalog()` method on the builder.
+
+```java
+public class MySchema extends AbstractSchema {
+  @Override
+  protected Map<String, Table> getTableMap() {
+    return Map.of(
+      "test_table",
+      new AbstractTable() {
+        @Override
+        public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+          return typeFactory.createStructType(
+            List.of(typeFactory.createSqlType(SqlTypeName.INTEGER)),
+            List.of("id"));
+        }
+      });
+  }
+}
+```
+
+## Future Work
+
+- Expand support to SQL language.
+- Extend planner to generate optimized physical plans using Calcite's optimization frameworks.

api/build.gradle

@@ -0,0 +1,71 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+plugins {
+    id 'java-library'
+    id 'jacoco'
+    id 'com.diffplug.spotless'
+}
+
+dependencies {
+    api project(':ppl')
+
+    testImplementation group: 'junit', name: 'junit', version: '4.13.2'
+    testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}"
+    testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}"
+    testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.38.0'
+}
+
+spotless {
+    java {
+        target fileTree('.') {
+            include '**/*.java'
+            exclude '**/build/**', '**/build-*/**', 'src/main/gen/**'
+        }
+        importOrder()
+        removeUnusedImports()
+        trimTrailingWhitespace()
+        endWithNewline()
+        googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format')
+    }
+}
+
+test {
+    testLogging {
+        events "passed", "skipped", "failed"
+        exceptionFormat "full"
+    }
+}
+
+jacocoTestReport {
+    reports {
+        html.required = true
+        xml.required = true
+    }
+    afterEvaluate {
+        classDirectories.setFrom(files(classDirectories.files.collect {
+            fileTree(dir: it,
+                    exclude: ['**/antlr/parser/**'])
+        }))
+    }
+}
+test.finalizedBy(project.tasks.jacocoTestReport)
+jacocoTestCoverageVerification {
+    violationRules {
+        rule {
+            limit {
+                minimum = 0.9
+            }
+
+        }
+    }
+    afterEvaluate {
+        classDirectories.setFrom(files(classDirectories.files.collect {
+            fileTree(dir: it,
+                    exclude: ['**/antlr/parser/**'])
+        }))
+    }
+}
+check.dependsOn jacocoTestCoverageVerification