extended.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Datafusion extended tests

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

# https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#running-your-pull_request-workflow-when-a-pull-request-merges
#
# These jobs only run on the `main` branch as they are time consuming
# and should not fail often.
#
# We still run them as they provide important coverage to ensure correctness
# in the (very rare) event of a hash failure or sqlite library query failure.
on:
  # Run on all commits to main
  push:
    branches:
      - main

jobs:
  # Check crate compiles and base cargo check passes
  linux-build-lib:
    name: linux build test
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v4
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: stable
      - name: Prepare cargo build
        run: |
          cargo check --profile ci --all-targets
          cargo clean

#  # Run extended tests (with feature 'extended_tests')
#  # Disabling as it is running out of disk space
#  # see https://github.com/apache/datafusion/issues/14576
#  linux-test-extended:
#    name: cargo test 'extended_tests' (amd64)
#    needs: linux-build-lib
#    runs-on: ubuntu-latest
#    container:
#      image: amd64/rust
#    steps:
#      - uses: actions/checkout@v4
#        with:
#          submodules: true
#          fetch-depth: 1
#      - name: Setup Rust toolchain
#        uses: ./.github/actions/setup-builder
#        with:
#          rust-version: stable
#      - name: Run tests (excluding doctests)
#        run: cargo test --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace,extended_tests
#      - name: Verify Working Directory Clean
#        run: git diff --exit-code
#      - name: Cleanup
#        run: cargo clean

  # Check answers are correct when hash values collide
  hash-collisions:
    name: cargo test hash collisions (amd64)
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 1
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: stable
      - name: Run tests
        run: |
          cd datafusion
          cargo test  --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --workspace --lib --tests --features=force_hash_collisions,avro,extended_tests
          cargo clean

  sqllogictest-sqlite:
    name: "Run sqllogictests with the sqlite test suite"
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 1
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: stable
      - name: Run sqllogictest
        run: |
          cargo test --profile release-nonlto --test sqllogictests -- --include-sqlite
          cargo clean