test-warehouse.yml

name: Test warehouse platform
on:
  workflow_dispatch:
    inputs:
      warehouse-type:
        type: choice
        required: true
        description: Type of warehouse platform
        options:
          - postgres
          - snowflake
          - bigquery
          - redshift
          - databricks
          - databricks_catalog
          - spark
          - athena
          - trino
      elementary-ref:
        type: string
        required: false
        description: Branch or tag to checkout for 'elementary' repository
      dbt-data-reliability-ref:
        type: string
        required: false
        description: Branch or tag to checkout for 'dbt-data-reliability' repository
      dbt-version:
        type: string
        required: false
        default: "latest_official"
        description: dbt's version to test with

  workflow_call:
    inputs:
      warehouse-type:
        type: string
        required: true
      elementary-ref:
        type: string
        required: false
      dbt-data-reliability-ref:
        type: string
        required: false
      dbt-version:
        type: string
        default: "latest_official"
        required: false

env:
  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
  TESTS_DIR: ${{ github.workspace }}/dbt-data-reliability/integration_tests

jobs:
  test:
    runs-on: ubuntu-20.04
    concurrency:
      # This is what eventually defines the schema name in the data platform.
      group: tests_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}_${{ github.head_ref || github.ref_name }}
      cancel-in-progress: true
    steps:
      - name: Checkout Elementary
        uses: actions/checkout@v4
        with:
          repository: elementary-data/elementary
          path: elementary
          ref: ${{ inputs.elementary-ref }}

      - name: Checkout dbt package
        uses: actions/checkout@v4
        with:
          path: dbt-data-reliability
          ref: ${{ inputs.dbt-data-reliability-ref }}

      - name: Start Postgres
        if: inputs.warehouse-type == 'postgres'
        working-directory: ${{ env.TESTS_DIR }}
        run: docker compose up -d postgres

      - name: Start Trino
        if: inputs.warehouse-type == 'trino'
        working-directory: ${{ env.TESTS_DIR }}
        run: docker compose -f docker-compose-trino.yml up -d

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"
          cache: "pip"

      - name: Install Spark requirements
        if: inputs.warehouse-type == 'spark'
        run: sudo apt-get install python-dev libsasl2-dev gcc

      - name: Install compatible databricks connector (not limited in older dbt-databricks versions)
        if: startsWith(inputs.warehouse-type, 'databricks') && inputs.dbt-version < '1.7.0'
        run: pip install databricks-sql-connector==2.9.3

      - name: Install dbt
        run:
          pip install${{ (inputs.dbt-version == 'latest_pre' && ' --pre') || '' }}
          "dbt-core${{ (!startsWith(inputs.dbt-version, 'latest') && format('=={0}', inputs.dbt-version)) || '' }}"
          "dbt-${{ (inputs.warehouse-type == 'databricks_catalog' && 'databricks') || (inputs.warehouse-type == 'spark' && 'spark[PyHive]') || (inputs.warehouse-type == 'athena' && 'athena-community') || inputs.warehouse-type }}${{ (!startsWith(inputs.dbt-version, 'latest') && format('<={0}', inputs.dbt-version)) || '' }}"

      - name: Install Elementary
        run: pip install "./elementary[${{ (inputs.warehouse-type == 'databricks_catalog' && 'databricks') || inputs.warehouse-type }}]"

      - name: Install dependencies
        working-directory: ${{ env.TESTS_DIR }}
        run: |
          dbt deps --project-dir dbt_project
          pip install -r requirements.txt

      - name: Write dbt profiles
        env:
          PROFILES_YML: ${{ secrets.CI_PROFILES_YML }}
        run: |
          mkdir -p ~/.dbt
          DBT_VERSION=$(pip show dbt-core | grep -i version | awk '{print $2}' | sed 's/\.//g')
          UNDERSCORED_REF_NAME=$(echo "${{ inputs.warehouse-type }}_dbt_${DBT_VERSION}_${BRANCH_NAME}" | awk '{print tolower($0)}' | head -c 40 | sed "s/-/_/g")
          echo "$PROFILES_YML" | base64 -d | sed "s/<SCHEMA_NAME>/dbt_pkg_$UNDERSCORED_REF_NAME/g" > ~/.dbt/profiles.yml

      - name: Check DWH connection
        working-directory: ${{ env.TESTS_DIR }}
        run: |
          dbt debug -t "${{ inputs.warehouse-type }}"

      - name: Test
        working-directory: "${{ env.TESTS_DIR }}/tests"
        run: py.test -n8 -vvv --target "${{ inputs.warehouse-type }}" --junit-xml=test-results.xml --html=detailed_report_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}.html --self-contained-html

      - name: Upload test results
        if: always()
        uses: pmeier/pytest-results-action@main
        with:
          path: ${{ env.TESTS_DIR }}/tests/test-results.xml
          summary: true
          display-options: fEX
          fail-on-empty: true

      - name: Upload HTML report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: detailed_report_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}
          path: ${{ env.TESTS_DIR }}/tests/detailed_report_${{ inputs.warehouse-type }}_dbt_${{ inputs.dbt-version }}.html