Python.yml

name: Python
on:
  push:
    paths-ignore:
      - '**.md'
  pull_request:
    paths-ignore:
      - '**.md'
      - 'examples/**'
      - 'test/**'
      - 'tools/juliapkg/**'
      - 'tools/nodejs/**'
      - 'tools/rpkg/**'
      - '.github/workflows/**'
      - '!.github/workflows/Python.yml'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/master' || github.sha }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

env:
  GH_TOKEN: ${{ secrets.GH_TOKEN }}
  TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
  AWS_ACCESS_KEY_ID: AKIAVBLKPL2ZW2T7TYFQ
  AWS_SECRET_ACCESS_KEY: ${{ secrets.NODE_PRE_GYP_SECRETACCESSKEY }}
  NODE_AUTH_TOKEN: ${{secrets.NODE_AUTH_TOKEN}}

jobs:
#  This is just a sanity check of Python 3.9 running with Arrow
   linux-python3-9:
    name: Python 3.9 Linux
    runs-on: ubuntu-20.04

    env:
      CIBW_BUILD: 'cp39-manylinux_x86_64'
      CIBW_TEST_COMMAND: 'python -m pytest {project}/tests'
      SETUPTOOLS_SCM_NO_LOCAL: 'yes'
      TWINE_USERNAME: 'hfmuehleisen'
      PYTEST_TIMEOUT: '600'

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - uses: actions/setup-python@v2
      with:
        python-version: '3.7'

    - name: Install
      run: pip install cibuildwheel twine

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}

    - name: Build
      run: |
        cd tools/pythonpkg
        python setup.py sdist
        mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball
        export DISTUTILS_C_COMPILER_LAUNCHER=ccache
        # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
        cibuildwheel --output-dir wheelhouse --config-file cibw.toml duckdb_tarball

   # All extensions uploaded by this job will be loaded by the test_extensions.py test during the linux-python3 job.
   centos7-extensions:
    name: CentOS 7 Extensions
    runs-on: ubuntu-latest
    container: centos:7
    needs: linux-python3-9
    strategy:
      matrix:
        arch: ['i686', 'x86_64']
    env:
      DISABLE_MAIN_DUCKDB_LIBRARY: 1
      BUILD_HTTPFS: 1
      BUILD_VISUALIZER: 1
      BUILD_ICU: 1
      BUILD_TPCH: 1
      BUILD_TPCDS: 1
      BUILD_FTS: 1
      BUILD_JSON: 1
      BUILD_EXCEL: 1
      BUILD_SUBSTRAIT_EXTENSION: 1
      STATIC_OPENSSL: 1
      TREAT_WARNINGS_AS_ERRORS: 1
      FORCE_WARN_UNUSED: 1
      DISABLE_BUILTIN_EXTENSIONS: 1
      FORCE_32_BIT: ${{ matrix.arch == 'i686' && 1 || 0}}
      AWS_ACCESS_KEY_ID: ${{secrets.S3_ID}}
      AWS_SECRET_ACCESS_KEY: ${{secrets.S3_KEY}}
      AWS_DEFAULT_REGION: us-east-1

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: Install dependencies
      run: |
        yum install -y gcc gcc-c++ git cmake make
        yum install -y epel-release
        yum install -y make gcc perl-core pcre-devel wget zlib-devel python3

    - name: Install AWS CLI
      run: |
        python3 -m pip install awscli
        aws --version

    - name: Install i686 dependencies
      if: ${{matrix.arch == 'i686'}}
      run: |
        yum install -y glibc-devel.i686 libgcc.i686 libstdc++-devel.i686 ncurses-devel.i686

      # the weird openssl findreplace fix with version numbers is from: https://github.com/h2o/h2o/issues/213
    - name: Download OpenSSL 1.1.1k
      run: |
        wget https://ftp.openssl.org/source/openssl-1.1.1k.tar.gz --no-check-certificate
        tar -xzvf openssl-1.1.1k.tar.gz
        cd openssl-1.1.1k
        find ./ -type f -exec sed -i -e 's/\#\ define\ OPENSSL\_VERSION\_NUMBER/\#define\ OPENSSL\_VERSION\_NUMBER/g' {} \;

    - name: Configure OpenSSL x86_64
      if: ${{matrix.arch == 'x86_64'}}
      run: |
        cd openssl-1.1.1k
        ./config --prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared zlib-dynamic

    - name: Configure OpenSSL i686
      if: ${{matrix.arch == 'i686'}}
      run: |
        cd openssl-1.1.1k
        ./Configure --prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared zlib-dynamic -m32 linux-generic32

    - name: Build OpenSSL
      run: |
        cd openssl-1.1.1k
        make
        make install

    - name: Version Check
      run: |
        cmake --version
        ldd --version ldd

    - name: Build
      run: make

    - uses: actions/upload-artifact@v2
      with:
        name: python-wheel-centos7-extensions-centos-${{ matrix.arch }}
        path: |
          build/release/extension/*/*.duckdb_extension

    - name: Deploy
      run: |
        if [[ "$GITHUB_REF" =~ ^(refs/heads/master|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
          ./scripts/extension-upload.sh linux_${{matrix.arch == 'i686' && 'i686' || 'amd64'}}_gcc4
          ./scripts/extension-upload-test.sh
        else
          ./scripts/extension-upload-test.sh local
        fi

   linux-python3:
    name: Python 3 Linux
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        arch: [i686, x86_64]
        python_build: [cp37-*, cp38-*, cp39-*,cp310-*]
        isRelease:
          - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master' }}
        exclude:
          - isRelease: false
            python_build: 'cp38-*'
          - isRelease: false
            python_build: 'cp39-*'
    needs: centos7-extensions
    env:
      CIBW_BUILD: ${{ matrix.python_build}}
      CIBW_ARCHS: ${{ matrix.arch == 'i686' && 'auto32' || 'auto64'}}
      SETUPTOOLS_SCM_NO_LOCAL: 'yes'
      TWINE_USERNAME: 'hfmuehleisen'
      PYTEST_TIMEOUT: '600'
      DUCKDB_BUILD_UNITY: 1

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - uses: actions/setup-python@v2
      with:
        python-version: '3.7'

    - name: Install
      run: pip install cibuildwheel twine

    - uses: actions/download-artifact@v3
      with:
        name: python-wheel-centos7-extensions-centos-${{ matrix.arch }}
        path: tools/pythonpkg

    - name: List extensions to be tested
      run: |
        find tools/pythonpkg -maxdepth 1 -type f -name "*.duckdb_extension"

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}-${{ matrix.arch }}-${{ matrix.python_build }}

    - name: Build
      run: |
        cd tools/pythonpkg
        python setup.py sdist
        mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball
        ls duckdb_tarball
        export DISTUTILS_C_COMPILER_LAUNCHER=ccache
        # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
        cibuildwheel --output-dir wheelhouse --config-file cibw.toml duckdb_tarball
        ls wheelhouse

    - name: Deploy
      run: |
        python scripts/asset-upload-gha.py duckdb_python_src.tar.gz=tools/pythonpkg/dist/duckdb-*.tar.gz
        if [[ "$GITHUB_REF" =~ ^(refs/heads/master|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
          twine upload --non-interactive --disable-progress-bar --skip-existing tools/pythonpkg/wheelhouse/*.whl tools/pythonpkg/dist/duckdb-*.tar.gz
        fi

   osx-python3:
      if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master'
      name: Python 3 OSX
      runs-on: macos-latest
      strategy:
       matrix:
        python_build: [cp36-*, cp37-*, cp38-*, cp39-*, cp310-*]
      needs: linux-python3-9
      env:
        CIBW_BUILD: ${{ matrix.python_build}}
        CIBW_ARCHS: 'x86_64 universal2 arm64'
        CIBW_TEST_COMMAND: 'python -m pytest {project}/tests/fast'
        SETUPTOOLS_SCM_NO_LOCAL: 'yes'
        TWINE_USERNAME: 'hfmuehleisen'
        DUCKDB_BUILD_UNITY: 1

      steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v2
        with:
          python-version: '3.7'

      - name: Install
        run: pip install cibuildwheel twine

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}-${{ matrix.python_build }}

      - name: Build
        run: |
          cd tools/pythonpkg
          python setup.py sdist
          mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball
          export DISTUTILS_C_COMPILER_LAUNCHER=ccache
          # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
          cibuildwheel --output-dir wheelhouse --config-file cibw.toml duckdb_tarball

      - name: Deploy
        run: |
          if [[ "$GITHUB_REF" =~ ^(refs/heads/master|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
            twine upload --non-interactive --disable-progress-bar --skip-existing tools/pythonpkg/wheelhouse/*.whl
          fi

   win-python3:
      name: Python 3 Windows
      runs-on: windows-latest
      strategy:
       matrix:
        python_build: [cp36-*, cp37-*, cp38-*, cp39-*, cp310-*]
        isRelease:
          - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master' }}
        exclude:
          - isRelease: false
            python_build: 'cp37-*'
          - isRelease: false
            python_build: 'cp38-*'
          - isRelease: false
            python_build: 'cp39-*'
      needs: linux-python3-9

      env:
        CIBW_BUILD: ${{ matrix.python_build}}
        CIBW_BEFORE_BUILD: 'pip install --prefer-binary "pandas>=0.24" "pytest>=4.3" "psutil>=5.9.0"'
        CIBW_TEST_REQUIRES: 'pytest'
        CIBW_BEFORE_TEST: 'pip install --prefer-binary "pandas>=0.24" "pytest>=4.3" mypy "psutil>=5.9.0"'
        CIBW_TEST_COMMAND: 'python -m pytest {project}/tests/fast'
        SETUPTOOLS_SCM_NO_LOCAL: 'yes'
        SETUPTOOLS_USE_DISTUTILS: 'stdlib'
        TWINE_USERNAME: 'hfmuehleisen'
        DUCKDB_BUILD_UNITY: 1

      steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v2
        with:
          python-version: '3.7'

      - name: Install
        run: pip install cibuildwheel twine

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}-${{ matrix.python_build }}

      - name: Build
        run: |
          cd tools/pythonpkg
          python setup.py sdist
          mkdir duckdb_tarball && tar xvf dist/duckdb-*.tar.gz --strip-components=1 -C duckdb_tarball
          export DISTUTILS_C_COMPILER_LAUNCHER=ccache
          # TODO: Use ccache inside container, see https://github.com/pypa/cibuildwheel/issues/1030
          cibuildwheel --output-dir wheelhouse duckdb_tarball

      - name: Deploy
        run: |
          if [[ "$GITHUB_REF" =~ ^(refs/heads/master|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
            twine upload --non-interactive --disable-progress-bar --skip-existing tools/pythonpkg/wheelhouse/*.whl
          fi

   linux-tarball:
      name: Python 3 Tarball
      runs-on: ubuntu-20.04
      needs: linux-python3-9

      steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v2
        with:
          python-version: '3.7'

      - name: Install
        run: pip install numpy pytest pandas mypy psutil

      - name: Setup Ccache
        uses: hendrikmuhs/ccache-action@main
        with:
          key: ${{ github.job }}

      - name: Build
        run: |
          python --version
          git archive --format zip --output test-tarball.zip HEAD
          mkdir duckdb-test-tarball
          mv test-tarball.zip duckdb-test-tarball
          cd duckdb-test-tarball
          unzip test-tarball.zip
          cd tools/pythonpkg
          export SETUPTOOLS_SCM_PRETEND_VERSION=0.2.2
          export DISTUTILS_C_COMPILER_LAUNCHER=ccache
          python setup.py install --user
          (cd tests/ && python -m pytest)

   linux-release-cleanup:
      if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master'
      name: PyPi Release Cleanup
      runs-on: ubuntu-20.04
      needs: linux-python3-9
      env:
        PYPI_PASSWORD: ${{secrets.PYPI_PASSWORD}}

      steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v2
        with:
          python-version: '3.7'

      - name: Cleanup Releases
        run: python3 scripts/pypi_cleanup.py