diff --git a/.github/actions/build-gk/action.yaml b/.github/actions/build-gk/action.yaml index 731a5f87..d6656e5c 100644 --- a/.github/actions/build-gk/action.yaml +++ b/.github/actions/build-gk/action.yaml @@ -24,23 +24,26 @@ runs: id: restore_gk_pkg uses: actions/cache/restore@v4 with: - key: mamba-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} + key: gk-tarballs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} path: | ~/conda-bld + # skip remaining steps (except configure-conda) on cache hit + - name: install mamba + if: ${{ steps.restore_gk_pkg.outputs.cache-hit != 'true' }} uses: ./.github/actions/install-mamba with: installer-url: https://micro.mamba.pm/api/micromamba/${{ inputs.platform }}/1.5.8 - name: create build env + if: ${{ steps.restore_gk_pkg.outputs.cache-hit != 'true' }} shell: bash -l -e {0} run: | set -x + # boa required for the mambabuild subcommand if [ ! -d "${HOME}/micromamba/envs/build" ]; then - micromamba create -yqn build -c conda-forge \ - boa==0.17.0 mamba==1.5.8 conda=24.5 \ - ccache + micromamba create -yqn build -c conda-forge boa==0.17.0 mamba==1.5.8 conda=24.5 ccache fi set +x @@ -50,6 +53,7 @@ runs: env: build - name: save mamba gk ccache + if: ${{ steps.restore_gk_pkg.outputs.cache-hit != 'true' }} id: save_mamba_gk_ccache uses: actions/cache@v4 with: @@ -61,6 +65,7 @@ runs: ~/.ccache - name: set compiler cache size + if: ${{ steps.restore_gk_pkg.outputs.cache-hit != 'true' }} shell: bash -l -e {0} run: | set -x @@ -81,6 +86,7 @@ runs: conda clean -it - name: cache mamba env + if: ${{ steps.restore_gk_pkg.outputs.cache-hit != 'true' }} id: save_mamba_cache uses: actions/cache/save@v4 with: @@ -91,9 +97,10 @@ runs: ~/bin/micromamba - name: cache gk package tarballs + if: ${{ steps.restore_gk_pkg.outputs.cache-hit != 'true' }} id: save_gk_pkg uses: actions/cache/save@v4 with: - key: mamba-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} + key: gk-tarballs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} path: | ~/conda-bld diff --git a/.github/actions/curl-meta-yaml/action.yaml b/.github/actions/curl-meta-yaml/action.yaml index 16bda7ab..1f40e81d 100644 --- a/.github/actions/curl-meta-yaml/action.yaml +++ b/.github/actions/curl-meta-yaml/action.yaml @@ -5,21 +5,41 @@ runs: steps: - shell: bash -l -e {0} run: | + # Fail if on main branch and conda-recipe/meta.yaml exists + if [[ "${GITHUB_REF_NAME}" == "main" && -f "conda-recipe/meta.yaml" ]]; then + echo "ERROR: conda-recipe/meta.yaml should not be committed to main. Failing build." + exit 1 + fi + set -x mkdir -p conda-recipe - curl -s -L -o conda-recipe/meta.yaml https://raw.githubusercontent.com/conda-forge/genomekit-feedstock/main/recipe/meta.yaml + + # If meta.yaml exists, use it (to allow quick iteration in feature branches) + if [[ -f "conda-recipe/meta.yaml" ]]; then + echo "WARN: Using existing conda-recipe/meta.yaml" + else + curl -s -L -o conda-recipe/meta.yaml https://raw.githubusercontent.com/conda-forge/genomekit-feedstock/main/recipe/meta.yaml + fi + export GK_VERSION=$(grep "version = " setup.py | awk -F'"' '{print $2}') export OS_TYPE=$(uname) - # - replace the (non-existent) tarball with local path - # - remove the related sha for that tarball - # - build for all locally allowed versions of python (conda-recipe/conda_build_config.yaml) - # - set the version to the local release-please version (for docs and docker publish) if [[ "$OS_TYPE" == "Darwin" ]]; then - sed -i '' -E "s|url: https://github.com/deepgenomics/GenomeKit/archive/refs/tags/v.*$|path: ../|1; /sha256: /d; /skip: true /d;s/{% set version = \".+\" %}/{% set version = \"${GK_VERSION}\" %}/" conda-recipe/meta.yaml + SED_CMD="sed -i '' -E" else - sed -i -e "s|url: https://github.com/deepgenomics/GenomeKit/archive/refs/tags/v.*$|path: ../|1; /sha256: /d; /skip: true /d;s/{% set version = \".\+\" %}/{% set version = \"${GK_VERSION}\" %}/" conda-recipe/meta.yaml + SED_CMD="sed -i" fi + # replace the (non-existent) tarball with local path + $SED_CMD "s|url: https://github.com/deepgenomics/GenomeKit/archive/refs/tags/v.*$|path: ../|1" conda-recipe/meta.yaml + # remove the related sha for that tarball + $SED_CMD "/sha256: /d" conda-recipe/meta.yaml + # build for all locally allowed versions of python (conda-recipe/conda_build_config.yaml) + $SED_CMD "/skip: true /d" conda-recipe/meta.yaml + # set the version to the local release-please version (for docs and docker publish) + $SED_CMD "s/{% set version = \".+\" %}/{% set version = \"${GK_VERSION}\" %}/" conda-recipe/meta.yaml + # avoid error "No module named 'setup'" due to "from setup import c_ext" in setup.py + # See if this can be removed + # $SED_CMD "s/script: {{ PYTHON }} -m pip install --no-deps --ignore-installed ./script: PYTHONPATH=. {{ PYTHON }} -m pip install --no-deps --ignore-installed ./" conda-recipe/meta.yaml - head -10 conda-recipe/meta.yaml + head -40 conda-recipe/meta.yaml set +x diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml index e849a468..d54db929 100644 --- a/.github/workflows/build-wheels.yaml +++ b/.github/workflows/build-wheels.yaml @@ -5,12 +5,14 @@ on: push: # only build+push docker on release-please tags tags: [ "v*" ] + branches: + - feat/py313 paths: - ".github/actions/**" - ".github/workflows/**" - "genome_kit/**" - "setup.py" - - "_pyproject.toml" + - "pyproject.toml" - "setup/**" - "src/**" - "tests/**" @@ -33,20 +35,16 @@ jobs: id: restore_linux_wheels uses: actions/cache/restore@v4 with: - key: linux-wheels-${{ matrix.arch }}-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} + key: linux-wheels-${{ matrix.arch }}-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} path: wheelhouse/*.whl # skip remaining steps on cache hit - - if: ${{ steps.restore_linux_wheels.outputs.cache-hit != 'true' }} - name: Rename pyproject.toml - run: mv _pyproject.toml pyproject.toml - - if: ${{ steps.restore_linux_wheels.outputs.cache-hit != 'true' }} name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - if: ${{ steps.restore_linux_wheels.outputs.cache-hit != 'true' }} name: Install cibuildwheel @@ -57,7 +55,7 @@ jobs: run: python -m cibuildwheel --output-dir wheelhouse && ls -l wheelhouse/ env: GK_BUILD_WHEELS: "1" - CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*" + CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-*" CIBW_ARCHS_LINUX: ${{ matrix.arch }} CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 @@ -65,11 +63,11 @@ jobs: CIBW_SKIP: "*-musllinux_*" - if: ${{ steps.restore_linux_wheels.outputs.cache-hit != 'true' }} - name: Test wheels across Python 3.9–3.12 + name: Test wheels across Python 3.9–3.13 run: | set -euxo pipefail - for PYVER in 3.9 3.10 3.11 3.12; do + for PYVER in 3.9 3.10 3.11 3.12 3.13; do PYVER_SHORT=${PYVER/./} echo "Testing on Python $PYVER" @@ -106,20 +104,16 @@ jobs: id: restore_macos_wheels uses: actions/cache/restore@v4 with: - key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} + key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} path: wheelhouse/*.whl # skip remaining steps on cache hit - - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} - name: Rename pyproject.toml - run: mv _pyproject.toml pyproject.toml - - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} name: Install cibuildwheel @@ -131,7 +125,7 @@ jobs: env: GK_BUILD_WHEELS: "1" CIBW_ARCHS_MACOS: x86_64 arm64 - CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*" + CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-macosx_arm64" - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} name: cache macos wheels @@ -154,6 +148,7 @@ jobs: - {"os": "macos-14", "arch": "arm64", "pyver": "3.10", "pyvershort": "310"} - {"os": "macos-14", "arch": "arm64", "pyver": "3.11", "pyvershort": "311"} - {"os": "macos-14", "arch": "arm64", "pyver": "3.12", "pyvershort": "312"} + - {"os": "macos-14", "arch": "arm64", "pyver": "3.13", "pyvershort": "313"} - {"os": "macos-13", "arch": "x86_64", "pyver": "3.9", "pyvershort": "39"} - {"os": "macos-13", "arch": "x86_64", "pyver": "3.10", "pyvershort": "310"} - {"os": "macos-13", "arch": "x86_64", "pyver": "3.11", "pyvershort": "311"} @@ -170,7 +165,7 @@ jobs: - name: Download built wheels uses: actions/cache/restore@v4 with: - key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} + key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} path: wheelhouse/*.whl fail-on-cache-miss: true @@ -212,7 +207,7 @@ jobs: # id: restore_linux_intel_wheels # uses: actions/cache/restore@v4 # with: -# key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} +# key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -220,7 +215,7 @@ jobs: # id: restore_linux_arm_wheels # uses: actions/cache/restore@v4 # with: -# key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} +# key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -228,7 +223,7 @@ jobs: # id: restore_macos_wheels # uses: actions/cache/restore@v4 # with: -# key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} +# key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -246,48 +241,48 @@ jobs: # verbose: 'true' # skip-existing: 'true' - publish-to-pypi: - name: Publish to PyPI - needs: [build-wheel-linux, test-wheel-macos] - runs-on: ubuntu-latest - - environment: - name: pypi - url: https://pypi.org/p/genomekit - - permissions: - id-token: write # mandatory for trusted publishing - - steps: - - uses: actions/checkout@v4 - - - name: Restore the linux intel wheels cache - id: restore_linux_intel_wheels - uses: actions/cache/restore@v4 - with: - key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} - path: wheelhouse/*.whl - fail-on-cache-miss: true - - - name: Restore the linux arm wheels cache - id: restore_linux_arm_wheels - uses: actions/cache/restore@v4 - with: - key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} - path: wheelhouse/*.whl - fail-on-cache-miss: true - - - name: Restore the macos wheels cache - id: restore_macos_wheels - uses: actions/cache/restore@v4 - with: - key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} - path: wheelhouse/*.whl - fail-on-cache-miss: true - - - name: Publish distribution to PyPI - # don't use a version tag with 3rd party actions - uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc - with: - packages-dir: wheelhouse - verbose: 'true' +# publish-to-pypi: +# name: Publish to PyPI +# needs: [build-wheel-linux, test-wheel-macos] +# runs-on: ubuntu-latest +# +# environment: +# name: pypi +# url: https://pypi.org/p/genomekit +# +# permissions: +# id-token: write # mandatory for trusted publishing +# +# steps: +# - uses: actions/checkout@v4 +# +# - name: Restore the linux intel wheels cache +# id: restore_linux_intel_wheels +# uses: actions/cache/restore@v4 +# with: +# key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} +# path: wheelhouse/*.whl +# fail-on-cache-miss: true +# +# - name: Restore the linux arm wheels cache +# id: restore_linux_arm_wheels +# uses: actions/cache/restore@v4 +# with: +# key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} +# path: wheelhouse/*.whl +# fail-on-cache-miss: true +# +# - name: Restore the macos wheels cache +# id: restore_macos_wheels +# uses: actions/cache/restore@v4 +# with: +# key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} +# path: wheelhouse/*.whl +# fail-on-cache-miss: true +# +# - name: Publish distribution to PyPI +# # don't use a version tag with 3rd party actions +# uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc +# with: +# packages-dir: wheelhouse +# verbose: 'true' diff --git a/.github/workflows/dockerize.yaml b/.github/workflows/dockerize.yaml index 55780aec..0f4aa8f1 100644 --- a/.github/workflows/dockerize.yaml +++ b/.github/workflows/dockerize.yaml @@ -39,7 +39,7 @@ jobs: id: restore_gk_pkg uses: actions/cache/restore@v4 with: - key: mamba-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} + key: gk-tarballs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} path: | ~/conda-bld fail-on-cache-miss: true diff --git a/.github/workflows/publish-docs.yaml b/.github/workflows/publish-docs.yaml index 5999bec2..435a66f0 100644 --- a/.github/workflows/publish-docs.yaml +++ b/.github/workflows/publish-docs.yaml @@ -40,7 +40,7 @@ jobs: id: restore_gk_pkg uses: actions/cache/restore@v4 with: - key: mamba-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} + key: gk-tarballs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} path: | ~/conda-bld fail-on-cache-miss: true diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index e8319d9a..871e24ee 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -39,6 +39,8 @@ jobs: - {"pyver-short": "311", "python-version": "3.11", "platform": "osx-64", "runs-on": "macos-latest"} - {"pyver-short": "312", "python-version": "3.12", "platform": "linux-64", "runs-on": "ubuntu-latest"} - {"pyver-short": "312", "python-version": "3.12", "platform": "osx-64", "runs-on": "macos-latest"} + - {"pyver-short": "313", "python-version": "3.13", "platform": "linux-64", "runs-on": "ubuntu-latest"} + - {"pyver-short": "313", "python-version": "3.13", "platform": "osx-64", "runs-on": "macos-latest"} runs-on: ${{ matrix.runs-on }} steps: - uses: actions/checkout@v4 @@ -61,7 +63,7 @@ jobs: id: restore_gk_pkg uses: actions/cache/restore@v4 with: - key: mamba-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} + key: gk-tarballs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} path: | ~/conda-bld fail-on-cache-miss: true @@ -74,10 +76,12 @@ jobs: - name: create test env shell: bash -l -e {0} run: | + # boa required for the mambabuild subcommand set -x + # inconsistently got "coverage 7.9.1 is not supported on this platform" so removed + # (not needed?) if [ ! -d "${HOME}/micromamba/envs/test" ]; then - micromamba create -yqn test -c conda-forge \ - boa==0.16.0 mamba==1.5.7 conda==24.1.2 coverage + micromamba create -yqn test -c conda-forge boa==0.16.0 mamba==1.5.7 conda==24.1.2 fi set +x diff --git a/_pyproject.toml b/_pyproject.toml deleted file mode 100644 index 5a1436f4..00000000 --- a/_pyproject.toml +++ /dev/null @@ -1,5 +0,0 @@ -# Specify build system requirements for pip wheel builds. -# This way is incompatible with conda, which is why the file can't have the name pyproject.toml. -# The file is renamed to pyproject.toml in Github Actions before starting the pip wheel build. -[build-system] -requires = ["setuptools>=42", "wheel", "numpy==2"] diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index 86572dcb..f35afa4d 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -11,3 +11,4 @@ python: - '3.10' - '3.11' - '3.12' +- '3.13' diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml new file mode 100644 index 00000000..905528b5 --- /dev/null +++ b/conda-recipe/meta.yaml @@ -0,0 +1,69 @@ +{% set version = "7.1.0" %} + +package: + name: genomekit + version: {{ version }} + +source: + url: https://github.com/deepgenomics/GenomeKit/archive/refs/tags/v{{ version }}.tar.gz + sha256: a124ef26234b42da3db8f83d676949b54995b0d2965e4eed22ede3ad8c73506a + +build: + number: 0 + script: {{ PYTHON }} -m pip install --no-deps --ignore-installed . + skip: true # [py>313 or py<39] + +requirements: + build: + - {{ compiler('cxx') }} + - {{ stdlib("c") }} + - python # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + - numpy # [build_platform != target_platform] + - setuptools # [build_platform != target_platform] + + host: + - appdirs + # don't know why, but if not specified, conda selects numpy v1.26, incompatible with py313 + - numpy 2 # [py>=313] + - numpy # [py<313] + - typing-extensions + - zlib + - python {{ python }} + - pip + - setuptools + + run: + - appdirs + # google-cloud-storage breaks the build on py313 + - google-cloud-storage # [py<313] + - boto3 + - typing-extensions + - importlib-metadata + - tqdm + - python {{ python }} + +test: + imports: + - genome_kit + requires: + # inconsistently got "coverage 7.9.1 is not supported on this platform" so removed + # (not needed?) + - pip + source_files: + - tests + commands: + - pip check + - python -m unittest discover + +about: + doc_url: https://deepgenomics.github.io/GenomeKit/ + home: https://github.com/deepgenomics/GenomeKit + license: "Apache-2.0" + license_file: "LICENSE" + summary: GenomeKit is a Python library for fast and easy access to genomic resources such as sequence, data tracks, and annotations. + +extra: + recipe-maintainers: + - s22chan + - ovesh diff --git a/docs-src/develop.rst b/docs-src/develop.rst index 7581ce00..3ac8f55b 100644 --- a/docs-src/develop.rst +++ b/docs-src/develop.rst @@ -29,7 +29,7 @@ On M1 macs, you might need to set up the environment differently:: Build the package in development mode:: - pip install -e . + PYTHONPATH=. pip install -e . --no-build-isolation This builds the C++ extension and copies it into your source tree (``genome_kit/_cxx.so``). @@ -65,7 +65,7 @@ Making changes If the C/C++ code changed, you must re-run the ``develop`` command:: - pip install -e . + PYTHONPATH=. pip install -e . --no-build-isolation This includes switching branches, merging changes, or editing the C/C++ code yourself. *Forgetting this step may lead to unpredictable behaviour.* @@ -161,7 +161,7 @@ Test data files reside in the source tree under ``tests/data``. To build them, you must have registered your source tree in develop mode:: - pip install -e . + PYTHONPATH=. pip install -e . --no-build-isolation Now that your source tree is the default `genome_kit` import, the ``build`` subcommand will be able to find diff --git a/genome_kit/data_manager.py b/genome_kit/data_manager.py index 679ac30d..96b62279 100644 --- a/genome_kit/data_manager.py +++ b/genome_kit/data_manager.py @@ -11,7 +11,12 @@ from pathlib import Path from typing import Dict, Any -from google.cloud import storage +_SUPPORT_GCS = True +try: + from google.cloud import storage +except ImportError: + # google.cloud breaks build on py313 + _SUPPORT_GCS = False import boto3 from botocore import UNSIGNED from botocore.config import Config @@ -167,109 +172,114 @@ def FileIO(filename, desc, mode, size, quiet): ) yield decorated -class GCSDataManager(DataManager): - """A minimal data manager implementation that retrieves files from a GCS bucket.""" - def __init__(self, data_dir: str, bucket_name: str): - """ - Args: - data_dir: location where local files are cached - bucket_name: GCS bucket - """ - self._bucket_name = bucket_name - super().__init__(data_dir) +if _SUPPORT_GCS is False: + class GCSDataManager(DataManager): + def __init__(self, data_dir: str, bucket_name: str): + raise NotImplementedError("GCS support not available on py313") +else: + class GCSDataManager(DataManager): + """A minimal data manager implementation that retrieves files from a GCS bucket.""" + def __init__(self, data_dir: str, bucket_name: str): + """ + Args: + data_dir: location where local files are cached + bucket_name: GCS bucket + """ + self._bucket_name = bucket_name + super().__init__(data_dir) + + @property + def bucket(self): + if not hasattr(self, "_bucket"): + gcloud_client = storage.Client() + try: + self._bucket = gcloud_client.bucket(self._bucket_name, user_project=os.environ.get("GENOMEKIT_GCS_BILLING_PROJECT", None)) + except Exception as e: + # give the user a hint in case of permission errors + print(e, file=sys.stderr) + raise + + return self._bucket + + def get_file(self, filename: str) -> str: + local_path = Path(self.data_dir, filename) + + if local_path.exists(): + return str(local_path) - @property - def bucket(self): - if not hasattr(self, "_bucket"): - gcloud_client = storage.Client() try: - self._bucket = gcloud_client.bucket(self._bucket_name, user_project=os.environ.get("GENOMEKIT_GCS_BILLING_PROJECT", None)) + blob = self.bucket.blob(filename) + if not blob.exists(): + raise FileNotFoundError(f"File '{filename}' not found in the GCS bucket") except Exception as e: - # give the user a hint in case of permission errors - print(e, file=sys.stderr) + if "GENOMEKIT_TRACE" in os.environ: + # give the user a hint in case of permission errors + print(e, file=sys.stderr) raise - return self._bucket - - def get_file(self, filename: str) -> str: - local_path = Path(self.data_dir, filename) + # form a temporary filename to make the download safe + temp_file = tempfile.NamedTemporaryFile(delete=False, mode="wb", dir=self.data_dir, prefix=filename, suffix=".part") + temp_file_path = str(Path(self.data_dir, temp_file.name)) + try: + temp_file.close() + with FileIO(temp_file_path, filename, "wb", blob.size, quiet=False) as f: + blob.download_to_file(f) + except: + os.remove(temp_file_path) + raise + # atomically (on POSIX) rename the file to the real one + os.rename(temp_file_path, local_path) - if local_path.exists(): return str(local_path) - try: - blob = self.bucket.blob(filename) - if not blob.exists(): - raise FileNotFoundError(f"File '{filename}' not found in the GCS bucket") - except Exception as e: - if "GENOMEKIT_TRACE" in os.environ: - # give the user a hint in case of permission errors - print(e, file=sys.stderr) - raise + def _remote_equal(self, blob: storage.Blob, file_path: Path) -> bool: + remote_checksum = base64.b64decode(blob.md5_hash).hex() + with open(file_path, 'rb') as f: + local_checksum = _hashfile(f, hashlib.md5()) + return remote_checksum == local_checksum - # form a temporary filename to make the download safe - temp_file = tempfile.NamedTemporaryFile(delete=False, mode="wb", dir=self.data_dir, prefix=filename, suffix=".part") - temp_file_path = str(Path(self.data_dir, temp_file.name)) - try: - temp_file.close() - with FileIO(temp_file_path, filename, "wb", blob.size, quiet=False) as f: - blob.download_to_file(f) - except: - os.remove(temp_file_path) - raise - # atomically (on POSIX) rename the file to the real one - os.rename(temp_file_path, local_path) - - return str(local_path) - - def _remote_equal(self, blob: storage.Blob, file_path: Path) -> bool: - remote_checksum = base64.b64decode(blob.md5_hash).hex() - with open(file_path, 'rb') as f: - local_checksum = _hashfile(f, hashlib.md5()) - return remote_checksum == local_checksum - - def upload_file(self, filepath: str, filename: str, metadata: Dict[str, str]=None): - blob = self.bucket.blob(filename) - - if blob.exists(): - blob.reload() - if self._remote_equal(blob, Path(filepath)): - logger.info(f"File '{filename}' already exists in the GCS bucket and is identical, skipping.") - return + def upload_file(self, filepath: str, filename: str, metadata: Dict[str, str]=None): + blob = self.bucket.blob(filename) - if "GK_UPLOAD_ALLOW_OVERWRITE" not in os.environ: - raise ValueError(f"File '{filename}' already exists in the GCS bucket." - "Set GK_UPLOAD_ALLOW_OVERWRITE=1 to overwrite.") + if blob.exists(): + blob.reload() + if self._remote_equal(blob, Path(filepath)): + logger.info(f"File '{filename}' already exists in the GCS bucket and is identical, skipping.") + return + + if "GK_UPLOAD_ALLOW_OVERWRITE" not in os.environ: + raise ValueError(f"File '{filename}' already exists in the GCS bucket." + "Set GK_UPLOAD_ALLOW_OVERWRITE=1 to overwrite.") + else: + logger.warning(f"Overwriting '{filename}' on the server.") + + if blob.metadata is None: + blob.metadata = metadata + else: + blob.metadata.update(metadata or {}) else: - logger.warning(f"Overwriting '{filename}' on the server.") - - if blob.metadata is None: blob.metadata = metadata - else: - blob.metadata.update(metadata or {}) - else: - blob.metadata = metadata - with FileIO(filepath, filename, "rb", os.path.getsize(filepath), quiet=False) as f: - blob.upload_from_file(f) + with FileIO(filepath, filename, "rb", os.path.getsize(filepath), quiet=False) as f: + blob.upload_from_file(f) - @lru_cache - def list_available_genomes(self): - names = set() + @lru_cache + def list_available_genomes(self): + names = set() - def get_genomes(filenames): - return ( - name.rpartition(".")[0] - for name in filenames - if name.endswith(".2bit") or name.endswith(".cfg") - ) + def get_genomes(filenames): + return ( + name.rpartition(".")[0] + for name in filenames + if name.endswith(".2bit") or name.endswith(".cfg") + ) - names.update(get_genomes(os.listdir(self.data_dir))) + names.update(get_genomes(os.listdir(self.data_dir))) - blobs = self.bucket.list_blobs() - names.update(get_genomes(blob.name for blob in blobs)) + blobs = self.bucket.list_blobs() + names.update(get_genomes(blob.name for blob in blobs)) - return sorted(names) + return sorted(names) class DefaultDataManager(DataManager): """A minimal data manager implementation that retrieves files from a S3 bucket. diff --git a/genome_kit/vcf_table.py b/genome_kit/vcf_table.py index b8afb4bd..ac590361 100644 --- a/genome_kit/vcf_table.py +++ b/genome_kit/vcf_table.py @@ -5,7 +5,7 @@ import os import subprocess import tempfile -from distutils.spawn import find_executable +import shutil from typing import Optional, Union from typing_extensions import Literal @@ -26,7 +26,7 @@ def _dump_fasta(genome, chrom_mapper): """Dumps a FASTA containing the DNA sequence of refg, indexes it, and returns a context that can be treated as the path to the fasta. The temporaries are deleted once the context is closed.""" - samtools = find_executable("samtools") + samtools = shutil.which("samtools") if not samtools: raise RuntimeError("samtools must be installed in order to normalize VCFs") # pragma: no cover @@ -216,7 +216,7 @@ def open_clinvar(vcfpath): else: if normalize: # Check requirements - bcftools = find_executable("bcftools") + bcftools = shutil.which("bcftools") if not bcftools: raise RuntimeError("bcftools must be installed to normalize VCFs") # pragma: no cover diff --git a/genomekit_dev.yml b/genomekit_dev.yml index 5ba2f85a..41708cb7 100644 --- a/genomekit_dev.yml +++ b/genomekit_dev.yml @@ -4,8 +4,8 @@ channels: - bioconda dependencies: - appdirs>=1.4.0 - - numpy <2.0dev0 - - python>=3.9.0,<3.13.0 + - numpy + - python>=3.9.0,<3.14.0 - tqdm - typing-extensions - zlib @@ -16,7 +16,7 @@ dependencies: # dev + test dependencies - ccache - ipython - - pyperformance==1.0.4 + # usage of pyperf was removed internally before open sourcing - pytest - sphinx - sphinx_rtd_theme diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..6069e17c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = [ + "setuptools>=61", + "wheel", + "numpy==2" +] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/setup.py b/setup.py index 4f245fa5..55e93ee3 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,19 @@ # Copyright (C) 2016-2023 Deep Genomics Inc. All Rights Reserved. import os +import platform +import sys +import sysconfig +from glob import glob from pathlib import Path +from traceback import extract_stack +from setuptools._distutils import ccompiler +import shutil +from setuptools import Extension from setuptools import setup, find_packages +from setuptools.command.build_ext import build_ext from setuptools.command.egg_info import egg_info -from setup import c_ext + COPYRIGHT_FILE = "COPYRIGHT.txt" LICENSE_FILE = "LICENSE" @@ -29,12 +38,355 @@ def run(self): egg_info.run(self) + + +libname = "genome_kit" + +############################################################## +# Define the C Extension build config +############################################################## + +debug = False # Set to True for C++ debug symbols and extra memory/bounds checks +debug_info = debug +debug_objects = False # Enable printing of Python C++ instances being constructed/destructed +toolset = "msvc" if platform.system() == "Windows" else "gcc" + +gen_dir = os.path.join('build', 'gen') +include_dirs = [ + sys.prefix + "/include", + gen_dir, + ] +library_dirs = [sys.prefix + '/lib'] +runtime_library_dirs = [] + +if not [x for x in extract_stack(limit=20) if 'load_setup_py_data' in x[2]]: + # don't load numpy if we just need the version to fill the conda-build meta.yaml template + import numpy as np + include_dirs.append(np.get_include()) + +sources = glob("src/*.cpp") +headers = glob("src/*.h") +libraries = [] + +define_macros = [ + ("GKPY_LIBNAME", libname), +] + +if debug_objects: + define_macros += [ + ("GKPY_TRACE_MEM", None), # ctor/dtor notifications from C objects + ] + +extra_compile_args = [] +extra_link_args = [] + +if toolset == "gcc": + # Using multiprocessing and ccache massively speeds up incremental builds + ccache = shutil.which('ccache') + if ccache: + os.environ["CC"] = "{} {}".format(ccache, os.environ.get("CC", "gcc")) + # TODO cannot use ccache in CXX since distutils hotpatches into LDSHARED: + # https://github.com/python/cpython/blob/069306312addf87252e2dbf250fc7632fc8b7da3/Lib/distutils/unixccompiler.py#L191 + os.environ["LDSHARED"] = sysconfig.get_config_var("LDSHARED") + else: + print("WARNING: did not find ccache installed; files will be built from scratch every time") + + define_macros += [ + ("_FILE_OFFSET_BITS", 64), + ] + if debug: + define_macros += [ + ("GK_DEBUG", None), # Enable debug assertions etc + ] + + libraries += [ + # python is not linked for conda's python + # https://github.com/ContinuumIO/anaconda-issues/issues/9078#issuecomment-378321357 + "z", + ] + + # GCC flags common to both debug and release modes + extra_compile_args += [ + "-std=c++20", + "-fvisibility=hidden", # reduce symbols for code size/load times + "-fvisibility-inlines-hidden", + "-Wall", + "-Wno-write-strings", # char* in Python API + "-Wno-invalid-offsetof", # offsetof non-POD GK types for Python API + ] + + if debug: + opt_args = [ + "-O0", + "-UNDEBUG", + ] + else: + opt_args = [ + "-O3", + ] + + if debug_info: + opt_args += [ + "-g3", + ] + else: + extra_link_args += [ + "-Wl,-S", + "-Wl,-x", + ] + + if platform.system() == "Darwin": + # >=10.15 required for std::filesystem::remove + osx_sdk = "-mmacosx-version-min={}".format(os.environ.get("MACOSX_DEPLOYMENT_TARGET", "10.15")) + + extra_compile_args += [ + osx_sdk, + "-isysroot{}".format(os.environ.get("CONDA_BUILD_SYSROOT", "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk")), + "-stdlib=libc++", + "-Wshorten-64-to-32", # catch implicit truncation as per MSVC + "-Wsign-compare", # match MSVC(/W3)/gcc(-Wall) + "-Wconditional-uninitialized", # gcc does better here but enable for safety + "-Wuninitialized", + "-Wno-unknown-warning-option", + ] + + extra_link_args += [ + osx_sdk, + "-isysroot{}".format(os.environ.get("CONDA_BUILD_SYSROOT", "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk")), + ] + define_macros += [ + # https://conda-forge.org/docs/maintainer/knowledge_base/#newer-c-features-with-old-sdk + ("_LIBCPP_DISABLE_AVAILABILITY", None), + ] + + extra_compile_args += opt_args + extra_link_args += opt_args # required for LTO + + +elif toolset == "msvc": + + os.environ["DISTUTILS_USE_SDK"] = "1" + os.environ["MSSdk"] = "1" + + condalib_dir = sys.prefix + "/Library" + condalib_inc = condalib_dir + "/include" + condalib_lib = condalib_dir + "/lib" + include_dirs.append(condalib_inc) + library_dirs.append(condalib_lib) + + define_macros += [ + ("_CRT_SECURE_NO_WARNINGS", None), + ] + + libraries += [ + "zlib", + ] + + # VC flags common to both debug and release modes + extra_compile_args += [ + "/std:c++20", + "/permissive-", + "/Zc:__cplusplus", + "/Zc:strictStrings-", # don"t let strings be written to by default + # compatibility with __VA_OPT__ + # see https://devblogs.microsoft.com/cppblog/announcing-full-support-for-a-c-c-conformant-preprocessor-in-msvc/ + "/Zc:preprocessor", + "/W3", # Warning level 3 + "/EHsc", # Enable C++ and structured exception handling (e.g. catch access violations) + "/wd5033", # Python usage of deprecated register keyword + ] + + extra_link_args += [ + "/PDB:%s\\_cxx.pdb" % libname, # Put it right beside the .pyd/.so file in "develop" mode + ] + + if debug: + # do NOT define _DEBUG; need non-debug runtime to match NDEBUG Python distribution + # define_macros += [ + # ("_DEBUG", None), + # ] + extra_compile_args += [ + "/GS", # Enable buffer overrun checks + "/Zi", # Enable debug information .pdb + "/Od", # Disable optimizations + ] + extra_link_args += [ + "/DEBUG", + ] + else: + extra_compile_args += [ + "/GL", # Enable whole-program optimization + "/Gy", # Enable function-level linking + "/Oy", # Omit frame pointers + "/Oi", # Enable intrinsics + #"/Zi", # Enable debug information .pdb + ] + extra_link_args += [ + "/LTCG", # Enable link-time code generation + #"/DEBUG", + ] + + +class NoCWarningsBuildExt(build_ext): + def build_extensions(self): + for x in ["-Wstrict-prototypes"]: + try: + self.compiler.compiler_so.remove(x) + except (AttributeError, ValueError): + continue + build_ext.build_extensions(self) + + +extension = Extension( + libname + "._cxx", + sources=sources, + depends=headers, + include_dirs=include_dirs, + define_macros=define_macros, + library_dirs=library_dirs, + libraries=libraries, + runtime_library_dirs=runtime_library_dirs, + extra_compile_args=extra_compile_args, + extra_link_args=extra_link_args, + ) + +############################################################## + +# monkey-patch for parallel compilation +# TODO +# taken from http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils + +PARALLEL_JOBS = 4 # number of parallel compilations + + +def gcc_parallel_ccompile(self, + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None): + # those lines are copied from distutils.ccompiler.CCompiler directly + macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + # parallel code + import multiprocessing.pool + + def _single_compile(obj): + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + # convert to list, imap is evaluated on-demand + list(multiprocessing.pool.ThreadPool(PARALLEL_JOBS).imap(_single_compile, objects)) + return objects + + +def windows_parallel_ccompile(self, + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None): + if not self.initialized: + self.initialize() + compile_info = self._setup_compile(output_dir, macros, include_dirs, sources, depends, extra_postargs) + macros, objects, extra_postargs, pp_opts, build = compile_info + + from distutils.errors import CompileError + from distutils.errors import DistutilsExecError + + compile_opts = extra_preargs or [] + compile_opts.append("/c") + compile_opts.extend(self.compile_options_debug if debug else self.compile_options) + + def _compile_obj(obj): + try: + src, ext = build[obj] + except KeyError: + return + if debug: + # pass the full pathname to MSVC in debug mode, + # this allows the debugger to find the source file + # without asking the user to browse for it + src = os.path.abspath(src) + + if ext in self._c_extensions: + input_opt = "/Tc" + src + elif ext in self._cpp_extensions: + input_opt = "/Tp" + src + elif ext in self._rc_extensions: + # compile .RC to .RES file + input_opt = src + output_opt = "/fo" + obj + try: + self.spawn([self.rc] + pp_opts + [output_opt] + [input_opt]) + except DistutilsExecError as msg: + raise CompileError(msg) + return + elif ext in self._mc_extensions: + # Compile .MC to .RC file to .RES file. + # * "-h dir" specifies the directory for the + # generated include file + # * "-r dir" specifies the target directory of the + # generated RC file and the binary message resource + # it includes + # + # For now (since there are no options to change this), + # we use the source-directory for the include file and + # the build directory for the RC file and message + # resources. This works at least for win32all. + h_dir = os.path.dirname(src) + rc_dir = os.path.dirname(obj) + try: + # first compile .MC to .RC and .H file + self.spawn([self.mc] + ["-h", h_dir, "-r", rc_dir] + [src]) + base, _ = os.path.splitext(os.path.basename(src)) + rc_file = os.path.join(rc_dir, base + ".rc") + # then compile .RC to .RES file + self.spawn([self.rc] + ["/fo" + obj] + [rc_file]) + + except DistutilsExecError as msg: + raise CompileError(msg) + return + else: + # how to handle this file? + raise CompileError("Don't know how to compile %s to %s" % (src, obj)) + + output_opt = "/Fo" + obj + try: + self.spawn([self.cc] + compile_opts + pp_opts + [input_opt, output_opt] + extra_postargs) + except DistutilsExecError as msg: + raise CompileError(msg) + + import multiprocessing.pool + list(multiprocessing.pool.ThreadPool(PARALLEL_JOBS).imap(_compile_obj, objects)) + + return objects + + +if sys.platform == "win32": + # TODO from setuptools._distutils._msvccompiler import MSVCCompiler + import distutils._msvccompiler + distutils._msvccompiler.MSVCCompiler.compile = windows_parallel_ccompile +else: + ccompiler.CCompiler.compile = gcc_parallel_ccompile + if __name__ == "__main__": install_requires = [] if os.environ.get("GK_BUILD_WHEELS", None) is not None: install_requires = [ "appdirs>=1.4.0", - "numpy<2.0dev0", + "numpy>=2.0.0; python_version>='3.13'", + "numpy<2.0dev0; python_version<'3.13'", "google-cloud-storage>=2.10.0", "boto3", "tqdm", @@ -53,6 +405,7 @@ def run(self): "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], description="GenomeKit is a Python library for fast and easy access to genomic resources such as sequence, data tracks, and annotations.", long_description=(Path(__file__).parent / "README.md").read_text(), @@ -66,10 +419,10 @@ def run(self): "Documentation": "https://deepgenomics.github.io/GenomeKit" }, cmdclass={ - 'build_ext': c_ext.NoCWarningsBuildExt, + 'build_ext': NoCWarningsBuildExt, 'egg_info': egg_info_ex }, - ext_modules=[c_ext.extension], + ext_modules=[extension], test_suite="tests", tests_require=tests_require, url=f"https://github.com/deepgenomics/GenomeKit", diff --git a/setup/c_ext.py b/setup/c_ext.py deleted file mode 100644 index 4993c6ae..00000000 --- a/setup/c_ext.py +++ /dev/null @@ -1,352 +0,0 @@ -# Copyright (C) 2016-2023 Deep Genomics Inc. All Rights Reserved. - -import os -import platform -import sys -import sysconfig - -import distutils.ccompiler - -from distutils.spawn import find_executable -from glob import glob -from setuptools.command.build_ext import build_ext -from setuptools import Extension -from traceback import extract_stack - -libname = "genome_kit" - -############################################################## -# Define the C Extension build config -############################################################## - -debug = False # Set to True for C++ debug symbols and extra memory/bounds checks -debug_info = debug -debug_objects = False # Enable printing of Python C++ instances being constructed/destructed -toolset = "msvc" if platform.system() == "Windows" else "gcc" - -gen_dir = os.path.join('build', 'gen') -include_dirs = [ - sys.prefix + "/include", - gen_dir, -] -library_dirs = [sys.prefix + '/lib'] -runtime_library_dirs = [] - -if not [x for x in extract_stack(limit=20) if 'load_setup_py_data' in x[2]]: - # don't load numpy if we just need the version to fill the conda-build meta.yaml template - import numpy as np - include_dirs.append(np.get_include()) - -sources = glob("src/*.cpp") -headers = glob("src/*.h") -libraries = [] - -define_macros = [ - ("GKPY_LIBNAME", libname), -] - -if debug_objects: - define_macros += [ - ("GKPY_TRACE_MEM", None), # ctor/dtor notifications from C objects - ] - -extra_compile_args = [] -extra_link_args = [] - -if toolset == "gcc": - # Using multiprocessing and ccache massively speeds up incremental builds - ccache = find_executable('ccache') - if ccache: - os.environ["CC"] = "{} {}".format(ccache, os.environ.get("CC", "gcc")) - # cannot use ccache in CXX since distutils hotpatches into LDSHARED: - # https://github.com/python/cpython/blob/069306312addf87252e2dbf250fc7632fc8b7da3/Lib/distutils/unixccompiler.py#L191 - os.environ["LDSHARED"] = sysconfig.get_config_var("LDSHARED") - else: - print("WARNING: did not find ccache installed; files will be built from scratch every time") - - define_macros += [ - ("_FILE_OFFSET_BITS", 64), - ] - if debug: - define_macros += [ - ("GK_DEBUG", None), # Enable debug assertions etc - ] - - libraries += [ - # python is not linked for conda's python - # https://github.com/ContinuumIO/anaconda-issues/issues/9078#issuecomment-378321357 - "z", - ] - - # GCC flags common to both debug and release modes - extra_compile_args += [ - "-std=c++20", - "-fvisibility=hidden", # reduce symbols for code size/load times - "-fvisibility-inlines-hidden", - "-Wall", - "-Wno-write-strings", # char* in Python API - "-Wno-invalid-offsetof", # offsetof non-POD GK types for Python API - ] - - if debug: - opt_args = [ - "-O0", - "-UNDEBUG", - ] - else: - opt_args = [ - "-O3", - ] - - if debug_info: - opt_args += [ - "-g3", - ] - else: - extra_link_args += [ - "-Wl,-S", - "-Wl,-x", - ] - - if platform.system() == "Darwin": - # >=10.15 required for std::filesystem::remove - osx_sdk = "-mmacosx-version-min={}".format(os.environ.get("MACOSX_DEPLOYMENT_TARGET", "10.15")) - - extra_compile_args += [ - osx_sdk, - "-isysroot{}".format(os.environ.get("CONDA_BUILD_SYSROOT", "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk")), - "-stdlib=libc++", - "-Wshorten-64-to-32", # catch implicit truncation as per MSVC - "-Wsign-compare", # match MSVC(/W3)/gcc(-Wall) - "-Wconditional-uninitialized", # gcc does better here but enable for safety - "-Wuninitialized", - "-Wno-unknown-warning-option", - ] - - extra_link_args += [ - osx_sdk, - "-isysroot{}".format(os.environ.get("CONDA_BUILD_SYSROOT", "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk")), - ] - define_macros += [ - # https://conda-forge.org/docs/maintainer/knowledge_base/#newer-c-features-with-old-sdk - ("_LIBCPP_DISABLE_AVAILABILITY", None), - ] - - extra_compile_args += opt_args - extra_link_args += opt_args # required for LTO - - -elif toolset == "msvc": - - os.environ["DISTUTILS_USE_SDK"] = "1" - os.environ["MSSdk"] = "1" - - condalib_dir = sys.prefix + "/Library" - condalib_inc = condalib_dir + "/include" - condalib_lib = condalib_dir + "/lib" - include_dirs.append(condalib_inc) - library_dirs.append(condalib_lib) - - define_macros += [ - ("_CRT_SECURE_NO_WARNINGS", None), - ] - - libraries += [ - "zlib", - ] - - # VC flags common to both debug and release modes - extra_compile_args += [ - "/std:c++20", - "/permissive-", - "/Zc:__cplusplus", - "/Zc:strictStrings-", # don"t let strings be written to by default - # compatibility with __VA_OPT__ - # see https://devblogs.microsoft.com/cppblog/announcing-full-support-for-a-c-c-conformant-preprocessor-in-msvc/ - "/Zc:preprocessor", - "/W3", # Warning level 3 - "/EHsc", # Enable C++ and structured exception handling (e.g. catch access violations) - "/wd5033", # Python usage of deprecated register keyword - ] - - extra_link_args += [ - "/PDB:%s\\_cxx.pdb" % libname, # Put it right beside the .pyd/.so file in "develop" mode - ] - - if debug: - # do NOT define _DEBUG; need non-debug runtime to match NDEBUG Python distribution - # define_macros += [ - # ("_DEBUG", None), - # ] - extra_compile_args += [ - "/GS", # Enable buffer overrun checks - "/Zi", # Enable debug information .pdb - "/Od", # Disable optimizations - ] - extra_link_args += [ - "/DEBUG", - ] - else: - extra_compile_args += [ - "/GL", # Enable whole-program optimization - "/Gy", # Enable function-level linking - "/Oy", # Omit frame pointers - "/Oi", # Enable intrinsics - #"/Zi", # Enable debug information .pdb - ] - extra_link_args += [ - "/LTCG", # Enable link-time code generation - #"/DEBUG", - ] - - -class NoCWarningsBuildExt(build_ext): - def build_extensions(self): - for x in ["-Wstrict-prototypes"]: - try: - self.compiler.compiler_so.remove(x) - except (AttributeError, ValueError): - continue - build_ext.build_extensions(self) - - -extension = Extension( - libname + "._cxx", - sources=sources, - depends=headers, - include_dirs=include_dirs, - define_macros=define_macros, - library_dirs=library_dirs, - libraries=libraries, - runtime_library_dirs=runtime_library_dirs, - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, -) - -############################################################## - -# monkey-patch for parallel compilation -# taken from http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils - -PARALLEL_JOBS = 4 # number of parallel compilations - - -def gcc_parallel_ccompile(self, - sources, - output_dir=None, - macros=None, - include_dirs=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - depends=None): - # those lines are copied from distutils.ccompiler.CCompiler directly - macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) - cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) - # parallel code - import multiprocessing.pool - - def _single_compile(obj): - try: - src, ext = build[obj] - except KeyError: - return - self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) - - # convert to list, imap is evaluated on-demand - list(multiprocessing.pool.ThreadPool(PARALLEL_JOBS).imap(_single_compile, objects)) - return objects - - -def windows_parallel_ccompile(self, - sources, - output_dir=None, - macros=None, - include_dirs=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - depends=None): - if not self.initialized: - self.initialize() - compile_info = self._setup_compile(output_dir, macros, include_dirs, sources, depends, extra_postargs) - macros, objects, extra_postargs, pp_opts, build = compile_info - - from distutils.errors import CompileError - from distutils.errors import DistutilsExecError - - compile_opts = extra_preargs or [] - compile_opts.append("/c") - compile_opts.extend(self.compile_options_debug if debug else self.compile_options) - - def _compile_obj(obj): - try: - src, ext = build[obj] - except KeyError: - return - if debug: - # pass the full pathname to MSVC in debug mode, - # this allows the debugger to find the source file - # without asking the user to browse for it - src = os.path.abspath(src) - - if ext in self._c_extensions: - input_opt = "/Tc" + src - elif ext in self._cpp_extensions: - input_opt = "/Tp" + src - elif ext in self._rc_extensions: - # compile .RC to .RES file - input_opt = src - output_opt = "/fo" + obj - try: - self.spawn([self.rc] + pp_opts + [output_opt] + [input_opt]) - except DistutilsExecError as msg: - raise CompileError(msg) - return - elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. - # * "-h dir" specifies the directory for the - # generated include file - # * "-r dir" specifies the target directory of the - # generated RC file and the binary message resource - # it includes - # - # For now (since there are no options to change this), - # we use the source-directory for the include file and - # the build directory for the RC file and message - # resources. This works at least for win32all. - h_dir = os.path.dirname(src) - rc_dir = os.path.dirname(obj) - try: - # first compile .MC to .RC and .H file - self.spawn([self.mc] + ["-h", h_dir, "-r", rc_dir] + [src]) - base, _ = os.path.splitext(os.path.basename(src)) - rc_file = os.path.join(rc_dir, base + ".rc") - # then compile .RC to .RES file - self.spawn([self.rc] + ["/fo" + obj] + [rc_file]) - - except DistutilsExecError as msg: - raise CompileError(msg) - return - else: - # how to handle this file? - raise CompileError("Don't know how to compile %s to %s" % (src, obj)) - - output_opt = "/Fo" + obj - try: - self.spawn([self.cc] + compile_opts + pp_opts + [input_opt, output_opt] + extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - import multiprocessing.pool - list(multiprocessing.pool.ThreadPool(PARALLEL_JOBS).imap(_compile_obj, objects)) - - return objects - - -if sys.platform == "win32": - import distutils._msvccompiler - distutils._msvccompiler.MSVCCompiler.compile = windows_parallel_ccompile -else: - distutils.ccompiler.CCompiler.compile = gcc_parallel_ccompile