From 9c8fcf7afd4835e20a1c96e16e91476367a5bb74 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Thu, 6 Feb 2025 10:36:03 +0100 Subject: [PATCH 1/4] DEV-24: Add initial changes for Python Wheel support --- .github/workflows/build-wheel-linux.yml | 46 ++++++++++++++++++ .github/workflows/build-wheel-macos.yml | 64 +++++++++++++++++++++++++ .gitignore | 4 +- python_wrapper/buildconfig | 18 +++++++ python_wrapper/setup.cfg | 5 ++ python_wrapper/setup.py | 2 + 6 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build-wheel-linux.yml create mode 100644 .github/workflows/build-wheel-macos.yml create mode 100644 python_wrapper/buildconfig create mode 100644 python_wrapper/setup.cfg create mode 100644 python_wrapper/setup.py diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml new file mode 100644 index 000000000..6b4b7944e --- /dev/null +++ b/.github/workflows/build-wheel-linux.yml @@ -0,0 +1,46 @@ +# (C) Copyright 2025- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + + +name: Build Python Wheel for Linux + +on: + # Trigger the workflow manually + workflow_dispatch: ~ + + # Allow to be called from another workflow + workflow_call: ~ + + # TODO automation trigger + +jobs: + build: + name: Build manylinux_2_28 + runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] + # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? + container: + image: eccr.ecmwf.int/wheelmaker/2_28:latest + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} + steps: + # TODO convert this to be matrix-friendly. Note it's a bit tricky since + # we'd ideally not reexecute the compile step multiple times, but it + # (non-essentially) depends on a matrix-based step + # NOTE we dont use action checkout because it doesnt cleanup after itself correctly + - run: git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY /proj + - run: cd /proj && /buildscripts/prepare_deps.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && /buildscripts/compile.sh ./python_wrapper/buildconfig + - run: cd /proj && PYTHONPATH=/buildscripts /buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && /buildscripts/upload-pypi.sh ./python_wrapper/buildconfig + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + # NOTE temporary thing until all the mess gets cleared + - run: rm -rf ./* ./.git ./.github diff --git a/.github/workflows/build-wheel-macos.yml b/.github/workflows/build-wheel-macos.yml new file mode 100644 index 000000000..4d576c643 --- /dev/null +++ b/.github/workflows/build-wheel-macos.yml @@ -0,0 +1,64 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + + +name: Build Python Wheel for MacOS + +on: + # Trigger the workflow manually + workflow_dispatch: ~ + + # Allow to be called from another workflow + workflow_call: ~ + + # TODO automation trigger + +jobs: + build: + name: Build macos wheel + # TODO enable after all tested + # strategy: + # matrix: + # arch_type: [ARM64, X64] + # runs-on: [self-hosted, macOS, "${{ matrix.arch_type }}"] + runs-on: [self-hosted, macOS, ARM64 ] + steps: + # TODO convert this to be matrix-friendly for python versions. Note it's a bit tricky since + # we'd ideally not reexecute the compile step multiple times, but it + # (non-essentially) depends on a matrix-based step + # NOTE we dont use action checkout because it doesnt cleanup after itself correctly + - run: | + if [ -z "$(which uv)" ] ; then curl -LsSf https://astral.sh/uv/install.sh | sh ; fi + rm -rf ecbuild wheelmaker + git clone --depth=1 https://github.com/ecmwf/ecbuild ecbuild + # git clone --depth=1 --branch="wheelmaker" https://github.com/ecmwf/ci-utils wheelmaker # TODO use token here to get rid of the checkout action below + - uses: actions/checkout@v4 + with: + repository: ecmwf/ci-utils + ref: develop + path: ci-utils + token: ${{ secrets.GH_REPO_READ_TOKEN }} + - run: rm -rf proj && git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY proj + - run: | + cd proj && $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/prepare_deps.sh ./python_wrapper/buildconfig 3.11 + - run: | + cd proj + if [[ -f ./python_wrapper/pre-compile.sh ]] ; then ./python_wrapper/pre-compile.sh ; fi + PATH="$PATH:$GITHUB_WORKSPACE/ecbuild/bin/" $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/compile.sh ./python_wrapper/buildconfig + - run: | + cd proj + rm -rf /tmp/buildvenv && uv python install python3.11 && uv venv --python python3.11 /tmp/buildvenv && source /tmp/buildvenv/bin/activate && uv pip install build twine + PYTHONPATH=$GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 + if [[ -f ./python_wrapper/post-build.sh ]] ; then ./python_wrapper/post-build.sh ; fi + $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 + $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/upload-pypi.sh ./python_wrapper/buildconfig + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + # NOTE temporary thing until all the mess gets cleared + - run: rm -rf ./* ./.git ./.github diff --git a/.gitignore b/.gitignore index 2e29f5ec8..0ef9b845c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,9 +6,9 @@ doc/latex *.sublime-workspace .*.sw* .*un~ -build* +build/* docs/_build **/.ccls-cache __pycache__ *.egg-info -*.ipynb_checkpoints \ No newline at end of file +*.ipynb_checkpoints diff --git a/python_wrapper/buildconfig b/python_wrapper/buildconfig new file mode 100644 index 000000000..39f195a59 --- /dev/null +++ b/python_wrapper/buildconfig @@ -0,0 +1,18 @@ +# (C) Copyright 2025- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# to be source'd by wheelmaker's compile.sh *and* wheel-linux.sh +# NOTE replace the whole thing with pyproject.toml? Less powerful, and quaint to use for sourcing ecbuild invocation +# TODO we duplicate information -- pyproject.toml's `name` and `packages` are derivable from $NAME and must stay consistent + +NAME="multio" +CMAKE_PARAMS1="-Deckit_ROOT=/tmp/multio/prereqs/eckitlib -Deccodes_ROOT=/tmp/multio/prereqs/eccodeslib -Dmetkit_ROOT=/tmp/multio/prereqs/metkitlib -Datlas_ROOT=/tmp/multio/prereqs/atlaslib -Dfdb5_ROOT=/tmp/multio/prereqs/fdblib -Dmir_ROOT=/tmp/multio/prereqs/mirlib -Dfckit_ROOT=/tmp/multio/prereqs/fckitlib" +CMAKE_PARAMS2="-DENABLE_ATLAS_IO=0" +CMAKE_PARAMS="$CMAKE_PARAMS1 $CMAKE_PARAMS2" +PYPROJECT_DIR="python_wrapper" +DEPENDENCIES='["eckitlib", "eccodeslib", "metkitlib", "atlaslib", "fdblib", "mirlib", "fckitlib"]' diff --git a/python_wrapper/setup.cfg b/python_wrapper/setup.cfg new file mode 100644 index 000000000..d68ff9cf5 --- /dev/null +++ b/python_wrapper/setup.cfg @@ -0,0 +1,5 @@ +[metadata] +description = "multiolib" +long_description = file: README.md +long_description_content_type = text/markdown +author = file: AUTHORS diff --git a/python_wrapper/setup.py b/python_wrapper/setup.py new file mode 100644 index 000000000..c64ad6d78 --- /dev/null +++ b/python_wrapper/setup.py @@ -0,0 +1,2 @@ +from setup_utils import plain_setup +plain_setup() From eb505de1be639044c6b87879e36c678004024d2b Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Thu, 13 Feb 2025 11:41:57 +0100 Subject: [PATCH 2/4] DEV-24: Reorganise python interface --- .../pymultio}/CHANGELOG.md | 0 {pymultio => python_interface/pymultio}/README.md | 0 .../pymultio}/example/plan.py | 0 .../pymultio}/example/test.grib | Bin .../pymultio}/example/test.py | 0 .../pymultio}/requirements.txt | 0 .../pymultio}/src/multio/__init__.py | 0 .../pymultio}/src/multio/constants.py | 0 .../pymultio}/src/multio/lib.py | 0 .../pymultio}/src/multio/metadata.py | 0 .../pymultio}/src/multio/multio.py | 0 .../pymultio}/src/multio/plans/__init__.py | 0 .../pymultio}/src/multio/plans/actions.py | 0 .../pymultio}/src/multio/plans/plans.py | 0 .../pymultio}/src/multio/plans/sinks.py | 0 .../pymultio}/src/multio/processed_multio.h | 0 .../pymultio}/src/multio/utils.py | 0 .../pymultio}/tests/test_actions.py | 0 .../pymultio}/tests/test_context_manager.py | 0 .../pymultio}/tests/test_multio_pythonapi.py | 0 .../pymultio}/tests/test_plans.py | 0 pyproject.toml => python_interface/pyproject.toml | 2 +- python_wrapper/buildconfig | 2 +- 23 files changed, 2 insertions(+), 2 deletions(-) rename {pymultio => python_interface/pymultio}/CHANGELOG.md (100%) rename {pymultio => python_interface/pymultio}/README.md (100%) rename {pymultio => python_interface/pymultio}/example/plan.py (100%) rename {pymultio => python_interface/pymultio}/example/test.grib (100%) rename {pymultio => python_interface/pymultio}/example/test.py (100%) rename {pymultio => python_interface/pymultio}/requirements.txt (100%) rename {pymultio => python_interface/pymultio}/src/multio/__init__.py (100%) rename {pymultio => python_interface/pymultio}/src/multio/constants.py (100%) rename {pymultio => python_interface/pymultio}/src/multio/lib.py (100%) rename {pymultio => python_interface/pymultio}/src/multio/metadata.py (100%) rename {pymultio => python_interface/pymultio}/src/multio/multio.py (100%) rename {pymultio => python_interface/pymultio}/src/multio/plans/__init__.py (100%) rename {pymultio => python_interface/pymultio}/src/multio/plans/actions.py (100%) rename {pymultio => python_interface/pymultio}/src/multio/plans/plans.py (100%) rename {pymultio => python_interface/pymultio}/src/multio/plans/sinks.py (100%) rename {pymultio => python_interface/pymultio}/src/multio/processed_multio.h (100%) rename {pymultio => python_interface/pymultio}/src/multio/utils.py (100%) rename {pymultio => python_interface/pymultio}/tests/test_actions.py (100%) rename {pymultio => python_interface/pymultio}/tests/test_context_manager.py (100%) rename {pymultio => python_interface/pymultio}/tests/test_multio_pythonapi.py (100%) rename {pymultio => python_interface/pymultio}/tests/test_plans.py (100%) rename pyproject.toml => python_interface/pyproject.toml (98%) diff --git a/pymultio/CHANGELOG.md b/python_interface/pymultio/CHANGELOG.md similarity index 100% rename from pymultio/CHANGELOG.md rename to python_interface/pymultio/CHANGELOG.md diff --git a/pymultio/README.md b/python_interface/pymultio/README.md similarity index 100% rename from pymultio/README.md rename to python_interface/pymultio/README.md diff --git a/pymultio/example/plan.py b/python_interface/pymultio/example/plan.py similarity index 100% rename from pymultio/example/plan.py rename to python_interface/pymultio/example/plan.py diff --git a/pymultio/example/test.grib b/python_interface/pymultio/example/test.grib similarity index 100% rename from pymultio/example/test.grib rename to python_interface/pymultio/example/test.grib diff --git a/pymultio/example/test.py b/python_interface/pymultio/example/test.py similarity index 100% rename from pymultio/example/test.py rename to python_interface/pymultio/example/test.py diff --git a/pymultio/requirements.txt b/python_interface/pymultio/requirements.txt similarity index 100% rename from pymultio/requirements.txt rename to python_interface/pymultio/requirements.txt diff --git a/pymultio/src/multio/__init__.py b/python_interface/pymultio/src/multio/__init__.py similarity index 100% rename from pymultio/src/multio/__init__.py rename to python_interface/pymultio/src/multio/__init__.py diff --git a/pymultio/src/multio/constants.py b/python_interface/pymultio/src/multio/constants.py similarity index 100% rename from pymultio/src/multio/constants.py rename to python_interface/pymultio/src/multio/constants.py diff --git a/pymultio/src/multio/lib.py b/python_interface/pymultio/src/multio/lib.py similarity index 100% rename from pymultio/src/multio/lib.py rename to python_interface/pymultio/src/multio/lib.py diff --git a/pymultio/src/multio/metadata.py b/python_interface/pymultio/src/multio/metadata.py similarity index 100% rename from pymultio/src/multio/metadata.py rename to python_interface/pymultio/src/multio/metadata.py diff --git a/pymultio/src/multio/multio.py b/python_interface/pymultio/src/multio/multio.py similarity index 100% rename from pymultio/src/multio/multio.py rename to python_interface/pymultio/src/multio/multio.py diff --git a/pymultio/src/multio/plans/__init__.py b/python_interface/pymultio/src/multio/plans/__init__.py similarity index 100% rename from pymultio/src/multio/plans/__init__.py rename to python_interface/pymultio/src/multio/plans/__init__.py diff --git a/pymultio/src/multio/plans/actions.py b/python_interface/pymultio/src/multio/plans/actions.py similarity index 100% rename from pymultio/src/multio/plans/actions.py rename to python_interface/pymultio/src/multio/plans/actions.py diff --git a/pymultio/src/multio/plans/plans.py b/python_interface/pymultio/src/multio/plans/plans.py similarity index 100% rename from pymultio/src/multio/plans/plans.py rename to python_interface/pymultio/src/multio/plans/plans.py diff --git a/pymultio/src/multio/plans/sinks.py b/python_interface/pymultio/src/multio/plans/sinks.py similarity index 100% rename from pymultio/src/multio/plans/sinks.py rename to python_interface/pymultio/src/multio/plans/sinks.py diff --git a/pymultio/src/multio/processed_multio.h b/python_interface/pymultio/src/multio/processed_multio.h similarity index 100% rename from pymultio/src/multio/processed_multio.h rename to python_interface/pymultio/src/multio/processed_multio.h diff --git a/pymultio/src/multio/utils.py b/python_interface/pymultio/src/multio/utils.py similarity index 100% rename from pymultio/src/multio/utils.py rename to python_interface/pymultio/src/multio/utils.py diff --git a/pymultio/tests/test_actions.py b/python_interface/pymultio/tests/test_actions.py similarity index 100% rename from pymultio/tests/test_actions.py rename to python_interface/pymultio/tests/test_actions.py diff --git a/pymultio/tests/test_context_manager.py b/python_interface/pymultio/tests/test_context_manager.py similarity index 100% rename from pymultio/tests/test_context_manager.py rename to python_interface/pymultio/tests/test_context_manager.py diff --git a/pymultio/tests/test_multio_pythonapi.py b/python_interface/pymultio/tests/test_multio_pythonapi.py similarity index 100% rename from pymultio/tests/test_multio_pythonapi.py rename to python_interface/pymultio/tests/test_multio_pythonapi.py diff --git a/pymultio/tests/test_plans.py b/python_interface/pymultio/tests/test_plans.py similarity index 100% rename from pymultio/tests/test_plans.py rename to python_interface/pymultio/tests/test_plans.py diff --git a/pyproject.toml b/python_interface/pyproject.toml similarity index 98% rename from pyproject.toml rename to python_interface/pyproject.toml index 092d17dad..dfc239bd0 100644 --- a/pyproject.toml +++ b/python_interface/pyproject.toml @@ -24,7 +24,7 @@ name = "pymultio" readme = "README.md" description = "A Python interface to multio." keywords = [ "multio", "multiopython", "tools" ] -license = { file = "LICENSE" } +license = { file = "../LICENSE" } authors = [ { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" }, ] diff --git a/python_wrapper/buildconfig b/python_wrapper/buildconfig index 39f195a59..4a60c4aab 100644 --- a/python_wrapper/buildconfig +++ b/python_wrapper/buildconfig @@ -12,7 +12,7 @@ NAME="multio" CMAKE_PARAMS1="-Deckit_ROOT=/tmp/multio/prereqs/eckitlib -Deccodes_ROOT=/tmp/multio/prereqs/eccodeslib -Dmetkit_ROOT=/tmp/multio/prereqs/metkitlib -Datlas_ROOT=/tmp/multio/prereqs/atlaslib -Dfdb5_ROOT=/tmp/multio/prereqs/fdblib -Dmir_ROOT=/tmp/multio/prereqs/mirlib -Dfckit_ROOT=/tmp/multio/prereqs/fckitlib" -CMAKE_PARAMS2="-DENABLE_ATLAS_IO=0" +CMAKE_PARAMS2="-DENABLE_ATLAS_IO=1" CMAKE_PARAMS="$CMAKE_PARAMS1 $CMAKE_PARAMS2" PYPROJECT_DIR="python_wrapper" DEPENDENCIES='["eckitlib", "eccodeslib", "metkitlib", "atlaslib", "fdblib", "mirlib", "fckitlib"]' From fa05ea62e4f609332642a5e3a1b895c6dabd4350 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Thu, 13 Feb 2025 11:47:12 +0100 Subject: [PATCH 3/4] DEV-24: Switch to reusable action --- .github/workflows/build-wheel-linux.yml | 46 ---------------- .github/workflows/build-wheel-macos.yml | 64 ----------------------- .github/workflows/build-wheel-wrapper.yml | 25 +++++++++ 3 files changed, 25 insertions(+), 110 deletions(-) delete mode 100644 .github/workflows/build-wheel-linux.yml delete mode 100644 .github/workflows/build-wheel-macos.yml create mode 100644 .github/workflows/build-wheel-wrapper.yml diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml deleted file mode 100644 index 6b4b7944e..000000000 --- a/.github/workflows/build-wheel-linux.yml +++ /dev/null @@ -1,46 +0,0 @@ -# (C) Copyright 2025- ECMWF. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - - -name: Build Python Wheel for Linux - -on: - # Trigger the workflow manually - workflow_dispatch: ~ - - # Allow to be called from another workflow - workflow_call: ~ - - # TODO automation trigger - -jobs: - build: - name: Build manylinux_2_28 - runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] - # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? - container: - image: eccr.ecmwf.int/wheelmaker/2_28:latest - credentials: - username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} - password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} - steps: - # TODO convert this to be matrix-friendly. Note it's a bit tricky since - # we'd ideally not reexecute the compile step multiple times, but it - # (non-essentially) depends on a matrix-based step - # NOTE we dont use action checkout because it doesnt cleanup after itself correctly - - run: git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY /proj - - run: cd /proj && /buildscripts/prepare_deps.sh ./python_wrapper/buildconfig 3.11 - - run: cd /proj && /buildscripts/compile.sh ./python_wrapper/buildconfig - - run: cd /proj && PYTHONPATH=/buildscripts /buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 - - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 - - run: cd /proj && /buildscripts/upload-pypi.sh ./python_wrapper/buildconfig - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - # NOTE temporary thing until all the mess gets cleared - - run: rm -rf ./* ./.git ./.github diff --git a/.github/workflows/build-wheel-macos.yml b/.github/workflows/build-wheel-macos.yml deleted file mode 100644 index 4d576c643..000000000 --- a/.github/workflows/build-wheel-macos.yml +++ /dev/null @@ -1,64 +0,0 @@ -# (C) Copyright 2024- ECMWF. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# In applying this licence, ECMWF does not waive the privileges and immunities -# granted to it by virtue of its status as an intergovernmental organisation -# nor does it submit to any jurisdiction. - - -name: Build Python Wheel for MacOS - -on: - # Trigger the workflow manually - workflow_dispatch: ~ - - # Allow to be called from another workflow - workflow_call: ~ - - # TODO automation trigger - -jobs: - build: - name: Build macos wheel - # TODO enable after all tested - # strategy: - # matrix: - # arch_type: [ARM64, X64] - # runs-on: [self-hosted, macOS, "${{ matrix.arch_type }}"] - runs-on: [self-hosted, macOS, ARM64 ] - steps: - # TODO convert this to be matrix-friendly for python versions. Note it's a bit tricky since - # we'd ideally not reexecute the compile step multiple times, but it - # (non-essentially) depends on a matrix-based step - # NOTE we dont use action checkout because it doesnt cleanup after itself correctly - - run: | - if [ -z "$(which uv)" ] ; then curl -LsSf https://astral.sh/uv/install.sh | sh ; fi - rm -rf ecbuild wheelmaker - git clone --depth=1 https://github.com/ecmwf/ecbuild ecbuild - # git clone --depth=1 --branch="wheelmaker" https://github.com/ecmwf/ci-utils wheelmaker # TODO use token here to get rid of the checkout action below - - uses: actions/checkout@v4 - with: - repository: ecmwf/ci-utils - ref: develop - path: ci-utils - token: ${{ secrets.GH_REPO_READ_TOKEN }} - - run: rm -rf proj && git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY proj - - run: | - cd proj && $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/prepare_deps.sh ./python_wrapper/buildconfig 3.11 - - run: | - cd proj - if [[ -f ./python_wrapper/pre-compile.sh ]] ; then ./python_wrapper/pre-compile.sh ; fi - PATH="$PATH:$GITHUB_WORKSPACE/ecbuild/bin/" $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/compile.sh ./python_wrapper/buildconfig - - run: | - cd proj - rm -rf /tmp/buildvenv && uv python install python3.11 && uv venv --python python3.11 /tmp/buildvenv && source /tmp/buildvenv/bin/activate && uv pip install build twine - PYTHONPATH=$GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 - if [[ -f ./python_wrapper/post-build.sh ]] ; then ./python_wrapper/post-build.sh ; fi - $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 - $GITHUB_WORKSPACE/ci-utils/wheelmaker/buildscripts/upload-pypi.sh ./python_wrapper/buildconfig - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - # NOTE temporary thing until all the mess gets cleared - - run: rm -rf ./* ./.git ./.github diff --git a/.github/workflows/build-wheel-wrapper.yml b/.github/workflows/build-wheel-wrapper.yml new file mode 100644 index 000000000..bea8ed8c3 --- /dev/null +++ b/.github/workflows/build-wheel-wrapper.yml @@ -0,0 +1,25 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + + +name: Build Python Wrapper Wheel + +on: + # Trigger the workflow manually + workflow_dispatch: ~ + + # Allow to be called from another workflow + workflow_call: ~ + + # TODO automation trigger + +jobs: + python-wrapper-wheel: + name: Python Wrapper Wheel + uses: ecmwf-actions/reusable-workflows/.github/workflows/python-wrapper-wheel.yml@main + secrets: inherit From 3a998026e6f476ff190c094d48d8f690d6662a99 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 24 Mar 2025 13:40:16 +0100 Subject: [PATCH 4/4] DEV-24: Reorganise python packages in std fashion --- .github/workflows/build-wheel-wrapper.yml | 4 +++- python_interface/pyproject.toml => pyproject.toml | 6 ++++-- {python_wrapper => python/multiolib}/buildconfig | 2 +- {python_wrapper => python/multiolib}/setup.cfg | 0 {python_wrapper => python/multiolib}/setup.py | 0 {python_interface => python}/pymultio/CHANGELOG.md | 0 {python_interface => python}/pymultio/README.md | 0 .../pymultio/example/plan.py | 0 .../pymultio/example/test.grib | Bin .../pymultio/example/test.py | 0 .../pymultio/requirements.txt | 0 .../pymultio/src/multio/__init__.py | 0 .../pymultio/src/multio/constants.py | 0 .../pymultio/src/multio/lib.py | 0 .../pymultio/src/multio/metadata.py | 0 .../pymultio/src/multio/multio.py | 0 .../pymultio/src/multio/plans/__init__.py | 0 .../pymultio/src/multio/plans/actions.py | 0 .../pymultio/src/multio/plans/plans.py | 0 .../pymultio/src/multio/plans/sinks.py | 0 .../pymultio/src/multio/processed_multio.h | 0 .../pymultio/src/multio/utils.py | 0 .../pymultio/tests/test_actions.py | 0 .../pymultio/tests/test_context_manager.py | 0 .../pymultio/tests/test_multio_pythonapi.py | 0 .../pymultio/tests/test_plans.py | 0 26 files changed, 8 insertions(+), 4 deletions(-) rename python_interface/pyproject.toml => pyproject.toml (90%) rename {python_wrapper => python/multiolib}/buildconfig (97%) rename {python_wrapper => python/multiolib}/setup.cfg (100%) rename {python_wrapper => python/multiolib}/setup.py (100%) rename {python_interface => python}/pymultio/CHANGELOG.md (100%) rename {python_interface => python}/pymultio/README.md (100%) rename {python_interface => python}/pymultio/example/plan.py (100%) rename {python_interface => python}/pymultio/example/test.grib (100%) rename {python_interface => python}/pymultio/example/test.py (100%) rename {python_interface => python}/pymultio/requirements.txt (100%) rename {python_interface => python}/pymultio/src/multio/__init__.py (100%) rename {python_interface => python}/pymultio/src/multio/constants.py (100%) rename {python_interface => python}/pymultio/src/multio/lib.py (100%) rename {python_interface => python}/pymultio/src/multio/metadata.py (100%) rename {python_interface => python}/pymultio/src/multio/multio.py (100%) rename {python_interface => python}/pymultio/src/multio/plans/__init__.py (100%) rename {python_interface => python}/pymultio/src/multio/plans/actions.py (100%) rename {python_interface => python}/pymultio/src/multio/plans/plans.py (100%) rename {python_interface => python}/pymultio/src/multio/plans/sinks.py (100%) rename {python_interface => python}/pymultio/src/multio/processed_multio.h (100%) rename {python_interface => python}/pymultio/src/multio/utils.py (100%) rename {python_interface => python}/pymultio/tests/test_actions.py (100%) rename {python_interface => python}/pymultio/tests/test_context_manager.py (100%) rename {python_interface => python}/pymultio/tests/test_multio_pythonapi.py (100%) rename {python_interface => python}/pymultio/tests/test_plans.py (100%) diff --git a/.github/workflows/build-wheel-wrapper.yml b/.github/workflows/build-wheel-wrapper.yml index bea8ed8c3..19a500cf9 100644 --- a/.github/workflows/build-wheel-wrapper.yml +++ b/.github/workflows/build-wheel-wrapper.yml @@ -21,5 +21,7 @@ on: jobs: python-wrapper-wheel: name: Python Wrapper Wheel - uses: ecmwf-actions/reusable-workflows/.github/workflows/python-wrapper-wheel.yml@main + uses: ecmwf/reusable-workflows/.github/workflows/python-wrapper-wheel.yml@main + with: + wheel_directory: python/multiolib secrets: inherit diff --git a/python_interface/pyproject.toml b/pyproject.toml similarity index 90% rename from python_interface/pyproject.toml rename to pyproject.toml index dfc239bd0..68b631b6e 100644 --- a/python_interface/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,8 @@ # https://packaging.python.org/en/latest/guides/writing-pyproject-toml/ +# TODO migrate this to python/pymultio once the ci action is modular enough. Don't forget to update relative `license` and + [tool.pytest.ini_options] minversion = "6.0" addopts = "-vv -s" @@ -24,7 +26,7 @@ name = "pymultio" readme = "README.md" description = "A Python interface to multio." keywords = [ "multio", "multiopython", "tools" ] -license = { file = "../LICENSE" } +license = { file = "./LICENSE" } authors = [ { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" }, ] @@ -64,7 +66,7 @@ urls.Repository = "https://github.com/ecmwf/multio/" [tool.setuptools] packages = ["multio"] -package-dir = { "multio" = "./pymultio/src/multio" } +package-dir = { "multio" = "./python/pymultio/src/multio" } include-package-data = true zip-safe = true diff --git a/python_wrapper/buildconfig b/python/multiolib/buildconfig similarity index 97% rename from python_wrapper/buildconfig rename to python/multiolib/buildconfig index 4a60c4aab..02b7417be 100644 --- a/python_wrapper/buildconfig +++ b/python/multiolib/buildconfig @@ -14,5 +14,5 @@ NAME="multio" CMAKE_PARAMS1="-Deckit_ROOT=/tmp/multio/prereqs/eckitlib -Deccodes_ROOT=/tmp/multio/prereqs/eccodeslib -Dmetkit_ROOT=/tmp/multio/prereqs/metkitlib -Datlas_ROOT=/tmp/multio/prereqs/atlaslib -Dfdb5_ROOT=/tmp/multio/prereqs/fdblib -Dmir_ROOT=/tmp/multio/prereqs/mirlib -Dfckit_ROOT=/tmp/multio/prereqs/fckitlib" CMAKE_PARAMS2="-DENABLE_ATLAS_IO=1" CMAKE_PARAMS="$CMAKE_PARAMS1 $CMAKE_PARAMS2" -PYPROJECT_DIR="python_wrapper" +PYPROJECT_DIR="python/multiolib" DEPENDENCIES='["eckitlib", "eccodeslib", "metkitlib", "atlaslib", "fdblib", "mirlib", "fckitlib"]' diff --git a/python_wrapper/setup.cfg b/python/multiolib/setup.cfg similarity index 100% rename from python_wrapper/setup.cfg rename to python/multiolib/setup.cfg diff --git a/python_wrapper/setup.py b/python/multiolib/setup.py similarity index 100% rename from python_wrapper/setup.py rename to python/multiolib/setup.py diff --git a/python_interface/pymultio/CHANGELOG.md b/python/pymultio/CHANGELOG.md similarity index 100% rename from python_interface/pymultio/CHANGELOG.md rename to python/pymultio/CHANGELOG.md diff --git a/python_interface/pymultio/README.md b/python/pymultio/README.md similarity index 100% rename from python_interface/pymultio/README.md rename to python/pymultio/README.md diff --git a/python_interface/pymultio/example/plan.py b/python/pymultio/example/plan.py similarity index 100% rename from python_interface/pymultio/example/plan.py rename to python/pymultio/example/plan.py diff --git a/python_interface/pymultio/example/test.grib b/python/pymultio/example/test.grib similarity index 100% rename from python_interface/pymultio/example/test.grib rename to python/pymultio/example/test.grib diff --git a/python_interface/pymultio/example/test.py b/python/pymultio/example/test.py similarity index 100% rename from python_interface/pymultio/example/test.py rename to python/pymultio/example/test.py diff --git a/python_interface/pymultio/requirements.txt b/python/pymultio/requirements.txt similarity index 100% rename from python_interface/pymultio/requirements.txt rename to python/pymultio/requirements.txt diff --git a/python_interface/pymultio/src/multio/__init__.py b/python/pymultio/src/multio/__init__.py similarity index 100% rename from python_interface/pymultio/src/multio/__init__.py rename to python/pymultio/src/multio/__init__.py diff --git a/python_interface/pymultio/src/multio/constants.py b/python/pymultio/src/multio/constants.py similarity index 100% rename from python_interface/pymultio/src/multio/constants.py rename to python/pymultio/src/multio/constants.py diff --git a/python_interface/pymultio/src/multio/lib.py b/python/pymultio/src/multio/lib.py similarity index 100% rename from python_interface/pymultio/src/multio/lib.py rename to python/pymultio/src/multio/lib.py diff --git a/python_interface/pymultio/src/multio/metadata.py b/python/pymultio/src/multio/metadata.py similarity index 100% rename from python_interface/pymultio/src/multio/metadata.py rename to python/pymultio/src/multio/metadata.py diff --git a/python_interface/pymultio/src/multio/multio.py b/python/pymultio/src/multio/multio.py similarity index 100% rename from python_interface/pymultio/src/multio/multio.py rename to python/pymultio/src/multio/multio.py diff --git a/python_interface/pymultio/src/multio/plans/__init__.py b/python/pymultio/src/multio/plans/__init__.py similarity index 100% rename from python_interface/pymultio/src/multio/plans/__init__.py rename to python/pymultio/src/multio/plans/__init__.py diff --git a/python_interface/pymultio/src/multio/plans/actions.py b/python/pymultio/src/multio/plans/actions.py similarity index 100% rename from python_interface/pymultio/src/multio/plans/actions.py rename to python/pymultio/src/multio/plans/actions.py diff --git a/python_interface/pymultio/src/multio/plans/plans.py b/python/pymultio/src/multio/plans/plans.py similarity index 100% rename from python_interface/pymultio/src/multio/plans/plans.py rename to python/pymultio/src/multio/plans/plans.py diff --git a/python_interface/pymultio/src/multio/plans/sinks.py b/python/pymultio/src/multio/plans/sinks.py similarity index 100% rename from python_interface/pymultio/src/multio/plans/sinks.py rename to python/pymultio/src/multio/plans/sinks.py diff --git a/python_interface/pymultio/src/multio/processed_multio.h b/python/pymultio/src/multio/processed_multio.h similarity index 100% rename from python_interface/pymultio/src/multio/processed_multio.h rename to python/pymultio/src/multio/processed_multio.h diff --git a/python_interface/pymultio/src/multio/utils.py b/python/pymultio/src/multio/utils.py similarity index 100% rename from python_interface/pymultio/src/multio/utils.py rename to python/pymultio/src/multio/utils.py diff --git a/python_interface/pymultio/tests/test_actions.py b/python/pymultio/tests/test_actions.py similarity index 100% rename from python_interface/pymultio/tests/test_actions.py rename to python/pymultio/tests/test_actions.py diff --git a/python_interface/pymultio/tests/test_context_manager.py b/python/pymultio/tests/test_context_manager.py similarity index 100% rename from python_interface/pymultio/tests/test_context_manager.py rename to python/pymultio/tests/test_context_manager.py diff --git a/python_interface/pymultio/tests/test_multio_pythonapi.py b/python/pymultio/tests/test_multio_pythonapi.py similarity index 100% rename from python_interface/pymultio/tests/test_multio_pythonapi.py rename to python/pymultio/tests/test_multio_pythonapi.py diff --git a/python_interface/pymultio/tests/test_plans.py b/python/pymultio/tests/test_plans.py similarity index 100% rename from python_interface/pymultio/tests/test_plans.py rename to python/pymultio/tests/test_plans.py