From 7a47485bb6202393097f18bea2fcb370fdc459fe Mon Sep 17 00:00:00 2001 From: Michele Dolfi Date: Wed, 4 Dec 2024 19:09:11 +0100 Subject: [PATCH 1/3] add mergify Signed-off-by: Michele Dolfi --- .github/mergify.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .github/mergify.yml diff --git a/.github/mergify.yml b/.github/mergify.yml new file mode 100644 index 00000000..c699f7de --- /dev/null +++ b/.github/mergify.yml @@ -0,0 +1,9 @@ +merge_protections: + - name: Enforce conventional commit + description: Make sure that we follow https://www.conventionalcommits.org/en/v1.0.0/ + if: + - base = main + success_conditions: + - "title ~= + ^(fix|feat|docs|style|refactor|perf|test|build|ci|chore|revert)(?:\\(.+\ + \\))?:" From 0c901c5951b36ccca5e649e265a9326845a7483b Mon Sep 17 00:00:00 2001 From: Michele Dolfi Date: Wed, 4 Dec 2024 19:10:04 +0100 Subject: [PATCH 2/3] make CD on-demand Signed-off-by: Michele Dolfi --- .github/workflows/cd.yml | 4 +--- .github/workflows/ci.yml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 2d8bf1c3..1f0502dc 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,9 +1,7 @@ name: "Run CD" on: - push: - branches: - - main + workflow_dispatch: env: # disable keyring (https://github.com/actions/runner-images/issues/6185): diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd5c271f..09ca138a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ on: push: branches: - "**" - - "!main" + # - "!main" - "!gh-pages" env: From ae2e4b5deeab774c980133e0a8a495d2e8365809 Mon Sep 17 00:00:00 2001 From: Michele Dolfi Date: Wed, 4 Dec 2024 19:19:35 +0100 Subject: [PATCH 3/3] ci: publish ci artifacts, simplify mac builds, publish all artifacts together Signed-off-by: Michele Dolfi --- .github/workflows/ci.yml | 3 + .github/workflows/release.yml | 7 +- .github/workflows/wheels.yml | 134 ++++++++++++++++++++++++---------- 3 files changed, 103 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09ca138a..ee45bfc5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,9 @@ jobs: build-wheels: if: ${{ github.event_name == 'push' || (github.event.pull_request.head.repo.full_name != 'DS4SD/deepsearch-glm' && github.event.pull_request.head.repo.full_name != 'ds4sd/deepsearch-glm') }} uses: ./.github/workflows/wheels.yml + permissions: + id-token: write # needed also if not used (see publish condition) + contents: write # needed also if not used (see publish condition) rhel-build: if: ${{ github.event_name == 'push' || (github.event.pull_request.head.repo.full_name != 'DS4SD/deepsearch-glm' && github.event.pull_request.head.repo.full_name != 'ds4sd/deepsearch-glm') }} uses: ./.github/workflows/rhel.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 92817980..e45c51e7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,4 +10,9 @@ permissions: jobs: build-and-publish: uses: ./.github/workflows/wheels.yml - secrets: inherit \ No newline at end of file + with: + publish: true + secrets: inherit + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + contents: write # IMPORTANT: mandatory for adding artifacts to GitHub Releases diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 4c1d6069..b89a606e 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -1,12 +1,39 @@ on: workflow_call: + inputs: + publish: + type: boolean + description: "If true, the packages will be published." + default: false env: CMAKE_BUILD_TYPE: Release jobs: + build_sdist: + name: Build sdist artifacts + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + - name: Install poetry + run: | + pipx install poetry + - name: Build sdit + run: | + poetry build -f sdist + ls ./dist + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions-sdist + path: dist/ + build_wheels: - name: Build wheel for py${{ matrix.python-version }} ${{ matrix.os.platform_id }} ${{ matrix.os.macos_version }} + name: Build wheel for py${{ matrix.python-version }} ${{ matrix.os.platform_id }} runs-on: ${{ matrix.os.name }} strategy: @@ -23,27 +50,17 @@ jobs: - name: "macos-13" platform: "macos" - macos_version: "13" - platform_id: "macosx_x86_64" - - - name: "macos-13" - platform: "macos" - macos_version: "13" - platform_id: "macosx_arm64" - - - name: "macos-14" - platform: "macos" - macos_version: "14" + min_macos_version: "13" platform_id: "macosx_x86_64" - name: "macos-14" platform: "macos" - macos_version: "14" + min_macos_version: "14" platform_id: "macosx_arm64" - name: "windows-latest" platform: "windows" - platform_id: "win_amd64" + platform_id: "win_amd64" steps: - name: Checkout @@ -64,7 +81,7 @@ jobs: python3 --version echo "pythonpath: ${{ steps.py.outputs.python-path }}" ${{ steps.py.outputs.python-path }} --version - pipx install poetry==1.8.3 + pipx install poetry==1.8.4 poetry env use ${{ steps.py.outputs.python-path }} - name: Set up custom PATH and set py version to cpXYZ [windows] @@ -86,12 +103,6 @@ jobs: cp_version="cp${version//.}" echo "python_cp_version=$cp_version" >> $GITHUB_ENV - - name: Setup pypi for poetry [for releases only] - if: ${{ startsWith(github.ref, 'refs/tags/') }} - run: | - poetry config keyring.enabled false - poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} - - name: Build wheels [macos-x86_64] if: ${{matrix.os.platform_id == 'macosx_x86_64'}} env: @@ -104,7 +115,7 @@ jobs: CIBW_BUILD_VERBOSITY: 3 CMAKE_OSX_ARCHITECTURES: x86_64 CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" # do not run delocate-wheel before the re-tag - CIBW_ENVIRONMENT: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.macos_version }}.0" + CIBW_ENVIRONMENT: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.min_macos_version }}.0" ARCHFLAGS: -arch x86_64 BUILD_THREADS: "4" PYTORCH_MPS_HIGH_WATERMARK_RATIO: "0.0" @@ -121,7 +132,7 @@ jobs: poetry run python -m cibuildwheel --output-dir wheelhouse echo "step 1" ls -l wheelhouse - poetry run wheel tags --platform-tag macosx_${{ matrix.os.macos_version }}_0_x86_64 ./wheelhouse/*.whl + poetry run wheel tags --remove --platform-tag macosx_${{ matrix.os.min_macos_version }}_0_x86_64 ./wheelhouse/*.whl rm -f ./wheelhouse/*arm64.whl echo "step 2" ls -l wheelhouse @@ -154,7 +165,7 @@ jobs: CIBW_BUILD_VERBOSITY: 3 CMAKE_OSX_ARCHITECTURES: arm64 CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" # do not run delocate-wheel before the re-tag - CIBW_ENVIRONMENT: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.macos_version }}.0" + CIBW_ENVIRONMENT: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.min_macos_version }}.0" ARCHFLAGS: -arch arm64 BUILD_THREADS: "4" PYTORCH_MPS_HIGH_WATERMARK_RATIO: "0.0" @@ -172,7 +183,7 @@ jobs: poetry run python -m cibuildwheel --output-dir wheelhouse echo "step 1" ls -l wheelhouse - poetry run wheel tags --platform-tag macosx_${{ matrix.os.macos_version }}_0_arm64 ./wheelhouse/*.whl + poetry run wheel tags --remove --platform-tag macosx_${{ matrix.os.min_macos_version }}_0_arm64 ./wheelhouse/*.whl rm -f ./wheelhouse/*x86_64.whl echo "step 2" ls -l wheelhouse @@ -196,17 +207,10 @@ jobs: with: platforms: all - - name: Build sdist - # build only on Linux to avoid too many duplicates of the sdist - if: matrix.os.name == 'ubuntu-latest' - run: | - echo "Building wheel ${CIBW_BUILD}" - poetry build -f sdist - - name: Build wheels [linux] if: matrix.os.name == 'ubuntu-latest' env: - ## CIBW_BUILD: ${{ env.python_cp_version }}-${{ matrix.os.platform_id }} + # CIBW_BUILD: ${{ env.python_cp_version }}-${{ matrix.os.platform_id }} CIBW_ARCHS: auto x86_64 aarch64 CIBW_PLATFORM: linux CIBW_SKIP: "pp* *musllinux_* *_i686* *_s390* *pypy*" @@ -303,12 +307,62 @@ jobs: poetry install --no-interaction --no-root --only=test poetry run python -c 'from deepsearch_glm import andromeda_glm' poetry run pytest ./tests/test_glm.py -v - - - name: Publish wheels (dry run) - run: | - poetry publish --skip-existing --dry-run --no-interaction -vvv - - name: Publish wheels (on publishing) [for releases only] - if: ${{ startsWith(github.ref, 'refs/tags/') }} + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions-py${{ matrix.python-version }}-${{ matrix.os.platform_id }} + path: dist/ + + publish-packages: + name: >- + Publish Python 🐍 distribution 📦 to PyPI + if: inputs.publish && startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build_sdist + - build_wheels + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/deepsearch-glm + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + contents: write # IMPORTANT: mandatory for adding artifacts to GitHub Releases + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + merge-multiple: true + path: dist/ + - name: List dist/ run: | - poetry publish --skip-existing --no-interaction -vvv + ls ./dist + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v3.0.0 + with: + release-signing-artifacts: false + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: List dist/ + run: | + ls ./dist + - name: Upload artifact signatures to GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + # Upload to GitHub Release using the `gh` CLI. + # `dist/` contains the built packages, and the + # sigstore-produced signatures and certificates. + run: >- + gh release upload + '${{ github.ref_name }}' dist/** + --repo '${{ github.repository }}' + # PyPI does not accept .sigstore artifacts and + # gh-action-pypi-publish has no option to ignore them. + - name: Remove sigstore signatures before uploading to PyPI + run: rm ./dist/*.sigstore.json + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + # currently not working with reusable workflows + attestations: false