ci: use ci artifacts, simplify mac builds and publish artifacts toget…

…her (#94) Signed-off-by: Michele Dolfi <[email protected]>
DS4SD · Dec 4, 2024 · 49d2cd8 · 49d2cd8
1 parent 5beeb00
commit 49d2cd8
Show file tree

Hide file tree

Showing 5 changed files with 114 additions and 45 deletions.
diff --git a/.github/mergify.yml b/.github/mergify.yml
@@ -0,0 +1,9 @@
+merge_protections:
+  - name: Enforce conventional commit
+    description: Make sure that we follow https://www.conventionalcommits.org/en/v1.0.0/
+    if:
+      - base = main
+    success_conditions:
+      - "title ~=
+        ^(fix|feat|docs|style|refactor|perf|test|build|ci|chore|revert)(?:\\(.+\
+        \\))?:"
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -1,9 +1,7 @@
 name: "Run CD"
 
 on:
-  push:
-    branches:
-      - main
+  workflow_dispatch:
 
 env:
   # disable keyring (https://github.com/actions/runner-images/issues/6185):

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -9,7 +9,7 @@ on:
   push:
     branches:
       - "**"
-      - "!main"
+      # - "!main"
       - "!gh-pages"
 
 env:
@@ -23,6 +23,9 @@ jobs:
   build-wheels:
     if: ${{ github.event_name == 'push' || (github.event.pull_request.head.repo.full_name != 'DS4SD/deepsearch-glm' && github.event.pull_request.head.repo.full_name != 'ds4sd/deepsearch-glm') }}
     uses: ./.github/workflows/wheels.yml
+    permissions:
+      id-token: write  # needed also if not used (see publish condition)
+      contents: write  # needed also if not used (see publish condition)
   rhel-build:
     if: ${{ github.event_name == 'push' || (github.event.pull_request.head.repo.full_name != 'DS4SD/deepsearch-glm' && github.event.pull_request.head.repo.full_name != 'ds4sd/deepsearch-glm') }}
     uses: ./.github/workflows/rhel.yml
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -10,4 +10,9 @@ permissions:
 jobs:
   build-and-publish:
     uses: ./.github/workflows/wheels.yml
-    secrets: inherit
+    with:
+      publish: true
+    secrets: inherit
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+      contents: write  # IMPORTANT: mandatory for adding artifacts to GitHub Releases
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -1,12 +1,39 @@
 on:
   workflow_call:
+    inputs:
+      publish:
+        type: boolean
+        description: "If true, the packages will be published."
+        default: false
 
 env:
   CMAKE_BUILD_TYPE: Release
 
 jobs:
+  build_sdist:
+    name: Build sdist artifacts
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+      - name: Install poetry
+        run: |
+          pipx install poetry
+      - name: Build sdit
+        run: |
+          poetry build -f sdist
+          ls ./dist
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions-sdist
+          path: dist/
+
   build_wheels:
-    name: Build wheel for py${{ matrix.python-version }} ${{ matrix.os.platform_id }} ${{ matrix.os.macos_version }}
+    name: Build wheel for py${{ matrix.python-version }} ${{ matrix.os.platform_id }}
     runs-on: ${{ matrix.os.name }}
 
     strategy:
@@ -23,27 +50,17 @@ jobs:
 
           - name: "macos-13"
             platform: "macos"
-            macos_version: "13"
-            platform_id: "macosx_x86_64"
-
-          - name: "macos-13"
-            platform: "macos"
-            macos_version: "13"
-            platform_id: "macosx_arm64"
-
-          - name: "macos-14"
-            platform: "macos"
-            macos_version: "14"
+            min_macos_version: "13"
             platform_id: "macosx_x86_64"
 
           - name: "macos-14"
             platform: "macos"
-            macos_version: "14"
+            min_macos_version: "14"
             platform_id: "macosx_arm64"
 
           - name: "windows-latest"
             platform: "windows"
-            platform_id: "win_amd64"
+            platform_id: "win_amd64"      
 
     steps:
       - name: Checkout
@@ -64,7 +81,7 @@ jobs:
           python3 --version
           echo "pythonpath: ${{ steps.py.outputs.python-path }}"
           ${{ steps.py.outputs.python-path }} --version
-          pipx install poetry==1.8.3
+          pipx install poetry==1.8.4
           poetry env use ${{ steps.py.outputs.python-path }}
 
       - name: Set up custom PATH and set py version to cpXYZ [windows]
@@ -86,12 +103,6 @@ jobs:
           cp_version="cp${version//.}"
           echo "python_cp_version=$cp_version" >> $GITHUB_ENV
 
-      - name: Setup pypi for poetry [for releases only]
-        if: ${{ startsWith(github.ref, 'refs/tags/') }}
-        run: |
-          poetry config keyring.enabled false
-          poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }}
-          
       - name: Build wheels [macos-x86_64]
         if: ${{matrix.os.platform_id == 'macosx_x86_64'}}
         env:
@@ -104,7 +115,7 @@ jobs:
           CIBW_BUILD_VERBOSITY: 3
           CMAKE_OSX_ARCHITECTURES: x86_64
           CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""  # do not run delocate-wheel before the re-tag
-          CIBW_ENVIRONMENT: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.macos_version }}.0"
+          CIBW_ENVIRONMENT: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.min_macos_version }}.0"
           ARCHFLAGS: -arch x86_64
           BUILD_THREADS: "4"
           PYTORCH_MPS_HIGH_WATERMARK_RATIO: "0.0"
@@ -121,7 +132,7 @@ jobs:
           poetry run python -m cibuildwheel --output-dir wheelhouse
           echo "step 1"
           ls -l wheelhouse
-          poetry run wheel tags --platform-tag macosx_${{ matrix.os.macos_version }}_0_x86_64 ./wheelhouse/*.whl
+          poetry run wheel tags --remove --platform-tag macosx_${{ matrix.os.min_macos_version }}_0_x86_64 ./wheelhouse/*.whl
           rm -f ./wheelhouse/*arm64.whl
           echo "step 2"
           ls -l wheelhouse
@@ -154,7 +165,7 @@ jobs:
           CIBW_BUILD_VERBOSITY: 3
           CMAKE_OSX_ARCHITECTURES: arm64
           CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""  # do not run delocate-wheel before the re-tag
-          CIBW_ENVIRONMENT: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.macos_version }}.0"
+          CIBW_ENVIRONMENT: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.min_macos_version }}.0"
           ARCHFLAGS: -arch arm64
           BUILD_THREADS: "4"
           PYTORCH_MPS_HIGH_WATERMARK_RATIO: "0.0"
@@ -172,7 +183,7 @@ jobs:
           poetry run python -m cibuildwheel --output-dir wheelhouse
           echo "step 1"
           ls -l wheelhouse
-          poetry run wheel tags --platform-tag macosx_${{ matrix.os.macos_version }}_0_arm64 ./wheelhouse/*.whl
+          poetry run wheel tags --remove --platform-tag macosx_${{ matrix.os.min_macos_version }}_0_arm64 ./wheelhouse/*.whl
           rm -f ./wheelhouse/*x86_64.whl
           echo "step 2"
           ls -l wheelhouse
@@ -196,17 +207,10 @@ jobs:
         with:
             platforms: all
 
-      - name: Build sdist
-        # build only on Linux to avoid too many duplicates of the sdist
-        if: matrix.os.name == 'ubuntu-latest' 
-        run: |
-          echo "Building wheel ${CIBW_BUILD}"
-          poetry build -f sdist
-
       - name: Build wheels [linux]
         if: matrix.os.name == 'ubuntu-latest'
         env:
-          ## CIBW_BUILD: ${{ env.python_cp_version }}-${{ matrix.os.platform_id }}
+          # CIBW_BUILD: ${{ env.python_cp_version }}-${{ matrix.os.platform_id }}
           CIBW_ARCHS: auto x86_64 aarch64
           CIBW_PLATFORM: linux
           CIBW_SKIP: "pp* *musllinux_* *_i686* *_s390* *pypy*"
@@ -303,12 +307,62 @@ jobs:
           poetry install --no-interaction --no-root --only=test
           poetry run python -c 'from deepsearch_glm import andromeda_glm'
           poetry run pytest ./tests/test_glm.py -v
-          
-      - name: Publish wheels (dry run)
-        run: |
-          poetry publish --skip-existing --dry-run --no-interaction -vvv
 
-      - name: Publish wheels (on publishing) [for releases only]
-        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions-py${{ matrix.python-version }}-${{ matrix.os.platform_id }}
+          path: dist/
+
+  publish-packages:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: inputs.publish && startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+      - build_sdist
+      - build_wheels
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/deepsearch-glm
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+      contents: write  # IMPORTANT: mandatory for adding artifacts to GitHub Releases
+    steps:
+      - name: Download all the dists
+        uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
+          path: dist/
+      - name: List dist/
         run: |
-          poetry publish --skip-existing --no-interaction -vvv
+          ls ./dist
+      - name: Sign the dists with Sigstore
+        uses: sigstore/[email protected]
+        with:
+          release-signing-artifacts: false
+          inputs: >-
+            ./dist/*.tar.gz
+            ./dist/*.whl
+      - name: List dist/
+        run: |
+          ls ./dist
+      - name: Upload artifact signatures to GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        # Upload to GitHub Release using the `gh` CLI.
+        # `dist/` contains the built packages, and the
+        # sigstore-produced signatures and certificates.
+        run: >-
+          gh release upload
+          '${{ github.ref_name }}' dist/**
+          --repo '${{ github.repository }}'
+      # PyPI does not accept .sigstore artifacts and
+      # gh-action-pypi-publish has no option to ignore them.
+      - name: Remove sigstore signatures before uploading to PyPI
+        run: rm ./dist/*.sigstore.json
+      - name: Publish distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          # currently not working with reusable workflows
+          attestations: false