ci(python): replace custom Dockerfile with PyO3/maturin-action for Linux wheels

phamgialinhlx · phamgialinhlx · commit 3ce0d9ab108e · 2026-05-21T04:06:23.000Z
diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml
@@ -105,59 +105,44 @@ jobs:
           - arch: amd64
             runner: build-amd64
             artifact: linux-amd64
-            task: python:build:linux:amd64
-            output_path: target/wheels/linux-amd64/*.whl
+            target: x86_64-unknown-linux-gnu
           - arch: arm64
             runner: build-arm64
             artifact: linux-arm64
-            task: python:build:linux:arm64
-            output_path: target/wheels/linux-arm64/*.whl
+            target: aarch64-unknown-linux-gnu
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 120
-    container:
-      image: ghcr.io/nvidia/openshell/ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-      options: --privileged
-      volumes:
-        - /var/run/docker.sock:/var/run/docker.sock
-    env:
-      MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }}
-      OPENSHELL_IMAGE_TAG: dev
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - name: Set up Docker Buildx
-        uses: ./.github/actions/setup-buildx
+      - name: Set up mise
+        uses: jdx/mise-action@v2
 
-      - name: Mark workspace safe for git
-        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - name: Generate Python protobuf stubs
+        run: uv sync --group dev && mise run python:proto
 
-      - name: Sync Python dependencies
-        run: uv sync
+      - name: Patch workspace version
+        if: needs.compute-versions.outputs.cargo_version != ''
+        run: |
+          sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "${{ needs.compute-versions.outputs.cargo_version }}"/}' Cargo.toml
 
-      - name: Cache Rust target and registry
-        uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
+      - name: Build Python wheel
+        uses: PyO3/maturin-action@v1
         with:
-          shared-key: python-wheel-linux-${{ matrix.arch }}
-          cache-directories: .cache/sccache
-          cache-targets: "true"
-
-      - name: Build Python wheels
-        run: |
-          set -euo pipefail
-          OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run ${{ matrix.task }}
-          ls -la ${{ matrix.output_path }}
+          target: ${{ matrix.target }}
+          manylinux: 2_28
+          args: --release --features bundled-z3 --compatibility manylinux_2_28 --out dist
+          before-script-linux: |
+            dnf install -y --setopt=install_weak_deps=False \
+              clang llvm-devel openssl-devel perl-core perl-IPC-Cmd
 
       - name: Upload wheel artifacts
         uses: actions/upload-artifact@v4
         with:
           name: python-wheels-${{ matrix.artifact }}
-          path: ${{ matrix.output_path }}
+          path: dist/*.whl
           retention-days: 5
 
   build-python-wheel-macos:
diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml
@@ -125,60 +125,45 @@ jobs:
           - arch: amd64
             runner: build-amd64
             artifact: linux-amd64
-            task: python:build:linux:amd64
-            output_path: target/wheels/linux-amd64/*.whl
+            target: x86_64-unknown-linux-gnu
           - arch: arm64
             runner: build-arm64
             artifact: linux-arm64
-            task: python:build:linux:arm64
-            output_path: target/wheels/linux-arm64/*.whl
+            target: aarch64-unknown-linux-gnu
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 120
-    container:
-      image: ghcr.io/nvidia/openshell/ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-      options: --privileged
-      volumes:
-        - /var/run/docker.sock:/var/run/docker.sock
-    env:
-      MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }}
-      OPENSHELL_IMAGE_TAG: ${{ needs.compute-versions.outputs.semver }}
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.tag || github.ref }}
           fetch-depth: 0
 
-      - name: Set up Docker Buildx
-        uses: ./.github/actions/setup-buildx
+      - name: Set up mise
+        uses: jdx/mise-action@v2
 
-      - name: Mark workspace safe for git
-        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - name: Generate Python protobuf stubs
+        run: uv sync --group dev && mise run python:proto
 
-      - name: Sync Python dependencies
-        run: uv sync
+      - name: Patch workspace version
+        if: needs.compute-versions.outputs.cargo_version != ''
+        run: |
+          sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "${{ needs.compute-versions.outputs.cargo_version }}"/}' Cargo.toml
 
-      - name: Cache Rust target and registry
-        uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
+      - name: Build Python wheel
+        uses: PyO3/maturin-action@v1
         with:
-          shared-key: python-wheel-linux-${{ matrix.arch }}
-          cache-directories: .cache/sccache
-          cache-targets: "true"
-
-      - name: Build Python wheels
-        run: |
-          set -euo pipefail
-          OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run ${{ matrix.task }}
-          ls -la ${{ matrix.output_path }}
+          target: ${{ matrix.target }}
+          manylinux: 2_28
+          args: --release --features bundled-z3 --compatibility manylinux_2_28 --out dist
+          before-script-linux: |
+            dnf install -y --setopt=install_weak_deps=False \
+              clang llvm-devel openssl-devel perl-core perl-IPC-Cmd
 
       - name: Upload wheel artifacts
         uses: actions/upload-artifact@v4
         with:
           name: python-wheels-${{ matrix.artifact }}
-          path: ${{ matrix.output_path }}
+          path: dist/*.whl
           retention-days: 5
 
   build-python-wheel-macos:
diff --git a/architecture/build-containers.md b/architecture/build-containers.md
@@ -43,8 +43,8 @@ Both the standalone artifact and the deployed container image use the `openshell
 
 OpenShell also publishes Python wheels for `linux/amd64`, `linux/arm64`, and macOS ARM64.
 
-- Released Linux wheels are built per-arch inside `deploy/docker/Dockerfile.python-wheels-linux`, which uses the PyPA `manylinux_2_28_{x86_64,aarch64}` images as a base. The resulting wheels are tagged `manylinux_2_28` and install on any Linux distribution shipping glibc >= 2.28 (RHEL 8, Debian 10+, Ubuntu 18.04+). The matrix job invokes `mise run python:build:linux:amd64` / `python:build:linux:arm64`, which fan out to `build:python:wheel:linux:{amd64,arm64}:docker`.
-- For fast local iteration, `build:python:wheel:linux:amd64` / `build:python:wheel:linux:arm64` still produce a non-portable wheel tagged for the host's glibc. These tasks are no longer used by release workflows.
+- Released Linux wheels are built per-arch using `PyO3/maturin-action` with `manylinux: 2_28`. The action pulls the PyPA `manylinux_2_28` container, installs Rust from `rust-toolchain.toml`, and runs `maturin build --features bundled-z3 --compatibility manylinux_2_28`. The resulting wheels install on any Linux with glibc >= 2.28 (RHEL 8+, Ubuntu 18.04+, Debian 10+). This follows the same pattern used by ruff and uv.
+- For fast local iteration, `build:python:wheel:linux:{amd64,arm64}` build natively on the host (wheels tagged for the host glibc, not portable).
 - The macOS ARM64 wheel is cross-compiled with `deploy/docker/Dockerfile.python-wheels-macos` via `build:python:wheel:macos`.
 - Release workflows mirror the CLI layout: a Linux matrix job for amd64/arm64, a separate macOS job, and release jobs that download the per-platform wheel artifacts directly before publishing.
 
diff --git a/deploy/docker/Dockerfile.python-wheels-linux b/deploy/docker/Dockerfile.python-wheels-linux
diff --git a/tasks/python.toml b/tasks/python.toml
@@ -92,84 +92,17 @@ description = "Build Python wheel for Linux arm64 natively (host glibc; non-port
 depends = ["EXPECTED_HOST_ARCH=arm64 WHEEL_OUTPUT_DIR=target/wheels/linux-arm64 build:python:wheel:linux"]
 hide = true
 
-["build:python:wheel:linux:docker"]
-description = "Build a portable manylinux_2_28 Python wheel via Docker (glibc >= 2.28)"
-depends = ["python:proto"]
-run = """
-#!/usr/bin/env bash
-set -euo pipefail
-
-source tasks/scripts/container-engine.sh
-
-WHEEL_OUTPUT_DIR=${WHEEL_OUTPUT_DIR:?Set WHEEL_OUTPUT_DIR to a per-platform wheel output directory}
-TARGETARCH=${TARGETARCH:?Set TARGETARCH to amd64 or arm64}
-
-sha256_16() {
-  if command -v sha256sum >/dev/null 2>&1; then
-    sha256sum "$1" | awk '{print substr($1, 1, 16)}'
-  else
-    shasum -a 256 "$1" | awk '{print substr($1, 1, 16)}'
-  fi
-}
-
-sha256_16_stdin() {
-  if command -v sha256sum >/dev/null 2>&1; then
-    sha256sum | awk '{print substr($1, 1, 16)}'
-  else
-    shasum -a 256 | awk '{print substr($1, 1, 16)}'
-  fi
-}
-
-CARGO_VERSION=${OPENSHELL_CARGO_VERSION:-}
-if [ -z "$CARGO_VERSION" ] && [ -n "${CI:-}" ]; then
-  CARGO_VERSION=$(uv run python tasks/scripts/release.py get-version --cargo)
-fi
-
-LOCK_HASH=$(sha256_16 Cargo.lock)
-RUST_SCOPE=${RUST_TOOLCHAIN_SCOPE:-rustup-1.95.0}
-CACHE_SCOPE_INPUT="v1|python-wheels-linux-${TARGETARCH}|manylinux_2_28|${LOCK_HASH}|${RUST_SCOPE}"
-CARGO_TARGET_CACHE_SCOPE=$(printf '%s' "$CACHE_SCOPE_INPUT" | sha256_16_stdin)
-
-rm -rf "$WHEEL_OUTPUT_DIR"
-mkdir -p "$WHEEL_OUTPUT_DIR"
-
-ce build \
-  -f deploy/docker/Dockerfile.python-wheels-linux \
-  --target wheels \
-  --build-arg "TARGETARCH=${TARGETARCH}" \
-  --build-arg "CARGO_TARGET_CACHE_SCOPE=${CARGO_TARGET_CACHE_SCOPE}" \
-  ${CARGO_VERSION:+--build-arg "OPENSHELL_CARGO_VERSION=${CARGO_VERSION}"} \
-  ${OPENSHELL_IMAGE_TAG:+--build-arg "OPENSHELL_IMAGE_TAG=${OPENSHELL_IMAGE_TAG}"} \
-  --output "type=local,dest=${WHEEL_OUTPUT_DIR}" \
-  .
-
-ls -la "$WHEEL_OUTPUT_DIR"/*.whl
-"""
-hide = true
-
-["build:python:wheel:linux:amd64:docker"]
-description = "Build portable manylinux_2_28 wheel for Linux amd64"
-depends = ["TARGETARCH=amd64 WHEEL_OUTPUT_DIR=target/wheels/linux-amd64 build:python:wheel:linux:docker"]
-hide = true
-
-["build:python:wheel:linux:arm64:docker"]
-description = "Build portable manylinux_2_28 wheel for Linux arm64"
-depends = ["TARGETARCH=arm64 WHEEL_OUTPUT_DIR=target/wheels/linux-arm64 build:python:wheel:linux:docker"]
-hide = true
-
-# Release-pipeline aliases. These produce manylinux_2_28-tagged wheels via
-# Docker so the published artifacts install on any glibc >= 2.28 host (RHEL 8,
-# Ubuntu 18.04+, Debian 10+). The native `build:python:wheel:linux:amd64` /
-# `build:python:wheel:linux:arm64` tasks remain available for fast local
-# iteration and produce wheels tagged for the host glibc only.
+# Release-pipeline aliases. CI uses PyO3/maturin-action directly (see
+# .github/workflows/release-*.yml); these aliases remain for local iteration
+# and produce wheels tagged for the host glibc only.
 ["python:build:linux:amd64"]
-description = "Build portable manylinux_2_28 wheel for Linux amd64 (release path)"
-depends = ["build:python:wheel:linux:amd64:docker"]
+description = "Build Python wheel for Linux amd64 (local dev; CI uses maturin-action)"
+depends = ["build:python:wheel:linux:amd64"]
 hide = true
 
 ["python:build:linux:arm64"]
-description = "Build portable manylinux_2_28 wheel for Linux arm64 (release path)"
-depends = ["build:python:wheel:linux:arm64:docker"]
+description = "Build Python wheel for Linux arm64 (local dev; CI uses maturin-action)"
+depends = ["build:python:wheel:linux:arm64"]
 hide = true
 
 ["build:python:wheel:macos"]
