From 5a48a5ff830404e7ddfb1c5280727e681269ba9a Mon Sep 17 00:00:00 2001 From: Max Dubrinsky Date: Fri, 26 Jun 2026 13:51:09 -0400 Subject: [PATCH] fix(python): include generated proto stubs in Linux wheels The generated protobuf/gRPC modules under python/openshell/_proto/ are gitignored. maturin honors .gitignore when collecting python-source files from a git checkout, so native builds (Linux release CI, local pip install .) dropped these modules and shipped an unimportable wheel. The macOS wheel only included them by accident: it builds in a Docker context whose COPY omits .git, so maturin had no gitignore to apply. Pin the stubs back in with explicit, package-relative [tool.maturin].include globs so inclusion no longer depends on whether .git is present in the build context. Add a wheel smoke check to the release-tag and release-dev workflows that installs each Linux wheel in a clean python:3.12-slim image and imports openshell.sandbox, and document the invariant in architecture/build.md. Closes #1705 Signed-off-by: Max Dubrinsky --- .github/workflows/release-dev.yml | 33 ++++++++++++++++++++++++++++- .github/workflows/release-tag.yml | 35 ++++++++++++++++++++++++++++++- architecture/build.md | 10 +++++++++ pyproject.toml | 7 +++++++ 4 files changed, 83 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index dbf28dce3..6d8c4d2a5 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -679,7 +679,7 @@ jobs: smoke-linux-dev-artifacts: name: Smoke Linux Dev Artifacts (${{ matrix.name }}) - needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm] + needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm, build-python-wheels-linux] timeout-minutes: 20 strategy: fail-fast: false @@ -709,6 +709,18 @@ jobs: kind: rpm artifact_arch: arm64 rpm_arch: aarch64 + - name: python-wheel-amd64 + runner: linux-amd64-cpu8 + image: python:3.12-slim + kind: wheel + artifact_arch: amd64 + rpm_arch: x86_64 + - name: python-wheel-arm64 + runner: linux-arm64-cpu8 + image: python:3.12-slim + kind: wheel + artifact_arch: arm64 + rpm_arch: aarch64 runs-on: ${{ matrix.runner }} container: image: ${{ matrix.image }} @@ -743,6 +755,25 @@ jobs: dnf install -y ./package-input/openshell-[0-9]*.rpm ./package-input/openshell-gateway-*.rpm LD_BIND_NOW=1 openshell-gateway --version + - name: Download Python wheel artifact + if: matrix.kind == 'wheel' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: python-wheels-linux-${{ matrix.artifact_arch }} + path: wheel-input/ + + - name: Smoke Python wheel + if: matrix.kind == 'wheel' + run: | + set -euo pipefail + pip install --no-cache-dir wheel-input/*.whl + python - <<'PY' + import importlib + for name in ("openshell", "openshell._proto", "openshell.sandbox"): + importlib.import_module(name) + print(name, "OK") + PY + # --------------------------------------------------------------------------- # Create / update the dev GitHub Release with CLI, gateway, driver, and wheels # --------------------------------------------------------------------------- diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 02cf0ee19..3ab070fd8 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -715,7 +715,7 @@ jobs: smoke-linux-release-artifacts: name: Smoke Linux Release Artifacts (${{ matrix.name }}) - needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm] + needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm, build-python-wheels-linux] timeout-minutes: 20 strategy: fail-fast: false @@ -763,6 +763,20 @@ jobs: artifact_arch: arm64 rpm_arch: aarch64 target: aarch64-unknown-linux-gnu + - name: python-wheel + runner: linux-amd64-cpu8 + image: python:3.12-slim + kind: wheel + artifact_arch: amd64 + rpm_arch: x86_64 + target: x86_64-unknown-linux-gnu + - name: python-wheel-arm64 + runner: linux-arm64-cpu8 + image: python:3.12-slim + kind: wheel + artifact_arch: arm64 + rpm_arch: aarch64 + target: aarch64-unknown-linux-gnu runs-on: ${{ matrix.runner }} container: image: ${{ matrix.image }} @@ -822,6 +836,25 @@ jobs: dnf install -y ./package-input/openshell-[0-9]*.rpm ./package-input/openshell-gateway-*.rpm LD_BIND_NOW=1 openshell-gateway --version + - name: Download Python wheel artifact + if: matrix.kind == 'wheel' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: python-wheels-linux-${{ matrix.artifact_arch }} + path: wheel-input/ + + - name: Smoke Python wheel + if: matrix.kind == 'wheel' + run: | + set -euo pipefail + pip install --no-cache-dir wheel-input/*.whl + python - <<'PY' + import importlib + for name in ("openshell", "openshell._proto", "openshell.sandbox"): + importlib.import_module(name) + print(name, "OK") + PY + # --------------------------------------------------------------------------- # Create a tagged GitHub Release with CLI, gateway, driver, and wheels # --------------------------------------------------------------------------- diff --git a/architecture/build.md b/architecture/build.md index 6cd7b15d2..a3cb2e25f 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -126,6 +126,16 @@ contexts use `KIND_EXPERIMENTAL_PROVIDER=docker|podman` when set, and ambiguous or unknown contexts require an explicit `CONTAINER_ENGINE`. Other image builds do not infer from kube context. +## Python Wheel Packaging + +The generated protobuf/gRPC stubs under `python/openshell/_proto/` are gitignored +build outputs of `mise run python:proto`. maturin honors `.gitignore` when +collecting `python-source` files, so native builds (Linux CI, local +`pip install .`) would drop them and ship an unimportable wheel. `pyproject.toml` +pins them back in with `[tool.maturin].include` globs. The release workflows +install each Linux wheel in a clean image and import `openshell.sandbox` as a +smoke check. + ## CI and E2E Required checks run on GitHub Actions. Workflows that use NVIDIA self-hosted runners trigger from copy-pr-bot mirror branches, so trusted PRs are mirrored into `pull-request/` branches before those workflows run. diff --git a/pyproject.toml b/pyproject.toml index b45081e3d..dab81c57f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,13 @@ bindings = "bin" manifest-path = "crates/openshell-cli/Cargo.toml" python-source = "python" module-name = "openshell" +# _proto/ stubs are gitignored; force them into the wheel so git-aware +# maturin builds don't drop them (package-relative paths). +include = [ + "openshell/_proto/*_pb2.py", + "openshell/_proto/*_pb2_grpc.py", + "openshell/_proto/*.pyi", +] [tool.ruff] target-version = "py312"