Test enable groups as specified by PR labels

.circleci/config.yml

@@ -1,24 +1,41 @@
-version: 2
+version: 2.1
+
+commands:
+  cibw_prepare_environment:
+    description: "Prepare the environment for testing."
+    steps:
+      - run:
+          name: Prepare the environment.
+          command: bash .circleci/prepare.sh
+  cibw_run_tests:
+    description: "Runs tests, with CIBW_ENABLE=all on the main branch"
+    steps:
+      - run:
+          name: Test
+          command: |
+            if [ "${CIRCLE_BRANCH}" == "main" ]; then
+              echo "INFO: Exporting CIBW_ENABLE=all for main branch test run."
+              export CIBW_ENABLE=all
+            else
+              echo "INFO: CIBW_ENABLE not set for this branch test run."
+            fi
+
+            venv/bin/python ./bin/run_tests.py
+          no_output_timeout: 30m
 
 jobs:
-  osx-python3.12:
+  osx-python312:
     macos:
       xcode: 15.4.0
     resource_class: macos.m1.medium.gen1
     environment:
       PYTHON: python3
     steps:
       - checkout
+      - cibw_prepare_environment
+      - cibw_run_tests
 
-      - run:
-          name: Prepare the environment.
-          command: bash .circleci/prepare.sh
-      - run:
-          name: Test.
-          command: venv/bin/python ./bin/run_tests.py
-          no_output_timeout: 30m
-
-  linux-python3.12:
+  linux-python312:
     docker:
       - image: cimg/python:3.12
     environment:
@@ -29,14 +46,8 @@ jobs:
     steps:
       - checkout
       - setup_remote_docker
-
-      - run:
-          name: Prepare the environment.
-          command: bash .circleci/prepare.sh
-      - run:
-          name: Test.
-          command: venv/bin/python ./bin/run_tests.py
-          no_output_timeout: 30m
+      - cibw_prepare_environment
+      - cibw_run_tests
 
   linux-aarch64:
     machine:
@@ -49,19 +60,13 @@ jobs:
       PYTEST_ADDOPTS: -k "unit_test or main_tests or test_0_basic or test_docker_images"
     steps:
       - checkout
-
-      - run:
-          name: Prepare the environment.
-          command: bash .circleci/prepare.sh
-      - run:
-          name: Test.
-          command: venv/bin/python ./bin/run_tests.py
-          no_output_timeout: 30m
+      - cibw_prepare_environment
+      - cibw_run_tests
 
 workflows:
   version: 2
   all-tests:
     jobs:
-      - osx-python3.12
-      - linux-python3.12
+      - osx-python312
+      - linux-python312
       - linux-aarch64

.github/workflows/test.yml

@@ -6,6 +6,11 @@ on:
       - main
       - 2.x
   pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
     paths-ignore:
       - 'docs/**'
       - .pre-commit-config.yaml
@@ -68,6 +73,26 @@ jobs:
       run: |
         uv sync --no-dev --group test
 
+    - uses: joerick/[email protected]
+    - name: Set CIBW_ENABLE
+      shell: bash
+      run: |
+        if [[ "${{ github.ref_name }}" == "main" ]]; then
+          CIBW_ENABLE=all
+        else
+          # get the default CIBW_ENABLE value from the test module
+          CIBW_ENABLE=$(uv run --no-sync python -c 'import sys, test.conftest as c; sys.stdout.write(c.DEFAULT_CIBW_ENABLE)')
+
+          # if this is a PR, check for labels
+          if [[ -n "$GITHUB_PR_LABEL_CI_PYPY" ]]; then
+            CIBW_ENABLE+=" pypy"
+          fi
+          if [[ -n "$GITHUB_PR_LABEL_CI_GRAALPY" ]]; then
+            CIBW_ENABLE+=" graalpy"
+          fi
+        fi
+        echo "CIBW_ENABLE=${CIBW_ENABLE}" >> $GITHUB_ENV
+
     - name: Generate a sample project
       run: |
         uv run --no-sync -m test.test_projects test.test_0_basic.basic_project sample_proj
@@ -80,7 +105,6 @@ jobs:
       env:
         CIBW_ARCHS_MACOS: x86_64 universal2 arm64
         CIBW_BUILD_FRONTEND: 'build[uv]'
-        CIBW_ENABLE: "cpython-prerelease cpython-freethreading pypy graalpy"
 
     - name: Run a sample build (GitHub Action, only)
       uses: ./

.gitlab-ci.yml

@@ -13,6 +13,10 @@ linux:
     # skip all but the basic tests
     # (comment the below line in a PR to debug a Gitlab-specific issue)
     PYTEST_ADDOPTS: -k "unit_test or test_0_basic" --suppress-no-test-exit-code
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      variables:
+        CIBW_ENABLE: "all"
   script:
     - curl -sSL https://get.docker.com/ | sh
     - docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
@@ -26,6 +30,10 @@ windows:
     PYTEST_ADDOPTS: -k "unit_test or test_0_basic" --suppress-no-test-exit-code
   before_script:
     - choco install python -y --version 3.12.4
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      variables:
+        CIBW_ENABLE: "all"
   script:
     - py -m pip install dependency-groups
     - py -m pip install -e. pytest-custom-exit-code $(py -m dependency_groups test)
@@ -37,6 +45,10 @@ macos:
   image: macos-14-xcode-15
   variables:
     PYTEST_ADDOPTS: -k "unit_test or test_0_basic" --suppress-no-test-exit-code
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      variables:
+        CIBW_ENABLE: "all"
   script:
     - python3 -m pip install dependency-groups
     - python3 -m dependency_groups test | xargs python3 -m pip install -e. pytest-custom-exit-code

.travis.yml

@@ -14,13 +14,17 @@ jobs:
     - name: Linux | x86_64 + i686 | Python 3.12
       python: 3.12
       services: docker
-      env: PYTHON=python
+      env:
+        - PYTHON=python
+        - CIBW_ENABLE=all
 
     - name: Linux | arm64 | Python 3.12
       python: 3.12
       services: docker
       arch: arm64
-      env: PYTHON=python
+      env:
+        - PYTHON=python
+        - CIBW_ENABLE=all
 
     - name: Linux | ppc64le | Python 3.12
       python: 3.12
@@ -32,6 +36,7 @@ jobs:
         # skip test_manylinuxXXXX_only, it uses too much disk space
         # c.f. https://travis-ci.community/t/running-out-of-disk-space-quota-when-using-docker-on-ppc64le/11634
         - PYTEST_ADDOPTS='-k "not test_manylinuxXXXX_only"'
+        - CIBW_ENABLE=all
 
     - name: Windows | x86_64 | Python 3.12
       os: windows
@@ -40,13 +45,16 @@ jobs:
         - choco upgrade python3 -y --version 3.12.8 --limit-output --params "/InstallDir:C:\\Python312"
       env:
         - PYTHON=C:\\Python312\\python
+        - CIBW_ENABLE=all
 
     - name: Linux | s390x | Python 3.12
       python: 3.12
       services: docker
       arch: s390x
       allow_failure: True
-      env: PYTHON=python
+      env:
+        - PYTHON=python
+        - CIBW_ENABLE=all
 
 install:
 - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all; fi

azure-pipelines.yml

@@ -16,6 +16,12 @@ jobs:
         docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
         python -m pip install dependency-groups
         python -m dependency_groups test | xargs python -m pip install -e.
+        if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then
+          echo "INFO: Exporting CIBW_ENABLE=all for main branch test run."
+          export CIBW_ENABLE=all
+        else
+          echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))."
+        fi
         python ./bin/run_tests.py
 
 - job: macos_311
@@ -28,6 +34,12 @@ jobs:
     - bash: |
         python -m pip install dependency-groups
         python -m dependency_groups test | xargs python -m pip install -e.
+        if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then
+          echo "INFO: Exporting CIBW_ENABLE=all for main branch test run."
+          export CIBW_ENABLE=all
+        else
+          echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))."
+        fi
         python ./bin/run_tests.py
 
 - job: windows_311
@@ -40,4 +52,10 @@ jobs:
     - bash: |
         python -m pip install dependency-groups
         python -m dependency_groups test | xargs python -m pip install -e.
+        if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then
+          echo "INFO: Exporting CIBW_ENABLE=all for main branch test run."
+          export CIBW_ENABLE=all
+        else
+          echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))."
+        fi
         python ./bin/run_tests.py

cibuildwheel/options.py

@@ -634,8 +634,10 @@ def globals(self) -> GlobalOptions:
             "enable", env_plat=False, option_format=ListFormat(sep=" "), env_rule=InheritRule.APPEND
         )
         try:
-            enable = {EnableGroup(group) for group in enable_groups.split()}
-            enable.update(EnableGroup(command_line_group) for command_line_group in args.enable)
+            enable = {
+                *EnableGroup.parse_option_value(enable_groups),
+                *EnableGroup.parse_option_value(" ".join(args.enable)),
+            }
         except ValueError as e:
             msg = f"Failed to parse enable group. {e}. Valid group names are: {', '.join(g.value for g in EnableGroup)}"
             raise errors.ConfigurationError(msg) from e

cibuildwheel/selector.py

@@ -39,6 +39,23 @@ class EnableGroup(StrEnum):
     def all_groups(cls) -> frozenset["EnableGroup"]:
         return frozenset(cls)
 
+    @classmethod
+    def parse_option_value(cls, value: str) -> frozenset["EnableGroup"]:
+        """
+        Parses a string of space-separated values into a set of EnableGroup
+        members. The string may contain group names or "all".
+        """
+        result = set()
+        for group in value.strip().split():
+            if group == "all":
+                return cls.all_groups()
+            try:
+                result.add(cls(group))
+            except ValueError:
+                msg = f"Unknown enable group: {group}"
+                raise ValueError(msg) from None
+        return frozenset(result)
+
 
 @dataclass(frozen=True, kw_only=True)
 class BuildSelector:

docs/contributing.md

@@ -82,6 +82,8 @@ A few notes-
 
 - Running the macOS integration tests requires _system installs_ of Python from python.org for all the versions that are tested. We won't attempt to install these when running locally, but you can do so manually using the URL in the error message that is printed when the install is not found.
 
+- The 'enable groups' run by default are just 'cpython-prerelease' and 'cpython-freethreading'. You can add other groups like pypy or graalpy by setting the [CIBW_ENABLE](options.md#enable) environment variable. On GitHub PRs, you can add a label to the PR to enable these groups.
+
 #### Running pytest directly
 
 More advanced users might prefer to invoke pytest directly. Set up a [dev environment](#setting-up-a-dev-environment), then,

docs/options.md

@@ -324,7 +324,7 @@ values are:
   are disabled by default as they can't be uploaded to PyPI and a PEP will most likely
   be required before this can happen.
 - `graalpy`: Enable GraalPy.
-
+- `all`: Enable all of the above.
 
 !!! caution
     `cpython-prerelease` is provided for testing purposes only. It is not

test/conftest.py

@@ -1,4 +1,5 @@
 import json
+import os
 import subprocess
 from collections.abc import Generator
 
@@ -12,6 +13,9 @@
 
 from .utils import EMULATED_ARCHS, platform
 
+# default to just cpython
+DEFAULT_CIBW_ENABLE = "cpython-freethreading cpython-prerelease cpython-experimental-riscv64"
+
 
 def pytest_addoption(parser: pytest.Parser) -> None:
     parser.addoption(
@@ -29,6 +33,8 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         help="macOS cp38 uses the universal2 installer",
     )
 
+    os.environ.setdefault("CIBW_ENABLE", DEFAULT_CIBW_ENABLE)
+
 
 def docker_warmup(request: pytest.FixtureRequest) -> None:
     machine = request.config.getoption("--run-emulation", default=None)

test/test_0_basic.py

@@ -1,8 +1,10 @@
+import os
 import textwrap
 
 import pytest
 
 from cibuildwheel.logger import Logger
+from cibuildwheel.selector import EnableGroup
 
 from . import test_projects, utils
 
@@ -38,11 +40,13 @@ def test(tmp_path, build_frontend_env, capfd):
     expected_wheels = utils.expected_wheels("spam", "0.1.0")
     assert set(actual_wheels) == set(expected_wheels)
 
-    # Verify pip warning not shown
-    captured = capfd.readouterr()
-    for stream in (captured.err, captured.out):
-        assert "WARNING: Running pip as the 'root' user can result" not in stream
-        assert "A new release of pip available" not in stream
+    enable_groups = EnableGroup.parse_option_value(os.environ.get("CIBW_ENABLE", ""))
+    if EnableGroup.GraalPy not in enable_groups:
+        # Verify pip warning not shown
+        captured = capfd.readouterr()
+        for stream in (captured.err, captured.out):
+            assert "WARNING: Running pip as the 'root' user can result" not in stream
+            assert "A new release of pip available" not in stream
 
 
 @pytest.mark.skip(reason="to keep test output clean")
@@ -61,16 +65,19 @@ def test_sample_build(tmp_path, capfd):
             logger.step_end()
 
 
-def test_build_identifiers(tmp_path):
+@pytest.mark.parametrize(
+    "enable_setting", ["", "cpython-prerelease", "pypy", "cpython-freethreading"]
+)
+def test_build_identifiers(tmp_path, enable_setting, monkeypatch):
     project_dir = tmp_path / "project"
     basic_project.generate(project_dir)
 
+    monkeypatch.setenv("CIBW_ENABLE", enable_setting)
+
     # check that the number of expected wheels matches the number of build
     # identifiers
     expected_wheels = utils.expected_wheels("spam", "0.1.0")
-    build_identifiers = utils.cibuildwheel_get_build_identifiers(
-        project_dir, prerelease_pythons=True
-    )
+    build_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir)
     assert len(expected_wheels) == len(build_identifiers), (
         f"{expected_wheels} vs {build_identifiers}"
     )