diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 27af90d7..131f61db 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,6 +10,9 @@ concurrency: env: FORCE_COLOR: 3 + # Increase this value to reset cache if emscripten_version has not changed + EMSDK_CACHE_FOLDER: 'emsdk-cache' + EMSDK_CACHE_NUMBER: 0 jobs: test: @@ -89,7 +92,13 @@ jobs: strategy: fail-fast: false matrix: - task: [test-recipe, test-src, test-integration-marker] + task: [ + {name: test-recipe, installer: pip}, + {name: test-src, installer: pip}, + {name: test-recipe, installer: uv}, + {name: test-src, installer: uv}, + {name: test-integration-marker, installer: pip}, # installer doesn't matter + ] os: [ubuntu-latest, macos-latest] if: needs.check-integration-test-trigger.outputs.run-integration-test steps: @@ -111,36 +120,47 @@ jobs: - name: Install the package run: | python -m pip install --upgrade pip - python -m pip install -e ."[test]" + python -m pip install -e ."[test,uv]" - name: Install xbuildenv run: | pyodide xbuildenv install echo EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) >> $GITHUB_ENV + - name: Cache emsdk + uses: actions/cache@v4 + with: + path: ${{ env.EMSDK_CACHE_FOLDER }} + key: ${{ env.EMSDK_CACHE_NUMBER }}-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }} + - name: Install Emscripten uses: mymindstorm/setup-emsdk@6ab9eb1bda2574c4ddb79809fc9247783eaf9021 # v14 with: version: ${{ env.EMSCRIPTEN_VERSION }} + actions-cache-folder: ${{env.EMSDK_CACHE_FOLDER}} - name: Get number of cores on the runner id: get-cores run: echo "CORES=$(nproc)" >> $GITHUB_OUTPUT - name: Run tests marked with integration - if: matrix.task == 'test-integration-marker' + if: matrix.task.name == 'test-integration-marker' run: pytest --junitxml=test-results/junit.xml --cov=pyodide-build pyodide_build -m "integration" - name: Run the recipe integration tests (${{ matrix.task }}) - if: matrix.task != 'test-integration-marker' + if: matrix.task.name != 'test-integration-marker' env: PYODIDE_JOBS: ${{ steps.get-cores.outputs.CORES }} working-directory: integration_tests - run: make ${{ matrix.task }} + run: | + if [[ "${{ matrix.task.installer }}" == "uv" ]]; then + export UV_RUN_PREFIX="uv run" + fi + make ${{ matrix.task.name }} - name: Upload coverage for tests marked with integration uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: matrix.task == 'test-integration-marker' + if: matrix.task.name == 'test-integration-marker' with: name: coverage-from-integration-${{ matrix.os }} path: .coverage diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e1c3468..e0f05a6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added basic support for uv. `uv tool install pyodide-cli --with pyodide-build`, or `uvx --from pyodide-cli --with pyodide-build pyodide --help`, or using `pyodide-build` in `uv`-managed virtual environments will now work. + [#132](https://github.com/pyodide/pyodide-build/pull/132) + ### Changed - The Rust toolchain version has been updated to `nightly-2025-01-18`. diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 75711cd7..31d27a12 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -6,7 +6,7 @@ all: test-recipe: check @echo "... Running integration tests for building recipes" - pyodide build-recipes --recipe-dir=recipes --install --force-rebuild "*" + $(UV_RUN_PREFIX) pyodide build-recipes --recipe-dir=recipes --install --force-rebuild "*" @echo "... Passed" diff --git a/integration_tests/src/numpy.sh b/integration_tests/src/numpy.sh index 3b01adaa..1abe6b44 100755 --- a/integration_tests/src/numpy.sh +++ b/integration_tests/src/numpy.sh @@ -10,4 +10,4 @@ tar -xf numpy-${VERSION}.tar.gz cd numpy-${VERSION} MESON_CROSS_FILE=$(pyodide config get meson_cross_file) -pyodide build -Csetup-args=-Dallow-noblas=true -Csetup-args=--cross-file="${MESON_CROSS_FILE}" +${UV_RUN_PREFIX} pyodide build -Csetup-args=-Dallow-noblas=true -Csetup-args=--cross-file="${MESON_CROSS_FILE}" diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index 845fb4e8..dab500f5 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -15,7 +15,7 @@ from build.env import DefaultIsolatedEnv from packaging.requirements import Requirement -from pyodide_build import _f2c_fixes, common, pywasmcross +from pyodide_build import _f2c_fixes, common, pywasmcross, uv_helper from pyodide_build.build_env import ( get_build_flag, get_hostsitepackages, @@ -149,7 +149,8 @@ def _build_in_isolated_env( # It will be left in the /tmp folder and can be inspected or entered as # needed. # _DefaultIsolatedEnv.__exit__ = lambda self, *args: print("Skipping removing isolated env in", self.path) - with _DefaultIsolatedEnv() as env: + installer = "uv" if uv_helper.should_use_uv() else "pip" + with _DefaultIsolatedEnv(installer=installer) as env: env = cast(_DefaultIsolatedEnv, env) builder = _ProjectBuilder.from_isolated_env( env, diff --git a/pyodide_build/recipe/graph_builder.py b/pyodide_build/recipe/graph_builder.py index f2a8fc95..80a70f6d 100755 --- a/pyodide_build/recipe/graph_builder.py +++ b/pyodide_build/recipe/graph_builder.py @@ -27,7 +27,7 @@ from rich.spinner import Spinner from rich.table import Table -from pyodide_build import build_env +from pyodide_build import build_env, uv_helper from pyodide_build.build_env import BuildArgs from pyodide_build.common import ( download_and_unpack_archive, @@ -130,8 +130,12 @@ def needs_rebuild(self, build_dir: Path) -> bool: return res def build(self, build_args: BuildArgs, build_dir: Path) -> None: + run_prefix = ( + [uv_helper.find_uv_bin(), "run"] if uv_helper.should_use_uv() else [] + ) p = subprocess.run( [ + *run_prefix, "pyodide", "build-recipes-no-deps", self.name, diff --git a/pyodide_build/tests/test_xbuildenv.py b/pyodide_build/tests/test_xbuildenv.py index fca03260..89396629 100644 --- a/pyodide_build/tests/test_xbuildenv.py +++ b/pyodide_build/tests/test_xbuildenv.py @@ -16,7 +16,7 @@ def monkeypatch_subprocess_run_pip(monkeypatch): orig_run = subprocess.run def monkeypatch_func(cmds, *args, **kwargs): - if cmds[0] == "pip": + if cmds[0] == "pip" or cmds[0:3] == [sys.executable, "-m", "pip"]: called_with.extend(cmds) return subprocess.CompletedProcess(cmds, 0, "", "") else: @@ -289,12 +289,20 @@ def test_install_cross_build_packages( xbuildenv_pyodide_root = xbuildenv_root / "pyodide-root" manager._install_cross_build_packages(xbuildenv_root, xbuildenv_pyodide_root) - assert len(pip_called_with) == 7 - assert pip_called_with[0:4] == ["pip", "install", "--no-user", "-t"] - assert pip_called_with[4].startswith( + assert len(pip_called_with) == 9 + assert pip_called_with[0:8] == [ + sys.executable, + "-m", + "pip", + "install", + "--no-user", + "-r", + str(xbuildenv_root / "requirements.txt"), + "--target", + ] + assert pip_called_with[8].startswith( str(xbuildenv_pyodide_root) ) # hostsitepackages - assert pip_called_with[5:7] == ["-r", str(xbuildenv_root / "requirements.txt")] hostsitepackages = manager._host_site_packages_dir(xbuildenv_pyodide_root) assert hostsitepackages.exists() diff --git a/pyodide_build/uv_helper.py b/pyodide_build/uv_helper.py new file mode 100644 index 00000000..52958e0f --- /dev/null +++ b/pyodide_build/uv_helper.py @@ -0,0 +1,28 @@ +import functools +import os +import shutil + + +@functools.cache +def find_uv_bin() -> str | None: + """ + Check if the uv executable is available. + + If the uv executable is available, return the path to the executable. + Otherwise, return None. + """ + try: + import uv + + return uv.find_uv_bin() + except (ModuleNotFoundError, FileNotFoundError): + return shutil.which("uv") + + return None + + +def should_use_uv() -> bool: + # UV environ is set to the uv executable path when the script is called with the uv executable. + uv_environ = os.environ.get("UV") + # double check by comparing the uv executable path with the one found by the uv package. + return uv_environ and uv_environ == find_uv_bin() diff --git a/pyodide_build/xbuildenv.py b/pyodide_build/xbuildenv.py index 38faaabe..3d1dc8cc 100644 --- a/pyodide_build/xbuildenv.py +++ b/pyodide_build/xbuildenv.py @@ -1,11 +1,12 @@ import json import shutil import subprocess +import sys from pathlib import Path from pyodide_lock import PyodideLockSpec -from pyodide_build import build_env +from pyodide_build import build_env, uv_helper from pyodide_build.common import download_and_unpack_archive from pyodide_build.create_package_index import create_package_index from pyodide_build.logger import logger @@ -284,15 +285,30 @@ def _install_cross_build_packages( """ host_site_packages = self._host_site_packages_dir(xbuildenv_pyodide_root) host_site_packages.mkdir(exist_ok=True, parents=True) - result = subprocess.run( + + install_prefix = ( [ + uv_helper.find_uv_bin(), + "pip", + "install", + ] + if uv_helper.should_use_uv() + else [ + sys.executable, + "-m", "pip", "install", "--no-user", - "-t", - str(host_site_packages), + ] + ) + + result = subprocess.run( + [ + *install_prefix, "-r", str(xbuildenv_root / "requirements.txt"), + "--target", + str(host_site_packages), ], capture_output=True, encoding="utf8", diff --git a/pyproject.toml b/pyproject.toml index d327c680..0b183d37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,9 @@ test = [ "pytest-cov", "types-requests", ] +uv = [ + "build[uv]~=1.2.0", +] [tool.hatch.version] source = "vcs"