diff --git a/CHANGELOG.md b/CHANGELOG.md index 13d710c8..2526743a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `micropip` now vendors `pypa/packaging` for better reliability. [#178](https://github.com/pyodide/micropip/pull/178) +- `micropip.install` adds optional `constraints`, similar to `pip install -c`, + which refine the version or direct URLs of requested packages and their + dependencies. This includes built-in packages, which are now installed after + any requested or constrained external packages. + [#181](https://github.com/pyodide/micropip/pull/181) +- `micropip.set_constraints` sets default constraints for later + calls to `micropip.install` that do not specify constraints. + [#181](https://github.com/pyodide/micropip/pull/181) ## [0.8.0] - 2024/12/15 diff --git a/docs/project/usage.md b/docs/project/usage.md index aafeacdc..60e5f663 100644 --- a/docs/project/usage.md +++ b/docs/project/usage.md @@ -71,14 +71,19 @@ You can pass multiple packages to `micropip.install`: await micropip.install(["pkg1", "pkg2"]) ``` -You can specify additional constraints: +A dependency can be refined as per the [PEP-508] spec: + +[pep-508]: https://peps.python.org/pep-0508 ```python await micropip.install("snowballstemmer==2.2.0") await micropip.install("snowballstemmer>=2.2.0") +await micropip.install("snowballstemmer @ https://.../snowballstemmer.*.whl") await micropip.install("snowballstemmer[all]") ``` +### Disabling dependency resolution + micropip does dependency resolution by default, but you can disable it, this is useful if you want to install a package that has a dependency which is not a pure Python package, but it is not mandatory for your use case: @@ -86,3 +91,59 @@ which is not a pure Python package, but it is not mandatory for your use case: ```python await micropip.install("pkg", deps=False) ``` + +### Constraining indirect dependencies + +Dependency resolution can be further customized with optional `constraints`: +these modify both _direct_ and _indirect_ dependency resolutions, while direct URLs +in either a requirement or constraint will bypass any other specifiers. + +As described in the [`pip` documentation][pip-constraints], each constraint: + +[pip-constraints]: https://pip.pypa.io/en/stable/user_guide/#constraints-files + + - _must_ provide a name + - _must_ provide exactly one of + - a set of version specifiers + - a URL + - _must not_ request any `[extras]` + +Multiple constraints of the same canonical name are merged. + +Invalid constraints will be silently discarded, or logged if `verbose` is provided. + +```python +await micropip.install( + "pkg", + constraints=[ + "other-pkg==0.1.1", + "some-other-pkg<2", + "some-other-pkg<3", # merged with the above + "yet-another-pkg@https://example.com/yet_another_pkg-0.1.2-py3-none-any.whl", + # silently discarded # why? + "yet-another-pkg >=1", # previously defined by URL + "yet_another_pkg-0.1.2-py3-none-any.whl", # missing name + "something-completely[different] ==0.1.1", # extras + "package-with-no-version", # missing version or URL + "other-pkg ==0.0.1 ; python_version < '3'", # not applicable + ] +) +``` + +Over-constrained requirements will fail to resolve, leaving the environment unmodified. + +```python +await micropip.install("pkg ==1", constraints=["pkg ==2"]) +# ValueError: Can't find a pure Python 3 wheel for 'pkg==1,==2'. +``` + +### Setting default constraints + +`micropip.set_constraints` replaces any default constraints for all subsequent +calls to `micropip.install` that don't specify `constraints`: + +```python +micropip.set_constraints(["other-pkg ==0.1.1"]) +await micropip.install("pkg") # uses defaults, if needed +await micropip.install("another-pkg", constraints=[]) # ignores defaults +``` diff --git a/micropip/__init__.py b/micropip/__init__.py index c1fea9c9..9b391639 100644 --- a/micropip/__init__.py +++ b/micropip/__init__.py @@ -9,6 +9,7 @@ install = _package_manager_singleton.install set_index_urls = _package_manager_singleton.set_index_urls +set_constraints = _package_manager_singleton.set_constraints list = _package_manager_singleton.list freeze = _package_manager_singleton.freeze add_mock_package = _package_manager_singleton.add_mock_package diff --git a/micropip/_utils.py b/micropip/_utils.py index e9bd886c..e9c6c92b 100644 --- a/micropip/_utils.py +++ b/micropip/_utils.py @@ -5,7 +5,10 @@ from sysconfig import get_config_var, get_platform from ._compat import REPODATA_PACKAGES -from ._vendored.packaging.src.packaging.requirements import Requirement +from ._vendored.packaging.src.packaging.requirements import ( + InvalidRequirement, + Requirement, +) from ._vendored.packaging.src.packaging.tags import Tag from ._vendored.packaging.src.packaging.tags import sys_tags as sys_tags_orig from ._vendored.packaging.src.packaging.utils import ( @@ -273,3 +276,91 @@ def fix_package_dependencies( (get_dist_info(dist) / "PYODIDE_REQUIRES").write_text( json.dumps(sorted(x for x in depends)) ) + + +def validate_constraints( + constraints: list[str] | None, + environment: dict[str, str] | None = None, +) -> tuple[dict[str, Requirement], dict[str, list[str]]]: + """Build a validated ``Requirement`` dictionary from raw constraint strings. + + Parameters + ---------- + constraints (list): + A list of PEP-508 dependency specs, expected to contain both a package + name and at least one specifier. + + environment (optional dict): + The markers for the current environment, such as OS, Python implementation. + If ``None``, the current execution environment will be used. + + Returns + ------- + A 2-tuple of: + - a dictionary of ``Requirement`` objects, keyed by canonical name + - a dictionary of message strings, keyed by constraint + """ + reqs: dict[str, Requirement] = {} + all_messages: dict[str, list[str]] = {} + + for raw_constraint in constraints or []: + messages: list[str] = [] + + try: + req = Requirement(raw_constraint) + req.name = canonicalize_name(req.name) + except InvalidRequirement as err: + all_messages[raw_constraint] = [f"failed to parse: {err}"] + continue + + if req.extras: + messages.append("may not provide [extras]") + + if not (req.url or len(req.specifier)): + messages.append("no version or URL") + + if req.marker and not req.marker.evaluate(environment): + messages.append(f"not applicable: {req.marker}") + + if messages: + all_messages[raw_constraint] = messages + elif req.name in reqs: + all_messages[raw_constraint] = [ + f"updated existing constraint for {req.name}" + ] + reqs[req.name] = constrain_requirement(req, reqs) + else: + reqs[req.name] = req + + return reqs, all_messages + + +def constrain_requirement( + requirement: Requirement, constrained_requirements: dict[str, Requirement] +) -> Requirement: + """Modify or replace a requirement based on a set of constraints. + + Parameters + ---------- + requirement (Requirement): + A ``Requirement`` to constrain. + + constrained_requirements (dict): + A dictionary of ``Requirement`` objects, keyed by canonical name. + + Returns + ------- + A constrained ``Requirement``. + """ + # URLs cannot be merged + if requirement.url: + return requirement + + constrained = constrained_requirements.get(canonicalize_name(requirement.name)) + + if constrained: + if constrained.url: + return constrained + requirement.specifier = requirement.specifier & constrained.specifier + + return requirement diff --git a/micropip/install.py b/micropip/install.py index ac4a0d8d..15cda548 100644 --- a/micropip/install.py +++ b/micropip/install.py @@ -1,8 +1,6 @@ import asyncio import importlib -from collections.abc import Coroutine from pathlib import Path -from typing import Any from ._compat import loadPackage, to_js from ._vendored.packaging.src.packaging.markers import default_environment @@ -19,6 +17,7 @@ async def install( credentials: str | None = None, pre: bool = False, *, + constraints: list[str] | None = None, verbose: bool | int | None = None, ) -> None: with setup_logging().ctx_level(verbose) as logger: @@ -48,6 +47,7 @@ async def install( fetch_kwargs=fetch_kwargs, verbose=verbose, index_urls=index_urls, + constraints=constraints, ) await transaction.gather_requirements(requirements) @@ -58,40 +58,33 @@ async def install( f"See: {FAQ_URLS['cant_find_wheel']}\n" ) - package_names = [pkg.name for pkg in transaction.pyodide_packages] + [ - pkg.name for pkg in transaction.wheels - ] + pyodide_packages, wheels = transaction.pyodide_packages, transaction.wheels + + package_names = [pkg.name for pkg in wheels + pyodide_packages] logger.debug( "Installing packages %r and wheels %r ", transaction.pyodide_packages, [w.filename for w in transaction.wheels], ) + if package_names: logger.info("Installing collected packages: %s", ", ".join(package_names)) - wheel_promises: list[Coroutine[Any, Any, None] | asyncio.Task[Any]] = [] + # Install PyPI packages + # detect whether the wheel metadata is from PyPI or from custom location + # wheel metadata from PyPI has SHA256 checksum digest. + await asyncio.gather(*(wheel.install(wheel_base) for wheel in wheels)) + # Install built-in packages - pyodide_packages = transaction.pyodide_packages - if len(pyodide_packages): + if pyodide_packages: # Note: branch never happens in out-of-browser testing because in # that case REPODATA_PACKAGES is empty. - wheel_promises.append( - asyncio.ensure_future( - loadPackage(to_js([name for [name, _, _] in pyodide_packages])) - ) + await asyncio.ensure_future( + loadPackage(to_js([name for [name, _, _] in pyodide_packages])) ) - # Now install PyPI packages - # detect whether the wheel metadata is from PyPI or from custom location - # wheel metadata from PyPI has SHA256 checksum digest. - wheel_promises.extend(wheel.install(wheel_base) for wheel in transaction.wheels) - - await asyncio.gather(*wheel_promises) - - packages = [ - f"{pkg.name}-{pkg.version}" for pkg in transaction.pyodide_packages - ] + [f"{pkg.name}-{pkg.version}" for pkg in transaction.wheels] + packages = [f"{pkg.name}-{pkg.version}" for pkg in pyodide_packages + wheels] if packages: logger.info("Successfully installed %s", ", ".join(packages)) diff --git a/micropip/package_manager.py b/micropip/package_manager.py index f9cd7108..55a1082d 100644 --- a/micropip/package_manager.py +++ b/micropip/package_manager.py @@ -26,6 +26,7 @@ def __init__(self) -> None: self.repodata_packages: dict[str, dict[str, Any]] = REPODATA_PACKAGES self.repodata_info: dict[str, str] = REPODATA_INFO + self.constraints: list[str] = [] pass @@ -38,6 +39,7 @@ async def install( pre: bool = False, index_urls: list[str] | str | None = None, *, + constraints: list[str] | None = None, verbose: bool | int | None = None, ): """Install the given package and all of its dependencies. @@ -122,6 +124,14 @@ async def install( - If a list of URLs is provided, micropip will try each URL in order until it finds a package. If no package is found, an error will be raised. + constraints : + + A list of requirements with versions/URLs which will be used only if + needed by any ``requirements``. + + Unlike ``requirements``, the package name _must_ be provided in the + PEP-508 format e.g. ``pkgname@https://...``. + verbose : Print more information about the process. By default, micropip does not change logger level. Setting ``verbose=True`` will print similar @@ -130,6 +140,9 @@ async def install( if index_urls is None: index_urls = self.index_urls + if constraints is None: + constraints = self.constraints + return await install( requirements, index_urls, @@ -137,6 +150,7 @@ async def install( deps, credentials, pre, + constraints=constraints, verbose=verbose, ) @@ -309,3 +323,16 @@ def set_index_urls(self, urls: List[str] | str): # noqa: UP006 urls = [urls] self.index_urls = urls[:] + + def set_constraints(self, constraints: List[str]): # noqa: UP006 + """ + Set the default constraints to use when looking up packages. + + Parameters + ---------- + constraints + A list of PEP-508 requirements, each of which must include a name and + version, but no ``[extras]``. + """ + + self.constraints = constraints[:] diff --git a/micropip/transaction.py b/micropip/transaction.py index f830f051..a13a54f6 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -8,8 +8,16 @@ from . import package_index from ._compat import REPODATA_PACKAGES -from ._utils import best_compatible_tag_index, check_compatible -from ._vendored.packaging.src.packaging.requirements import Requirement +from ._utils import ( + best_compatible_tag_index, + check_compatible, + constrain_requirement, + validate_constraints, +) +from ._vendored.packaging.src.packaging.requirements import ( + InvalidRequirement, + Requirement, +) from ._vendored.packaging.src.packaging.utils import canonicalize_name from .constants import FAQ_URLS from .package import PackageMetadata @@ -35,14 +43,23 @@ class Transaction: failed: list[Requirement] = field(default_factory=list) verbose: bool | int | None = None + constraints: list[str] | None = None - def __post_init__(self): + def __post_init__(self) -> None: # If index_urls is None, pyodide-lock.json have to be searched first. # TODO: when PyPI starts to support hosting WASM wheels, this might be deleted. self.search_pyodide_lock_first = ( self.index_urls == package_index.DEFAULT_INDEX_URLS ) + self.constrained_reqs, messages = validate_constraints( + self.constraints, self.ctx + ) + + if self.verbose and messages: + for constraint, msg in messages.items(): + logger.info("Transaction: constraint %s discarded: %s", constraint, msg) + async def gather_requirements( self, requirements: list[str] | list[Requirement], @@ -57,14 +74,24 @@ async def add_requirement(self, req: str | Requirement) -> None: if isinstance(req, Requirement): return await self.add_requirement_inner(req) - if not urlparse(req).path.endswith(".whl"): - return await self.add_requirement_inner(Requirement(req)) + try: + as_req = constrain_requirement(Requirement(req), self.constrained_reqs) + except InvalidRequirement: + as_req = None - # custom download location - wheel = WheelInfo.from_url(req) - check_compatible(wheel.filename) + if as_req: + if as_req.name and len(as_req.specifier): + return await self.add_requirement_inner(as_req) + if as_req.url: + req = as_req.url - await self.add_wheel(wheel, extras=set(), specifier="") + if urlparse(req).path.endswith(".whl"): + # custom download location + wheel = WheelInfo.from_url(req) + check_compatible(wheel.filename) + return await self.add_wheel(wheel, extras=set(), specifier="") + + return await self.add_requirement_inner(Requirement(req)) def check_version_satisfied(self, req: Requirement) -> tuple[bool, str]: ver = None @@ -95,9 +122,12 @@ async def add_requirement_inner( See PEP 508 for a description of the requirements. https://www.python.org/dev/peps/pep-0508 """ + # add [extras] first, as constraints will never add them for e in req.extras: self.ctx_extras.append({"extra": e}) + req = constrain_requirement(req, self.constrained_reqs) + if self.pre: req.specifier.prereleases = True @@ -134,6 +164,7 @@ def eval_marker(e: dict[str, str]) -> bool: eval_marker(e) for e in self.ctx_extras ): return + # Is some version of this package is already installed? req.name = canonicalize_name(req.name) @@ -144,7 +175,7 @@ def eval_marker(e: dict[str, str]) -> bool: try: if self.search_pyodide_lock_first: - if self._add_requirement_from_pyodide_lock(req): + if await self._add_requirement_from_pyodide_lock(req): logger.debug("Transaction: package found in lock file: %r", req) return @@ -160,7 +191,7 @@ def eval_marker(e: dict[str, str]) -> bool: # If the requirement is not found in package index, # we still have a chance to find it from pyodide lockfile. - if not self._add_requirement_from_pyodide_lock(req): + if not await self._add_requirement_from_pyodide_lock(req): logger.debug( "Transaction: package %r not found in lock file", req ) @@ -171,18 +202,21 @@ def eval_marker(e: dict[str, str]) -> bool: if not self.keep_going: raise - def _add_requirement_from_pyodide_lock(self, req: Requirement) -> bool: + async def _add_requirement_from_pyodide_lock(self, req: Requirement) -> bool: """ Find requirement from pyodide-lock.json. If the requirement is found, add it to the package list and return True. Otherwise, return False. """ - if req.name in REPODATA_PACKAGES and req.specifier.contains( + locked_package = REPODATA_PACKAGES.get(req.name) + if locked_package and req.specifier.contains( REPODATA_PACKAGES[req.name]["version"], prereleases=True ): - version = REPODATA_PACKAGES[req.name]["version"] + version = locked_package["version"] self.pyodide_packages.append( PackageMetadata(name=req.name, version=str(version), source="pyodide") ) + if self.constraints and locked_package["depends"]: + await self.gather_requirements(locked_package["depends"]) return True return False diff --git a/tests/conftest.py b/tests/conftest.py index ff24ab97..2091fbfb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,7 @@ import pytest from pytest_httpserver import HTTPServer from pytest_pyodide import spawn_web_server +from pytest_pyodide.runner import JavascriptException from micropip._vendored.packaging.src.packaging.utils import parse_wheel_filename @@ -409,3 +410,45 @@ def mock_package_index_simple_html_api(httpserver): suffix="_simple.html", content_type="text/html", ) + + +@pytest.fixture( + params=[ + None, + "pytest ==7.2.2", + "pytest >=7.2.1,<7.2.3", + "pytest @ {url}", + "pytest @ emfs:{wheel}", + ] +) +def valid_constraint(request, wheel_catalog): + wheel = wheel_catalog.get("pytest") + if not request.param: + return request.param + return request.param.format(url=wheel.url, wheel=wheel.url.split("/")[-1]) + + +INVALID_CONSTRAINT_MESSAGES = { + "": "parse", + "http://example.com": "name", + "a-package[with-extra]": "[extras]", + "a-package": "no version or URL", +} + + +@pytest.fixture(params=[*INVALID_CONSTRAINT_MESSAGES.keys()]) +def invalid_constraint(request): + return request.param + + +@pytest.fixture +def run_async_py_in_js(selenium_standalone_micropip): + def _run(*lines, error_match=None): + js = "\n".join(["await pyodide.runPythonAsync(`", *lines, "`);"]) + if error_match: + with pytest.raises(JavascriptException, match=error_match): + selenium_standalone_micropip.run_js(js) + else: + selenium_standalone_micropip.run_js(js) + + return _run diff --git a/tests/test_install.py b/tests/test_install.py index aa7eca1f..88eb63cd 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -99,6 +99,47 @@ def test_install_mixed_case2(selenium_standalone_micropip, jinja2): ) +@pytest.mark.parametrize("set_constraints", [False, True]) +def test_install_constraints( + set_constraints, + valid_constraint, + wheel_catalog, + run_async_py_in_js, +): + constraints = [valid_constraint] if valid_constraint else [] + run_async_py_in_js("import micropip") + + if valid_constraint and "emfs:" in valid_constraint: + url = wheel_catalog.get("pytest").url + wheel = url.split("/")[-1] + run_async_py_in_js( + "from pyodide.http import pyfetch", + f"resp = await pyfetch('{url}')", + f"await resp._into_file(open('{wheel}', 'wb'))", + ) + + if set_constraints: + run_async_py_in_js(f"micropip.set_constraints({constraints})") + install_args = "" + else: + install_args = f"constraints={constraints}" + + if constraints and "@" not in valid_constraint: + run_async_py_in_js( + f"await micropip.install('pytest ==7.2.3', {install_args})", + error_match="Can't find a pure Python 3 wheel", + ) + + run_async_py_in_js(f"await micropip.install('pytest', {install_args})") + + compare = "==" if constraints else "!=" + + run_async_py_in_js( + "import pytest", + f"assert pytest.__version__ {compare} '7.2.2', pytest.__version__", + ) + + @pytest.mark.asyncio async def test_package_with_extra(mock_fetch): mock_fetch.add_pkg_version("depa") diff --git a/tests/test_utils.py b/tests/test_utils.py index 7d0522bc..dc3c123c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,10 +2,11 @@ from importlib.metadata import distribution import pytest -from conftest import CPVER, EMSCRIPTEN_VER, PLATFORM +from conftest import CPVER, EMSCRIPTEN_VER, INVALID_CONSTRAINT_MESSAGES, PLATFORM from pytest_pyodide import run_in_pyodide import micropip._utils as _utils +from micropip._vendored.packaging.src.packaging.requirements import Requirement def test_get_root(): @@ -150,3 +151,61 @@ def test_best_compatible_tag(package, version, incompatible_tags, compatible_tag sorted_tags = sorted(tags, key=best_compatible_tag_index) sorted_tags.reverse() assert sorted_tags == tags + + +def test_validate_constraints_valid(valid_constraint): + constraints = [valid_constraint] if valid_constraint else [] + reqs, msgs = _utils.validate_constraints(constraints) + assert len(reqs) == len(constraints) + assert not msgs + + if not valid_constraint or "==" not in valid_constraint: + return + + extra_constraint = valid_constraint.replace("==", ">=") + marker_valid_constraint = f"{extra_constraint} ; python_version>'2.7'" + marker_invalid_constraint = f"{extra_constraint} ; python_version<'3'" + constraints += [ + extra_constraint, + marker_valid_constraint, + marker_invalid_constraint, + ] + + reqs, msgs = _utils.validate_constraints(constraints) + assert "updated existing" in f"{msgs[extra_constraint]}" + assert "updated existing" in f"{msgs[marker_valid_constraint]}" + assert "not applicable" in f"{msgs[marker_invalid_constraint]}" + + +def test_validate_constraints_invalid(invalid_constraint): + reqs, msgs = _utils.validate_constraints([invalid_constraint]) + assert not reqs + for constraint, msg in msgs.items(): + assert INVALID_CONSTRAINT_MESSAGES[constraint] in f"{msg}" + + +def test_constrain_requirement(valid_constraint): + req = Requirement("pytest") + constraints = [valid_constraint] if valid_constraint else [] + assert not req.specifier + constrained_reqs, msg = _utils.validate_constraints(constraints) + assert not msg + constrained = _utils.constrain_requirement(req, constrained_reqs) + + if constraints: + assert constrained.specifier or constrained.url + assert not (constrained.specifier and constrained.url) + else: + assert not (constrained.specifier or constrained.url) + + +def test_constrain_requirement_direct_url(valid_constraint, wheel_catalog): + constraints = [valid_constraint] if valid_constraint else [] + wheel = wheel_catalog.get("pytest") + url = f"{wheel.url}?foo" + req = Requirement(f"pytest @ {url}") + assert not req.specifier + constrained_reqs, msg = _utils.validate_constraints(constraints) + assert not msg + constrained = _utils.constrain_requirement(req, constrained_reqs) + assert constrained.url == url