Add support for constraints, install wheels before pyodide packages (#…

…181)

* strawman api/docs changes for #175

* update PR number

* apply constraints in transaction

* add minimal test

* test set_constraints

* add set_constraints

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* more tests

* handle more emfs cases

* address review comments

* update changelog

* update docs

* handle env markers and additive constraints

* continue gathering requreiments from pyodide packages, reverse pydodide/wheel install order

* fix pr number in changelog

* fix spelling

* tighten up utils, docstrings

* skip an await if not needed, rework changelog entry

* try more url checking

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

CHANGELOG.md

@@ -19,6 +19,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - `micropip` now vendors `pypa/packaging` for better reliability.
   [#178](https://github.com/pyodide/micropip/pull/178)
+- `micropip.install` adds optional `constraints`, similar to `pip install -c`,
+  which refine the version or direct URLs of requested packages and their
+  dependencies. This includes built-in packages, which are now installed after
+  any requested or constrained external packages.
+  [#181](https://github.com/pyodide/micropip/pull/181)
+- `micropip.set_constraints` sets default constraints for later
+  calls to `micropip.install` that do not specify constraints.
+  [#181](https://github.com/pyodide/micropip/pull/181)
 
 ## [0.8.0] - 2024/12/15
 

docs/project/usage.md

@@ -71,18 +71,79 @@ You can pass multiple packages to `micropip.install`:
 await micropip.install(["pkg1", "pkg2"])
 ```
 
-You can specify additional constraints:
+A dependency can be refined as per the [PEP-508] spec:
+
+[pep-508]: https://peps.python.org/pep-0508
 
 ```python
 await micropip.install("snowballstemmer==2.2.0")
 await micropip.install("snowballstemmer>=2.2.0")
+await micropip.install("snowballstemmer @ https://.../snowballstemmer.*.whl")
 await micropip.install("snowballstemmer[all]")
 ```
 
+### Disabling dependency resolution
+
 micropip does dependency resolution by default, but you can disable it,
 this is useful if you want to install a package that has a dependency
 which is not a pure Python package, but it is not mandatory for your use case:
 
 ```python
 await micropip.install("pkg", deps=False)
 ```
+
+### Constraining indirect dependencies
+
+Dependency resolution can be further customized with optional `constraints`:
+these modify both _direct_ and _indirect_ dependency resolutions, while direct URLs
+in either a requirement or constraint will bypass any other specifiers.
+
+As described in the [`pip` documentation][pip-constraints], each constraint:
+
+[pip-constraints]: https://pip.pypa.io/en/stable/user_guide/#constraints-files
+
+  - _must_ provide a name
+  - _must_ provide exactly one of
+    - a set of version specifiers
+    - a URL
+  - _must not_  request any `[extras]`
+
+Multiple constraints of the same canonical name are merged.
+
+Invalid constraints will be silently discarded, or logged if `verbose` is provided.
+
+```python
+await micropip.install(
+    "pkg",
+    constraints=[
+        "other-pkg==0.1.1",
+        "some-other-pkg<2",
+        "some-other-pkg<3",                           # merged with the above
+        "yet-another-pkg@https://example.com/yet_another_pkg-0.1.2-py3-none-any.whl",
+        # silently discarded                          # why?
+        "yet-another-pkg >=1",                        # previously defined by URL
+        "yet_another_pkg-0.1.2-py3-none-any.whl",     # missing name
+        "something-completely[different] ==0.1.1",    # extras
+        "package-with-no-version",                    # missing version or URL
+        "other-pkg ==0.0.1 ; python_version < '3'",   # not applicable
+    ]
+)
+```
+
+Over-constrained requirements will fail to resolve, leaving the environment unmodified.
+
+```python
+await micropip.install("pkg ==1", constraints=["pkg ==2"])
+# ValueError: Can't find a pure Python 3 wheel for 'pkg==1,==2'.
+```
+
+### Setting default constraints
+
+`micropip.set_constraints` replaces any default constraints for all subsequent
+calls to `micropip.install` that don't specify `constraints`:
+
+```python
+micropip.set_constraints(["other-pkg ==0.1.1"])
+await micropip.install("pkg")                         # uses defaults, if needed
+await micropip.install("another-pkg", constraints=[]) # ignores defaults
+```

micropip/__init__.py

@@ -9,6 +9,7 @@
 
 install = _package_manager_singleton.install
 set_index_urls = _package_manager_singleton.set_index_urls
+set_constraints = _package_manager_singleton.set_constraints
 list = _package_manager_singleton.list
 freeze = _package_manager_singleton.freeze
 add_mock_package = _package_manager_singleton.add_mock_package

micropip/_utils.py

@@ -5,7 +5,10 @@
 from sysconfig import get_config_var, get_platform
 
 from ._compat import REPODATA_PACKAGES
-from ._vendored.packaging.src.packaging.requirements import Requirement
+from ._vendored.packaging.src.packaging.requirements import (
+    InvalidRequirement,
+    Requirement,
+)
 from ._vendored.packaging.src.packaging.tags import Tag
 from ._vendored.packaging.src.packaging.tags import sys_tags as sys_tags_orig
 from ._vendored.packaging.src.packaging.utils import (
@@ -273,3 +276,91 @@ def fix_package_dependencies(
     (get_dist_info(dist) / "PYODIDE_REQUIRES").write_text(
         json.dumps(sorted(x for x in depends))
     )
+
+
+def validate_constraints(
+    constraints: list[str] | None,
+    environment: dict[str, str] | None = None,
+) -> tuple[dict[str, Requirement], dict[str, list[str]]]:
+    """Build a validated ``Requirement`` dictionary from raw constraint strings.
+
+    Parameters
+    ----------
+    constraints (list):
+        A list of PEP-508 dependency specs, expected to contain both a package
+        name and at least one specifier.
+
+    environment (optional dict):
+        The markers for the current environment, such as OS, Python implementation.
+        If ``None``, the current execution environment will be used.
+
+    Returns
+    -------
+        A 2-tuple of:
+        - a dictionary of ``Requirement`` objects, keyed by canonical name
+        - a dictionary of message strings, keyed by constraint
+    """
+    reqs: dict[str, Requirement] = {}
+    all_messages: dict[str, list[str]] = {}
+
+    for raw_constraint in constraints or []:
+        messages: list[str] = []
+
+        try:
+            req = Requirement(raw_constraint)
+            req.name = canonicalize_name(req.name)
+        except InvalidRequirement as err:
+            all_messages[raw_constraint] = [f"failed to parse: {err}"]
+            continue
+
+        if req.extras:
+            messages.append("may not provide [extras]")
+
+        if not (req.url or len(req.specifier)):
+            messages.append("no version or URL")
+
+        if req.marker and not req.marker.evaluate(environment):
+            messages.append(f"not applicable: {req.marker}")
+
+        if messages:
+            all_messages[raw_constraint] = messages
+        elif req.name in reqs:
+            all_messages[raw_constraint] = [
+                f"updated existing constraint for {req.name}"
+            ]
+            reqs[req.name] = constrain_requirement(req, reqs)
+        else:
+            reqs[req.name] = req
+
+    return reqs, all_messages
+
+
+def constrain_requirement(
+    requirement: Requirement, constrained_requirements: dict[str, Requirement]
+) -> Requirement:
+    """Modify or replace a requirement based on a set of constraints.
+
+    Parameters
+    ----------
+    requirement (Requirement):
+        A ``Requirement`` to constrain.
+
+    constrained_requirements (dict):
+        A dictionary of ``Requirement`` objects, keyed by canonical name.
+
+    Returns
+    -------
+        A constrained ``Requirement``.
+    """
+    # URLs cannot be merged
+    if requirement.url:
+        return requirement
+
+    constrained = constrained_requirements.get(canonicalize_name(requirement.name))
+
+    if constrained:
+        if constrained.url:
+            return constrained
+        requirement.specifier = requirement.specifier & constrained.specifier
+
+    return requirement

micropip/install.py

@@ -1,8 +1,6 @@
 import asyncio
 import importlib
-from collections.abc import Coroutine
 from pathlib import Path
-from typing import Any
 
 from ._compat import loadPackage, to_js
 from ._vendored.packaging.src.packaging.markers import default_environment
@@ -19,6 +17,7 @@ async def install(
     credentials: str | None = None,
     pre: bool = False,
     *,
+    constraints: list[str] | None = None,
     verbose: bool | int | None = None,
 ) -> None:
     with setup_logging().ctx_level(verbose) as logger:
@@ -48,6 +47,7 @@ async def install(
             fetch_kwargs=fetch_kwargs,
             verbose=verbose,
             index_urls=index_urls,
+            constraints=constraints,
         )
         await transaction.gather_requirements(requirements)
 
@@ -58,40 +58,33 @@ async def install(
                 f"See: {FAQ_URLS['cant_find_wheel']}\n"
             )
 
-        package_names = [pkg.name for pkg in transaction.pyodide_packages] + [
-            pkg.name for pkg in transaction.wheels
-        ]
+        pyodide_packages, wheels = transaction.pyodide_packages, transaction.wheels
+
+        package_names = [pkg.name for pkg in wheels + pyodide_packages]
 
         logger.debug(
             "Installing packages %r and wheels %r ",
             transaction.pyodide_packages,
             [w.filename for w in transaction.wheels],
         )
+
         if package_names:
             logger.info("Installing collected packages: %s", ", ".join(package_names))
 
-        wheel_promises: list[Coroutine[Any, Any, None] | asyncio.Task[Any]] = []
+        # Install PyPI packages
+        # detect whether the wheel metadata is from PyPI or from custom location
+        # wheel metadata from PyPI has SHA256 checksum digest.
+        await asyncio.gather(*(wheel.install(wheel_base) for wheel in wheels))
+
         # Install built-in packages
-        pyodide_packages = transaction.pyodide_packages
-        if len(pyodide_packages):
+        if pyodide_packages:
             # Note: branch never happens in out-of-browser testing because in
             # that case REPODATA_PACKAGES is empty.
-            wheel_promises.append(
-                asyncio.ensure_future(
-                    loadPackage(to_js([name for [name, _, _] in pyodide_packages]))
-                )
+            await asyncio.ensure_future(
+                loadPackage(to_js([name for [name, _, _] in pyodide_packages]))
             )
 
-        # Now install PyPI packages
-        # detect whether the wheel metadata is from PyPI or from custom location
-        # wheel metadata from PyPI has SHA256 checksum digest.
-        wheel_promises.extend(wheel.install(wheel_base) for wheel in transaction.wheels)
-
-        await asyncio.gather(*wheel_promises)
-
-        packages = [
-            f"{pkg.name}-{pkg.version}" for pkg in transaction.pyodide_packages
-        ] + [f"{pkg.name}-{pkg.version}" for pkg in transaction.wheels]
+        packages = [f"{pkg.name}-{pkg.version}" for pkg in pyodide_packages + wheels]
 
         if packages:
             logger.info("Successfully installed %s", ", ".join(packages))

micropip/package_manager.py

@@ -26,6 +26,7 @@ def __init__(self) -> None:
 
         self.repodata_packages: dict[str, dict[str, Any]] = REPODATA_PACKAGES
         self.repodata_info: dict[str, str] = REPODATA_INFO
+        self.constraints: list[str] = []
 
         pass
 
@@ -38,6 +39,7 @@ async def install(
         pre: bool = False,
         index_urls: list[str] | str | None = None,
         *,
+        constraints: list[str] | None = None,
         verbose: bool | int | None = None,
     ):
         """Install the given package and all of its dependencies.
@@ -122,6 +124,14 @@ async def install(
             - If a list of URLs is provided, micropip will try each URL in order until
             it finds a package. If no package is found, an error will be raised.
 
+        constraints :
+
+            A list of requirements with versions/URLs which will be used only if
+            needed by any ``requirements``.
+
+            Unlike ``requirements``, the package name _must_ be provided in the
+            PEP-508 format e.g. ``pkgname@https://...``.
+
         verbose :
             Print more information about the process. By default, micropip does not
             change logger level. Setting ``verbose=True`` will print similar
@@ -130,13 +140,17 @@ async def install(
         if index_urls is None:
             index_urls = self.index_urls
 
+        if constraints is None:
+            constraints = self.constraints
+
         return await install(
             requirements,
             index_urls,
             keep_going,
             deps,
             credentials,
             pre,
+            constraints=constraints,
             verbose=verbose,
         )
 
@@ -309,3 +323,16 @@ def set_index_urls(self, urls: List[str] | str):  # noqa: UP006
             urls = [urls]
 
         self.index_urls = urls[:]
+
+    def set_constraints(self, constraints: List[str]):  # noqa: UP006
+        """
+        Set the default constraints to use when looking up packages.
+
+        Parameters
+        ----------
+        constraints
+            A list of PEP-508 requirements, each of which must include a name and
+            version, but no ``[extras]``.
+        """
+
+        self.constraints = constraints[:]