From 48295cef45370bc588b7548e26592b9b15875ed4 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Thu, 23 Jan 2025 09:44:08 -0600 Subject: [PATCH 01/19] strawman api/docs changes for #175 --- CHANGELOG.md | 6 ++++++ docs/project/usage.md | 24 +++++++++++++++++++++++- micropip/install.py | 2 ++ micropip/package_manager.py | 14 ++++++++++++++ micropip/transaction.py | 1 + 5 files changed, 46 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81404672..5c522a6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added support for constraining indirect dependencies via + `micropip.install(..., constraints=[...])`. and `micropip.set_constraints([...])` + [#xyz](https://github.com/pyodide/micropip/pull/xyz) + ### Fixed - Fix a bug that prevented non-standard relative urls to be treated as such diff --git a/docs/project/usage.md b/docs/project/usage.md index aafeacdc..201a058e 100644 --- a/docs/project/usage.md +++ b/docs/project/usage.md @@ -71,7 +71,8 @@ You can pass multiple packages to `micropip.install`: await micropip.install(["pkg1", "pkg2"]) ``` -You can specify additional constraints: +A dependency can specify be refined per the [PEP-508] spec: + ```python await micropip.install("snowballstemmer==2.2.0") @@ -79,6 +80,10 @@ await micropip.install("snowballstemmer>=2.2.0") await micropip.install("snowballstemmer[all]") ``` +[PEP-508]: https://peps.python.org/pep-0508 + +### Disabling dependency resolution + micropip does dependency resolution by default, but you can disable it, this is useful if you want to install a package that has a dependency which is not a pure Python package, but it is not mandatory for your use case: @@ -86,3 +91,20 @@ which is not a pure Python package, but it is not mandatory for your use case: ```python await micropip.install("pkg", deps=False) ``` + +### Constraining indirect dependencies + +Dependency resolution can be further customized with optional `constraints`: these +provide the versions (or URLs of wheels) + +```python +await micropip.install("pkg", constraints=["other-pkg ==0.1.1"]) +``` + +Default `constraints` may be provided to be used by all subsequent calls to +`micropip.install`: + +```python +micropip.set_constraints = ["other-pkg ==0.1.1"] +await micropip.install("pkg") +``` diff --git a/micropip/install.py b/micropip/install.py index 5e32b501..cba8b011 100644 --- a/micropip/install.py +++ b/micropip/install.py @@ -20,6 +20,7 @@ async def install( credentials: str | None = None, pre: bool = False, *, + constraints: list[str] | None = None, verbose: bool | int | None = None, ) -> None: with setup_logging().ctx_level(verbose) as logger: @@ -49,6 +50,7 @@ async def install( fetch_kwargs=fetch_kwargs, verbose=verbose, index_urls=index_urls, + constraints=constraints, ) await transaction.gather_requirements(requirements) diff --git a/micropip/package_manager.py b/micropip/package_manager.py index f9cd7108..e5c66553 100644 --- a/micropip/package_manager.py +++ b/micropip/package_manager.py @@ -26,6 +26,7 @@ def __init__(self) -> None: self.repodata_packages: dict[str, dict[str, Any]] = REPODATA_PACKAGES self.repodata_info: dict[str, str] = REPODATA_INFO + self.constraints: list[str] = [] pass @@ -38,6 +39,7 @@ async def install( pre: bool = False, index_urls: list[str] | str | None = None, *, + constraints: list[str] | None = None, verbose: bool | int | None = None, ): """Install the given package and all of its dependencies. @@ -122,6 +124,14 @@ async def install( - If a list of URLs is provided, micropip will try each URL in order until it finds a package. If no package is found, an error will be raised. + constraints : + + A list of requirements with versions/URLs which will be used only if + needed by any ``requirements``. + + Unlike ``requirements``, the package name _must_ be provided in the + PEP-508 format e.g. ``pkgname@https://...``. + verbose : Print more information about the process. By default, micropip does not change logger level. Setting ``verbose=True`` will print similar @@ -130,6 +140,9 @@ async def install( if index_urls is None: index_urls = self.index_urls + if constraints is None: + constraints = self.constraints + return await install( requirements, index_urls, @@ -137,6 +150,7 @@ async def install( deps, credentials, pre, + constraints=constraints, verbose=verbose, ) diff --git a/micropip/transaction.py b/micropip/transaction.py index f29c48a3..c962396b 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -36,6 +36,7 @@ class Transaction: failed: list[Requirement] = field(default_factory=list) verbose: bool | int | None = None + constraints: list[str] | None = None def __post_init__(self): # If index_urls is None, pyodide-lock.json have to be searched first. From 64bbbe8611651b080471ba15cc9ee96c808405c1 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Thu, 23 Jan 2025 09:51:52 -0600 Subject: [PATCH 02/19] update PR number --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c522a6b..077c237a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added support for constraining indirect dependencies via `micropip.install(..., constraints=[...])`. and `micropip.set_constraints([...])` - [#xyz](https://github.com/pyodide/micropip/pull/xyz) + [#177](https://github.com/pyodide/micropip/pull/177) ### Fixed From e50f58cbb779d491f455e9c3dc071eac25ce6727 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 24 Jan 2025 08:19:52 -0600 Subject: [PATCH 03/19] apply constraints in transaction --- docs/project/usage.md | 26 ++++++++++++++++++++------ micropip/transaction.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/docs/project/usage.md b/docs/project/usage.md index 201a058e..f968b5d0 100644 --- a/docs/project/usage.md +++ b/docs/project/usage.md @@ -94,17 +94,31 @@ await micropip.install("pkg", deps=False) ### Constraining indirect dependencies -Dependency resolution can be further customized with optional `constraints`: these -provide the versions (or URLs of wheels) +Dependency resolution can be further customized with optional `constraints`: as +described in the [`pip`](https://pip.pypa.io/en/stable/user_guide/#constraints-files) +documentation, these must provide a name and version (or URL), and may not request +`[extras]`. ```python -await micropip.install("pkg", constraints=["other-pkg ==0.1.1"]) +await micropip.install( + "pkg", + constraints=[ + "other-pkg ==0.1.1", + "some-other-pkg <2", + "yet-another-pkg@https://example.com/yet_another_pkg-0.1.2-py3-none-any.whl", + # invalid examples # why? + # yet_another_pkg-0.1.2-py3-none-any.whl # missing name + # something-completely[different] ==0.1.1 # extras + # package-with-no-version # missing version or URL + ] +) ``` -Default `constraints` may be provided to be used by all subsequent calls to -`micropip.install`: +`micropip.set_constraints` replaces any default constraints for all subsequent +calls to `micropip.install` that don't specify constraints: ```python micropip.set_constraints = ["other-pkg ==0.1.1"] -await micropip.install("pkg") +await micropip.install("pkg") # uses defaults +await micropip.install("another-pkg", constraints=[]) # ignores defaults ``` diff --git a/micropip/transaction.py b/micropip/transaction.py index c962396b..3b0876f4 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -38,13 +38,29 @@ class Transaction: verbose: bool | int | None = None constraints: list[str] | None = None - def __post_init__(self): + def __post_init__(self) -> None: # If index_urls is None, pyodide-lock.json have to be searched first. # TODO: when PyPI starts to support hosting WASM wheels, this might be deleted. self.search_pyodide_lock_first = ( self.index_urls == package_index.DEFAULT_INDEX_URLS ) + self.constrained_reqs: dict[str, Requirement] = {} + + for constraint in self.constraints or []: + con = Requirement(constraint) + if not con.name: + logger.debug("Transaction: discarding nameless constraint: %s", con) + continue + if con.extras: + logger.debug("Transaction: discarding [extras] constraint: %s", con) + continue + if not con.url or len(con.specifier): + logger.debug("Transaction: discarding versionless constraint: %s", con) + continue + con.name = canonicalize_name(con.name) + self.constrained_reqs[con.name] = con + async def gather_requirements( self, requirements: list[str] | list[Requirement], @@ -88,6 +104,14 @@ def check_version_satisfied(self, req: Requirement) -> tuple[bool, str]: f"Requested '{req}', " f"but {req.name}=={ver} is already installed" ) + def constrain_requirement(self, req: Requirement) -> Requirement: + """Provide a constrained requirement, if available, or the original.""" + constrained_req = self.constrained_reqs.get(canonicalize_name(req.name)) + if constrained_req: + logger.debug("Transaction: %s constrained to %s", req, constrained_req) + return constrained_req + return req + async def add_requirement_inner( self, req: Requirement, @@ -100,6 +124,8 @@ async def add_requirement_inner( for e in req.extras: self.ctx_extras.append({"extra": e}) + req = self.constrain_requirement(req) + if self.pre: req.specifier.prereleases = True @@ -136,6 +162,7 @@ def eval_marker(e: dict[str, str]) -> bool: eval_marker(e) for e in self.ctx_extras ): return + # Is some version of this package is already installed? req.name = canonicalize_name(req.name) From 57de68c3f624c4255188677601ec04191749682f Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 24 Jan 2025 08:40:01 -0600 Subject: [PATCH 04/19] add minimal test --- tests/test_install.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_install.py b/tests/test_install.py index 9b7f7bc8..5f45ef73 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -99,6 +99,29 @@ def test_install_mixed_case2(selenium_standalone_micropip, jinja2): ) +def test_install_constraints(selenium_standalone_micropip): + selenium = selenium_standalone_micropip + selenium.run_js( + """ + await pyodide.runPythonAsync(` + import micropip + await micropip.install( + "pytz", + constraints=["pytz == 2020.5"] + ); + `); + """ + ) + selenium.run_js( + """ + await pyodide.runPythonAsync(` + import pytz + assert pytz.__version__ == "2020.5" + `); + """ + ) + + @pytest.mark.asyncio async def test_package_with_extra(mock_fetch): mock_fetch.add_pkg_version("depa") From 7e4dde5956aab4383b1126c02ab398a53dab93c2 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 24 Jan 2025 08:51:42 -0600 Subject: [PATCH 05/19] test set_constraints --- tests/test_install.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/test_install.py b/tests/test_install.py index 5f45ef73..c847dd62 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -116,7 +116,27 @@ def test_install_constraints(selenium_standalone_micropip): """ await pyodide.runPythonAsync(` import pytz - assert pytz.__version__ == "2020.5" + assert pytz.__version__ == "2020.5", pytz.__version__ + `); + """ + ) + +def test_install_constraints_defaults(selenium_standalone_micropip): + selenium = selenium_standalone_micropip + selenium.run_js( + """ + await pyodide.runPythonAsync(` + import micropip + micropip.set_constraints(["pytz == 2020.5"]) + await micropip.install("pytz"); + `); + """ + ) + selenium.run_js( + """ + await pyodide.runPythonAsync(` + import pytz + assert pytz.__version__ == "2020.5", pytz.__version__ `); """ ) From dab4b4299a646f78acf474b698534a7d9522abd6 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 24 Jan 2025 09:33:26 -0600 Subject: [PATCH 06/19] add set_constraints --- micropip/__init__.py | 1 + micropip/package_manager.py | 13 +++++++++++++ micropip/transaction.py | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/micropip/__init__.py b/micropip/__init__.py index c1fea9c9..9b391639 100644 --- a/micropip/__init__.py +++ b/micropip/__init__.py @@ -9,6 +9,7 @@ install = _package_manager_singleton.install set_index_urls = _package_manager_singleton.set_index_urls +set_constraints = _package_manager_singleton.set_constraints list = _package_manager_singleton.list freeze = _package_manager_singleton.freeze add_mock_package = _package_manager_singleton.add_mock_package diff --git a/micropip/package_manager.py b/micropip/package_manager.py index e5c66553..55a1082d 100644 --- a/micropip/package_manager.py +++ b/micropip/package_manager.py @@ -323,3 +323,16 @@ def set_index_urls(self, urls: List[str] | str): # noqa: UP006 urls = [urls] self.index_urls = urls[:] + + def set_constraints(self, constraints: List[str]): # noqa: UP006 + """ + Set the default constraints to use when looking up packages. + + Parameters + ---------- + constraints + A list of PEP-508 requirements, each of which must include a name and + version, but no ``[extras]``. + """ + + self.constraints = constraints[:] diff --git a/micropip/transaction.py b/micropip/transaction.py index 3b0876f4..b9967c21 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -55,7 +55,7 @@ def __post_init__(self) -> None: if con.extras: logger.debug("Transaction: discarding [extras] constraint: %s", con) continue - if not con.url or len(con.specifier): + if not (con.url or len(con.specifier)): logger.debug("Transaction: discarding versionless constraint: %s", con) continue con.name = canonicalize_name(con.name) From 5b5b1c87c5fe91c84cb8ab53f2542977ce6f1e69 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:33:50 +0000 Subject: [PATCH 07/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_install.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_install.py b/tests/test_install.py index c847dd62..f24f306b 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -121,6 +121,7 @@ def test_install_constraints(selenium_standalone_micropip): """ ) + def test_install_constraints_defaults(selenium_standalone_micropip): selenium = selenium_standalone_micropip selenium.run_js( From d5d26948127301b44111acfe8b70bae7472fdb0a Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 24 Jan 2025 10:24:04 -0600 Subject: [PATCH 08/19] more tests --- micropip/transaction.py | 18 +++++++++--------- tests/test_install.py | 31 ++++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/micropip/transaction.py b/micropip/transaction.py index b9967c21..42b326ec 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -72,17 +72,17 @@ async def gather_requirements( await asyncio.gather(*requirement_promises) async def add_requirement(self, req: str | Requirement) -> None: - if isinstance(req, Requirement): - return await self.add_requirement_inner(req) - - if not urlparse(req).path.endswith(".whl"): - return await self.add_requirement_inner(Requirement(req)) + req = self.constrain_requirement( + req if isinstance(req, Requirement) else Requirement(req) + ) - # custom download location - wheel = WheelInfo.from_url(req) - check_compatible(wheel.filename) + if req.url and urlparse(req.url).path.endswith(".whl"): + # custom download location + wheel = WheelInfo.from_url(req.url) + check_compatible(wheel.filename) + return await self.add_wheel(wheel, extras=set(), specifier="") - await self.add_wheel(wheel, extras=set(), specifier="") + await self.add_requirement_inner(req) def check_version_satisfied(self, req: Requirement) -> tuple[bool, str]: ver = None diff --git a/tests/test_install.py b/tests/test_install.py index c847dd62..4263a787 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -99,15 +99,39 @@ def test_install_mixed_case2(selenium_standalone_micropip, jinja2): ) -def test_install_constraints(selenium_standalone_micropip): +PYTZ_2020_5_WHEEL = "pytz-2020.5-py2.py3-none-any.whl" +PYTZ_2020_5_URL = f"https://files.pythonhosted.org/packages/89/06/2c2d3034b4d6bf22f2a4ae546d16925898658a33b4400cfb7e2c1e2871a3/{PYTZ_2020_5_WHEEL}" + + +@pytest.mark.parametrize( + "pytz", + [ + "pytz == 2020.5", + "pytz >=2020.4,<2020.6", + f"pytz @ {PYTZ_2020_5_URL}", + f"pytz @ emfs:{PYTZ_2020_5_WHEEL}", + ], +) +def test_install_constraints(pytz, selenium_standalone_micropip): selenium = selenium_standalone_micropip + if PYTZ_2020_5_WHEEL in pytz and PYTZ_2020_5_URL not in pytz: + selenium.run_js( + f""" + await pyodide.runPythonAsync(` + from pyodide.http import pyfetch + resp = await pyfetch("{PYTZ_2020_5_URL}") + await resp._into_file(open("{PYTZ_2020_5_WHEEL}", "wb")) + `); + """ + ) + selenium.run_js( - """ + f""" await pyodide.runPythonAsync(` import micropip await micropip.install( "pytz", - constraints=["pytz == 2020.5"] + constraints=["{pytz}"] ); `); """ @@ -121,6 +145,7 @@ def test_install_constraints(selenium_standalone_micropip): """ ) + def test_install_constraints_defaults(selenium_standalone_micropip): selenium = selenium_standalone_micropip selenium.run_js( From c61f4ad7e58e1a68938131e3a738816ffdad8953 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Fri, 24 Jan 2025 11:00:26 -0600 Subject: [PATCH 09/19] handle more emfs cases --- micropip/transaction.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/micropip/transaction.py b/micropip/transaction.py index 42b326ec..bf0cd5e4 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -6,7 +6,7 @@ from importlib.metadata import PackageNotFoundError from urllib.parse import urlparse -from packaging.requirements import Requirement +from packaging.requirements import InvalidRequirement, Requirement from packaging.utils import canonicalize_name from . import package_index @@ -72,18 +72,26 @@ async def gather_requirements( await asyncio.gather(*requirement_promises) async def add_requirement(self, req: str | Requirement) -> None: - req = self.constrain_requirement( - req if isinstance(req, Requirement) else Requirement(req) - ) + if isinstance(req, Requirement): + return await self.add_requirement_inner(req) - if req.url and urlparse(req.url).path.endswith(".whl"): + try: + req = self.constrain_requirement(Requirement(req)) + url = req.url + except InvalidRequirement: + url = f"{req}" + + if not (url and urlparse(url).path.endswith(".whl")): + return await self.add_requirement_inner( + req if isinstance(req, Requirement) else Requirement(req) + ) + + if url: # custom download location - wheel = WheelInfo.from_url(req.url) + wheel = WheelInfo.from_url(url) check_compatible(wheel.filename) return await self.add_wheel(wheel, extras=set(), specifier="") - await self.add_requirement_inner(req) - def check_version_satisfied(self, req: Requirement) -> tuple[bool, str]: ver = None try: From 48340edeb718bc7b2048acfb0ee9ca11339c849b Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 25 Jan 2025 13:17:38 -0600 Subject: [PATCH 10/19] address review comments --- docs/project/usage.md | 53 ++++++++++++++++------- micropip/_utils.py | 75 +++++++++++++++++++++++++++++++- micropip/transaction.py | 42 +++++++----------- tests/conftest.py | 43 +++++++++++++++++++ tests/test_install.py | 95 +++++++++++++++-------------------------- tests/test_utils.py | 45 ++++++++++++++++++- 6 files changed, 248 insertions(+), 105 deletions(-) diff --git a/docs/project/usage.md b/docs/project/usage.md index f968b5d0..5bf78c5b 100644 --- a/docs/project/usage.md +++ b/docs/project/usage.md @@ -71,17 +71,17 @@ You can pass multiple packages to `micropip.install`: await micropip.install(["pkg1", "pkg2"]) ``` -A dependency can specify be refined per the [PEP-508] spec: +A dependency can be refined as per the [PEP-508] spec: +[pep-508]: https://peps.python.org/pep-0508 ```python await micropip.install("snowballstemmer==2.2.0") await micropip.install("snowballstemmer>=2.2.0") +await micropip.install("snowballstemmer @ https://.../snowballstemmer.*.whl") await micropip.install("snowballstemmer[all]") ``` -[PEP-508]: https://peps.python.org/pep-0508 - ### Disabling dependency resolution micropip does dependency resolution by default, but you can disable it, @@ -94,31 +94,52 @@ await micropip.install("pkg", deps=False) ### Constraining indirect dependencies -Dependency resolution can be further customized with optional `constraints`: as -described in the [`pip`](https://pip.pypa.io/en/stable/user_guide/#constraints-files) -documentation, these must provide a name and version (or URL), and may not request -`[extras]`. +Dependency resolution can be further customized with optional `constraints`: +these modify both _direct_ and _indirect_ dependency resolutions, while direct URLs +in either a requirement or constraint will generally bypass any other specifiers. + +As described in the [`pip` documentation][pip-constraints], each constraint: + +[pip-constraints]: https://pip.pypa.io/en/stable/user_guide/#constraints-files + + - _must_ provide a name + - _must_ provide exactly one of + - a set of version specifiers + - a URL, and _may not_ request + - _must not_ request any `[extras]` + + +Invalid constraints will be silently discarded, or logged if `verbose` is provided. ```python await micropip.install( "pkg", constraints=[ - "other-pkg ==0.1.1", - "some-other-pkg <2", + "other-pkg==0.1.1", + "some-other-pkg<2", "yet-another-pkg@https://example.com/yet_another_pkg-0.1.2-py3-none-any.whl", - # invalid examples # why? - # yet_another_pkg-0.1.2-py3-none-any.whl # missing name - # something-completely[different] ==0.1.1 # extras - # package-with-no-version # missing version or URL + # silently discarded # why? + "yet_another_pkg-0.1.2-py3-none-any.whl", # missing name + "something-completely[different] ==0.1.1", # extras + "package-with-no-version", # missing version or URL ] ) ``` +Over-constrained requirements will fail to resolve, leaving the environment unmodified. + +```python +await micropip.install("pkg ==1", constraints=["pkg ==2"]) +# ValueError: Can't find a pure Python 3 wheel for 'pkg==1,==2'. +``` + +### Setting default constraints + `micropip.set_constraints` replaces any default constraints for all subsequent -calls to `micropip.install` that don't specify constraints: +calls to `micropip.install` that don't specify `constraints`: ```python -micropip.set_constraints = ["other-pkg ==0.1.1"] -await micropip.install("pkg") # uses defaults +micropip.set_constraints(["other-pkg ==0.1.1"]) +await micropip.install("pkg") # uses defaults, if needed await micropip.install("another-pkg", constraints=[]) # ignores defaults ``` diff --git a/micropip/_utils.py b/micropip/_utils.py index 70d4724e..c25c967c 100644 --- a/micropip/_utils.py +++ b/micropip/_utils.py @@ -4,7 +4,7 @@ from pathlib import Path from sysconfig import get_config_var, get_platform -from packaging.requirements import Requirement +from packaging.requirements import InvalidRequirement, Requirement from packaging.tags import Tag from packaging.tags import sys_tags as sys_tags_orig from packaging.utils import BuildTag, InvalidWheelFilename, canonicalize_name @@ -268,3 +268,76 @@ def fix_package_dependencies( (get_dist_info(dist) / "PYODIDE_REQUIRES").write_text( json.dumps(sorted(x for x in depends)) ) + + +def validate_constraints( + constraints: list[str] | None, +) -> tuple[dict[str, Requirement], dict[str, str]]: + """Build a validated ``Requirement`` dictionary from raw constraint strings. + + Parameters + ---------- + constraints (list): + A list of PEP-508 dependency specs, expected to contain both a package + name and at least one speicifier. + + Returns + ------- + A 2-tuple of: + - a dictionary of ``Requirement`` objects, keyed by canonical name + - a dictionary of messages strings, keyed by constraint + """ + constrained_reqs: dict[str, Requirement] = {} + ignore_messages: dict[str, str] = {} + + for raw_constraint in constraints or []: + try: + req = Requirement(raw_constraint) + req.name = canonicalize_name(req.name) + except InvalidRequirement as err: + ignore_messages[raw_constraint] = f"failed to parse: {err}" + continue + + if req.extras: + ignore_messages[raw_constraint] = "may not provide [extras]" + continue + + if not (req.url or len(req.specifier)): + ignore_messages[raw_constraint] = "no version or URL" + continue + + constrained_reqs[req.name] = req + + return constrained_reqs, ignore_messages + + +def constrain_requirement( + requirement: Requirement, constrained_requirements: dict[str, Requirement] +) -> Requirement: + """Refine or replace a requirement from a set of constraints. + + Parameters + ---------- + requirement (list): + A list of PEP-508 dependency specs, expected to contain both a package + name and at least one speicifier. + + Returns + ------- + A 2-tuple of: + - a dictionary of ``Requirement`` objects, keyed by canonical name + - a dictionary of messages strings, keyed by constraint + """ + # URLs cannot be merged + if requirement.url: + return requirement + + as_constrained = constrained_requirements.get(canonicalize_name(requirement.name)) + + if as_constrained: + if as_constrained.url: + requirement = as_constrained + else: + requirement.specifier = requirement.specifier & as_constrained.specifier + + return requirement diff --git a/micropip/transaction.py b/micropip/transaction.py index bf0cd5e4..3168fc7a 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -11,7 +11,12 @@ from . import package_index from ._compat import REPODATA_PACKAGES -from ._utils import best_compatible_tag_index, check_compatible +from ._utils import ( + best_compatible_tag_index, + check_compatible, + constrain_requirement, + validate_constraints, +) from .constants import FAQ_URLS from .package import PackageMetadata from .package_index import ProjectInfo @@ -45,21 +50,13 @@ def __post_init__(self) -> None: self.index_urls == package_index.DEFAULT_INDEX_URLS ) - self.constrained_reqs: dict[str, Requirement] = {} - - for constraint in self.constraints or []: - con = Requirement(constraint) - if not con.name: - logger.debug("Transaction: discarding nameless constraint: %s", con) - continue - if con.extras: - logger.debug("Transaction: discarding [extras] constraint: %s", con) - continue - if not (con.url or len(con.specifier)): - logger.debug("Transaction: discarding versionless constraint: %s", con) - continue - con.name = canonicalize_name(con.name) - self.constrained_reqs[con.name] = con + self.constrained_reqs, messages = validate_constraints(self.constraints) + + if self.verbose and messages: + for constraint, message in messages.items(): + logger.info( + "Transaction: constraint %s discarded: %s", constraint, message + ) async def gather_requirements( self, @@ -76,7 +73,7 @@ async def add_requirement(self, req: str | Requirement) -> None: return await self.add_requirement_inner(req) try: - req = self.constrain_requirement(Requirement(req)) + req = constrain_requirement(Requirement(req), self.constrained_reqs) url = req.url except InvalidRequirement: url = f"{req}" @@ -112,14 +109,6 @@ def check_version_satisfied(self, req: Requirement) -> tuple[bool, str]: f"Requested '{req}', " f"but {req.name}=={ver} is already installed" ) - def constrain_requirement(self, req: Requirement) -> Requirement: - """Provide a constrained requirement, if available, or the original.""" - constrained_req = self.constrained_reqs.get(canonicalize_name(req.name)) - if constrained_req: - logger.debug("Transaction: %s constrained to %s", req, constrained_req) - return constrained_req - return req - async def add_requirement_inner( self, req: Requirement, @@ -129,10 +118,11 @@ async def add_requirement_inner( See PEP 508 for a description of the requirements. https://www.python.org/dev/peps/pep-0508 """ + # add [extras] first, as constraints will never add them for e in req.extras: self.ctx_extras.append({"extra": e}) - req = self.constrain_requirement(req) + req = constrain_requirement(req, self.constrained_reqs) if self.pre: req.specifier.prereleases = True diff --git a/tests/conftest.py b/tests/conftest.py index a1a3a518..e59bb688 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ from packaging.utils import parse_wheel_filename from pytest_httpserver import HTTPServer from pytest_pyodide import spawn_web_server +from pytest_pyodide.runner import JavascriptException def pytest_addoption(parser): @@ -408,3 +409,45 @@ def mock_package_index_simple_html_api(httpserver): suffix="_simple.html", content_type="text/html", ) + + +@pytest.fixture( + params=[ + None, + "pytest ==7.2.2", + "pytest >=7.2.1,<7.2.3", + "pytest @ {url}", + "pytest @ emfs:{wheel}", + ] +) +def valid_constraint(request, wheel_catalog): + wheel = wheel_catalog.get("pytest") + if not request.param: + return request.param + return request.param.format(url=wheel.url, wheel=wheel.url.split("/")[-1]) + + +INVALID_CONSTRAINT_MESSAGES = { + "": "parse", + "http://example.com": "name", + "a-package[with-extra]": "[extras]", + "a-package": "no version or URL", +} + + +@pytest.fixture(params=[*INVALID_CONSTRAINT_MESSAGES.keys()]) +def invalid_constraint(request): + return request.param + + +@pytest.fixture +def run_async_py_in_js(selenium_standalone_micropip): + def _run(*lines, error_match=None): + js = "\n".join(["await pyodide.runPythonAsync(`", *lines, "`);"]) + if error_match: + with pytest.raises(JavascriptException, match=error_match): + selenium_standalone_micropip.run_js(js) + else: + selenium_standalone_micropip.run_js(js) + + return _run diff --git a/tests/test_install.py b/tests/test_install.py index 4263a787..9300bed7 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -99,71 +99,44 @@ def test_install_mixed_case2(selenium_standalone_micropip, jinja2): ) -PYTZ_2020_5_WHEEL = "pytz-2020.5-py2.py3-none-any.whl" -PYTZ_2020_5_URL = f"https://files.pythonhosted.org/packages/89/06/2c2d3034b4d6bf22f2a4ae546d16925898658a33b4400cfb7e2c1e2871a3/{PYTZ_2020_5_WHEEL}" - - -@pytest.mark.parametrize( - "pytz", - [ - "pytz == 2020.5", - "pytz >=2020.4,<2020.6", - f"pytz @ {PYTZ_2020_5_URL}", - f"pytz @ emfs:{PYTZ_2020_5_WHEEL}", - ], -) -def test_install_constraints(pytz, selenium_standalone_micropip): - selenium = selenium_standalone_micropip - if PYTZ_2020_5_WHEEL in pytz and PYTZ_2020_5_URL not in pytz: - selenium.run_js( - f""" - await pyodide.runPythonAsync(` - from pyodide.http import pyfetch - resp = await pyfetch("{PYTZ_2020_5_URL}") - await resp._into_file(open("{PYTZ_2020_5_WHEEL}", "wb")) - `); - """ +@pytest.mark.parametrize("set_constraints", [False, True]) +def test_install_constraints( + set_constraints, + valid_constraint, + wheel_catalog, + run_async_py_in_js, +): + constraints = [valid_constraint] if valid_constraint else [] + run_async_py_in_js("import micropip") + + if valid_constraint and "emfs:" in valid_constraint: + url = wheel_catalog.get("pytest").url + wheel = url.split("/")[-1] + run_async_py_in_js( + "from pyodide.http import pyfetch", + f"resp = await pyfetch('{url}')", + f"await resp._into_file(open('{wheel}', 'wb'))", ) - selenium.run_js( - f""" - await pyodide.runPythonAsync(` - import micropip - await micropip.install( - "pytz", - constraints=["{pytz}"] - ); - `); - """ - ) - selenium.run_js( - """ - await pyodide.runPythonAsync(` - import pytz - assert pytz.__version__ == "2020.5", pytz.__version__ - `); - """ - ) + if set_constraints: + run_async_py_in_js(f"micropip.set_constraints({constraints})") + install_args = "" + else: + install_args = f"constraints={constraints}" + if constraints and "@" not in valid_constraint: + run_async_py_in_js( + f"await micropip.install('pytest ==7.2.3', {install_args})", + error_match="Can't find a pure Python 3 wheel", + ) -def test_install_constraints_defaults(selenium_standalone_micropip): - selenium = selenium_standalone_micropip - selenium.run_js( - """ - await pyodide.runPythonAsync(` - import micropip - micropip.set_constraints(["pytz == 2020.5"]) - await micropip.install("pytz"); - `); - """ - ) - selenium.run_js( - """ - await pyodide.runPythonAsync(` - import pytz - assert pytz.__version__ == "2020.5", pytz.__version__ - `); - """ + run_async_py_in_js(f"await micropip.install('pytest', {install_args})") + + compare = "==" if constraints else "!=" + + run_async_py_in_js( + "import pytest", + f"assert pytest.__version__ {compare} '7.2.2', pytest.__version__", ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 7d0522bc..73859ea2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,7 +2,8 @@ from importlib.metadata import distribution import pytest -from conftest import CPVER, EMSCRIPTEN_VER, PLATFORM +from conftest import CPVER, EMSCRIPTEN_VER, INVALID_CONSTRAINT_MESSAGES, PLATFORM +from packaging.requirements import Requirement from pytest_pyodide import run_in_pyodide import micropip._utils as _utils @@ -150,3 +151,45 @@ def test_best_compatible_tag(package, version, incompatible_tags, compatible_tag sorted_tags = sorted(tags, key=best_compatible_tag_index) sorted_tags.reverse() assert sorted_tags == tags + + +def test_validate_constraints_valid(valid_constraint): + constraints = [valid_constraint] if valid_constraint else [] + reqs, msgs = _utils.validate_constraints(constraints) + assert len(reqs) == len(constraints) + assert not msgs + + +def test_validate_constraints_invalid(invalid_constraint): + reqs, msgs = _utils.validate_constraints([invalid_constraint]) + assert not reqs + for constraint, msg in msgs.items(): + msg = msgs[constraint] + assert INVALID_CONSTRAINT_MESSAGES[constraint] in msg + + +def test_constrain_requirement(valid_constraint): + req = Requirement("pytest") + constraints = [valid_constraint] if valid_constraint else [] + assert not req.specifier + constrained_reqs, msg = _utils.validate_constraints(constraints) + assert not msg + constrained = _utils.constrain_requirement(req, constrained_reqs) + + if constraints: + assert constrained.specifier or constrained.url + assert not (constrained.specifier and constrained.url) + else: + assert not (constrained.specifier or constrained.url) + + +def test_constrain_requirement_direct_url(valid_constraint, wheel_catalog): + constraints = [valid_constraint] if valid_constraint else [] + wheel = wheel_catalog.get("pytest") + url = f"{wheel.url}?foo" + req = Requirement(f"pytest @ {url}") + assert not req.specifier + constrained_reqs, msg = _utils.validate_constraints(constraints) + assert not msg + constrained = _utils.constrain_requirement(req, constrained_reqs) + assert constrained.url == url From 64024211898ec573885e36fc9bfc310ce83a86aa Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 25 Jan 2025 16:01:15 -0600 Subject: [PATCH 11/19] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 077c237a..4279a906 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added support for constraining indirect dependencies via +- Added support for constraining resolved requirements via `micropip.install(..., constraints=[...])`. and `micropip.set_constraints([...])` [#177](https://github.com/pyodide/micropip/pull/177) From a6106d5bc3ea02e20756bd202ddb2af1ddaf931f Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 25 Jan 2025 16:02:20 -0600 Subject: [PATCH 12/19] update docs --- docs/project/usage.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/project/usage.md b/docs/project/usage.md index 5bf78c5b..9c9ab4a2 100644 --- a/docs/project/usage.md +++ b/docs/project/usage.md @@ -105,10 +105,9 @@ As described in the [`pip` documentation][pip-constraints], each constraint: - _must_ provide a name - _must_ provide exactly one of - a set of version specifiers - - a URL, and _may not_ request + - a URL - _must not_ request any `[extras]` - Invalid constraints will be silently discarded, or logged if `verbose` is provided. ```python From 98d1faa3040d5770740fc32018e401df9d1499cb Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sun, 26 Jan 2025 09:17:33 -0600 Subject: [PATCH 13/19] handle env markers and additive constraints --- docs/project/usage.md | 7 ++++++- micropip/_utils.py | 38 +++++++++++++++++++++++++++----------- micropip/transaction.py | 10 +++++----- pyproject.toml | 3 +++ tests/test_utils.py | 20 ++++++++++++++++++-- 5 files changed, 59 insertions(+), 19 deletions(-) diff --git a/docs/project/usage.md b/docs/project/usage.md index 9c9ab4a2..60e5f663 100644 --- a/docs/project/usage.md +++ b/docs/project/usage.md @@ -96,7 +96,7 @@ await micropip.install("pkg", deps=False) Dependency resolution can be further customized with optional `constraints`: these modify both _direct_ and _indirect_ dependency resolutions, while direct URLs -in either a requirement or constraint will generally bypass any other specifiers. +in either a requirement or constraint will bypass any other specifiers. As described in the [`pip` documentation][pip-constraints], each constraint: @@ -108,6 +108,8 @@ As described in the [`pip` documentation][pip-constraints], each constraint: - a URL - _must not_ request any `[extras]` +Multiple constraints of the same canonical name are merged. + Invalid constraints will be silently discarded, or logged if `verbose` is provided. ```python @@ -116,11 +118,14 @@ await micropip.install( constraints=[ "other-pkg==0.1.1", "some-other-pkg<2", + "some-other-pkg<3", # merged with the above "yet-another-pkg@https://example.com/yet_another_pkg-0.1.2-py3-none-any.whl", # silently discarded # why? + "yet-another-pkg >=1", # previously defined by URL "yet_another_pkg-0.1.2-py3-none-any.whl", # missing name "something-completely[different] ==0.1.1", # extras "package-with-no-version", # missing version or URL + "other-pkg ==0.0.1 ; python_version < '3'", # not applicable ] ) ``` diff --git a/micropip/_utils.py b/micropip/_utils.py index c25c967c..e468e7c2 100644 --- a/micropip/_utils.py +++ b/micropip/_utils.py @@ -272,41 +272,57 @@ def fix_package_dependencies( def validate_constraints( constraints: list[str] | None, -) -> tuple[dict[str, Requirement], dict[str, str]]: + environment: dict[str, str] | None = None, +) -> tuple[dict[str, Requirement], dict[str, list[str]]]: """Build a validated ``Requirement`` dictionary from raw constraint strings. Parameters ---------- constraints (list): A list of PEP-508 dependency specs, expected to contain both a package - name and at least one speicifier. + name and at least one specififier. + + environment (optional dict): + The markers for the current environment, such as OS, Python implementation. + If ``None``, the current execution environment will be used. Returns ------- A 2-tuple of: - a dictionary of ``Requirement`` objects, keyed by canonical name - - a dictionary of messages strings, keyed by constraint + - a dictionary of message strings, keyed by constraint """ constrained_reqs: dict[str, Requirement] = {} - ignore_messages: dict[str, str] = {} + ignore_messages: dict[str, list[str]] = {} for raw_constraint in constraints or []: + constraint_messages: list[str] = [] + try: req = Requirement(raw_constraint) req.name = canonicalize_name(req.name) except InvalidRequirement as err: - ignore_messages[raw_constraint] = f"failed to parse: {err}" + ignore_messages[raw_constraint] = [f"failed to parse: {err}"] continue if req.extras: - ignore_messages[raw_constraint] = "may not provide [extras]" - continue + constraint_messages.append("may not provide [extras]") if not (req.url or len(req.specifier)): - ignore_messages[raw_constraint] = "no version or URL" - continue - - constrained_reqs[req.name] = req + constraint_messages.append("no version or URL") + + if req.marker and not req.marker.evaluate(environment): + constraint_messages.append(f"not applicable: {req.marker}") + + if constraint_messages: + ignore_messages[raw_constraint] = constraint_messages + elif req.name in constrained_reqs: + ignore_messages[raw_constraint] = [ + f"updated existing constraint for {req.name}" + ] + constrained_reqs[req.name] = constrain_requirement(req, constrained_reqs) + else: + constrained_reqs[req.name] = req return constrained_reqs, ignore_messages diff --git a/micropip/transaction.py b/micropip/transaction.py index 3168fc7a..bc363d25 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -50,13 +50,13 @@ def __post_init__(self) -> None: self.index_urls == package_index.DEFAULT_INDEX_URLS ) - self.constrained_reqs, messages = validate_constraints(self.constraints) + self.constrained_reqs, messages = validate_constraints( + self.constraints, self.ctx + ) if self.verbose and messages: - for constraint, message in messages.items(): - logger.info( - "Transaction: constraint %s discarded: %s", constraint, message - ) + for constraint, msg in messages.items(): + logger.info("Transaction: constraint %s discarded: %s", constraint, msg) async def gather_requirements( self, diff --git a/pyproject.toml b/pyproject.toml index d761a269..c40c466e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,3 +68,6 @@ python_version = "3.12" show_error_codes = true warn_unreachable = true ignore_missing_imports = true + +[tool.coverage.html] +show_contexts = true diff --git a/tests/test_utils.py b/tests/test_utils.py index 73859ea2..5add96ab 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -159,13 +159,29 @@ def test_validate_constraints_valid(valid_constraint): assert len(reqs) == len(constraints) assert not msgs + if not valid_constraint or "==" not in valid_constraint: + return + + extra_constraint = valid_constraint.replace("==", ">=") + marker_valid_constraint = f"{extra_constraint} ; python_version>'2.7'" + marker_invalid_constraint = f"{extra_constraint} ; python_version<'3'" + constraints += [ + extra_constraint, + marker_valid_constraint, + marker_invalid_constraint, + ] + + reqs, msgs = _utils.validate_constraints(constraints) + assert "updated existing" in f"{msgs[extra_constraint]}" + assert "updated existing" in f"{msgs[marker_valid_constraint]}" + assert "not applicable" in f"{msgs[marker_invalid_constraint]}" + def test_validate_constraints_invalid(invalid_constraint): reqs, msgs = _utils.validate_constraints([invalid_constraint]) assert not reqs for constraint, msg in msgs.items(): - msg = msgs[constraint] - assert INVALID_CONSTRAINT_MESSAGES[constraint] in msg + assert INVALID_CONSTRAINT_MESSAGES[constraint] in f"{msg}" def test_constrain_requirement(valid_constraint): From c59c754322105c73a48ce7dcbd66d0dfbd69784f Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 27 Jan 2025 16:21:26 -0600 Subject: [PATCH 14/19] continue gathering requreiments from pyodide packages, reverse pydodide/wheel install order --- CHANGELOG.md | 8 +++++++- micropip/install.py | 35 +++++++++++++---------------------- micropip/transaction.py | 13 ++++++++----- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4279a906..51f4153e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Changed + +- `micropip.install` now installs wheels from PyPI or custom indexes before built-in + Pyodide packages, reversing the previous behavior. + [#xxx](https://github.com/pyodide/micropip/pull/xxx) + ### Added - Added support for constraining resolved requirements via `micropip.install(..., constraints=[...])`. and `micropip.set_constraints([...])` - [#177](https://github.com/pyodide/micropip/pull/177) + [#xxx](https://github.com/pyodide/micropip/pull/xxx) ### Fixed diff --git a/micropip/install.py b/micropip/install.py index cba8b011..3f5a9853 100644 --- a/micropip/install.py +++ b/micropip/install.py @@ -1,8 +1,6 @@ import asyncio import importlib -from collections.abc import Coroutine from pathlib import Path -from typing import Any from packaging.markers import default_environment @@ -61,40 +59,33 @@ async def install( f"See: {FAQ_URLS['cant_find_wheel']}\n" ) - package_names = [pkg.name for pkg in transaction.pyodide_packages] + [ - pkg.name for pkg in transaction.wheels - ] + pyodide_packages, wheels = transaction.pyodide_packages, transaction.wheels + + package_names = [pkg.name for pkg in wheels + pyodide_packages] logger.debug( "Installing packages %r and wheels %r ", transaction.pyodide_packages, [w.filename for w in transaction.wheels], ) + if package_names: logger.info("Installing collected packages: %s", ", ".join(package_names)) - wheel_promises: list[Coroutine[Any, Any, None] | asyncio.Task[Any]] = [] + # Install PyPI packages + # detect whether the wheel metadata is from PyPI or from custom location + # wheel metadata from PyPI has SHA256 checksum digest. + await asyncio.gather(*(wheel.install(wheel_base) for wheel in wheels)) + # Install built-in packages - pyodide_packages = transaction.pyodide_packages - if len(pyodide_packages): + if pyodide_packages: # Note: branch never happens in out-of-browser testing because in # that case REPODATA_PACKAGES is empty. - wheel_promises.append( - asyncio.ensure_future( - loadPackage(to_js([name for [name, _, _] in pyodide_packages])) - ) + await asyncio.ensure_future( + loadPackage(to_js([name for [name, _, _] in pyodide_packages])) ) - # Now install PyPI packages - # detect whether the wheel metadata is from PyPI or from custom location - # wheel metadata from PyPI has SHA256 checksum digest. - wheel_promises.extend(wheel.install(wheel_base) for wheel in transaction.wheels) - - await asyncio.gather(*wheel_promises) - - packages = [ - f"{pkg.name}-{pkg.version}" for pkg in transaction.pyodide_packages - ] + [f"{pkg.name}-{pkg.version}" for pkg in transaction.wheels] + packages = [f"{pkg.name}-{pkg.version}" for pkg in pyodide_packages + wheels] if packages: logger.info("Successfully installed %s", ", ".join(packages)) diff --git a/micropip/transaction.py b/micropip/transaction.py index bc363d25..f35228d1 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -171,7 +171,7 @@ def eval_marker(e: dict[str, str]) -> bool: try: if self.search_pyodide_lock_first: - if self._add_requirement_from_pyodide_lock(req): + if await self._add_requirement_from_pyodide_lock(req): logger.debug("Transaction: package found in lock file: %r", req) return @@ -187,7 +187,7 @@ def eval_marker(e: dict[str, str]) -> bool: # If the requirement is not found in package index, # we still have a chance to find it from pyodide lockfile. - if not self._add_requirement_from_pyodide_lock(req): + if not await self._add_requirement_from_pyodide_lock(req): logger.debug( "Transaction: package %r not found in lock file", req ) @@ -198,18 +198,21 @@ def eval_marker(e: dict[str, str]) -> bool: if not self.keep_going: raise - def _add_requirement_from_pyodide_lock(self, req: Requirement) -> bool: + async def _add_requirement_from_pyodide_lock(self, req: Requirement) -> bool: """ Find requirement from pyodide-lock.json. If the requirement is found, add it to the package list and return True. Otherwise, return False. """ - if req.name in REPODATA_PACKAGES and req.specifier.contains( + locked_package = REPODATA_PACKAGES.get(req.name) + if locked_package and req.specifier.contains( REPODATA_PACKAGES[req.name]["version"], prereleases=True ): - version = REPODATA_PACKAGES[req.name]["version"] + version = locked_package["version"] self.pyodide_packages.append( PackageMetadata(name=req.name, version=str(version), source="pyodide") ) + if locked_package["depends"]: + await self.gather_requirements(locked_package["depends"]) return True return False From c47ee4134bafdc3f945798d591ee7c3d809cffd6 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 27 Jan 2025 16:29:31 -0600 Subject: [PATCH 15/19] fix pr number in changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51f4153e..d277d0f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,13 +10,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `micropip.install` now installs wheels from PyPI or custom indexes before built-in Pyodide packages, reversing the previous behavior. - [#xxx](https://github.com/pyodide/micropip/pull/xxx) + [#181](https://github.com/pyodide/micropip/pull/181) ### Added - Added support for constraining resolved requirements via `micropip.install(..., constraints=[...])`. and `micropip.set_constraints([...])` - [#xxx](https://github.com/pyodide/micropip/pull/xxx) + [#181](https://github.com/pyodide/micropip/pull/181) ### Fixed From 32d3836efa4d331286eb04086868c5896e143e63 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 27 Jan 2025 16:30:33 -0600 Subject: [PATCH 16/19] fix spelling --- micropip/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropip/_utils.py b/micropip/_utils.py index e468e7c2..e57c0381 100644 --- a/micropip/_utils.py +++ b/micropip/_utils.py @@ -280,7 +280,7 @@ def validate_constraints( ---------- constraints (list): A list of PEP-508 dependency specs, expected to contain both a package - name and at least one specififier. + name and at least one specifier. environment (optional dict): The markers for the current environment, such as OS, Python implementation. From a1ff4e2e7181bae7a86b01ec92fb69f0e215e3f8 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 27 Jan 2025 17:18:56 -0600 Subject: [PATCH 17/19] tighten up utils, docstrings --- micropip/_utils.py | 55 +++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/micropip/_utils.py b/micropip/_utils.py index e57c0381..97aa1350 100644 --- a/micropip/_utils.py +++ b/micropip/_utils.py @@ -292,68 +292,67 @@ def validate_constraints( - a dictionary of ``Requirement`` objects, keyed by canonical name - a dictionary of message strings, keyed by constraint """ - constrained_reqs: dict[str, Requirement] = {} - ignore_messages: dict[str, list[str]] = {} + reqs: dict[str, Requirement] = {} + all_messages: dict[str, list[str]] = {} for raw_constraint in constraints or []: - constraint_messages: list[str] = [] + messages: list[str] = [] try: req = Requirement(raw_constraint) req.name = canonicalize_name(req.name) except InvalidRequirement as err: - ignore_messages[raw_constraint] = [f"failed to parse: {err}"] + all_messages[raw_constraint] = [f"failed to parse: {err}"] continue if req.extras: - constraint_messages.append("may not provide [extras]") + messages.append("may not provide [extras]") - if not (req.url or len(req.specifier)): - constraint_messages.append("no version or URL") + if not (req.url or req.specifier): + messages.append("no version or URL") if req.marker and not req.marker.evaluate(environment): - constraint_messages.append(f"not applicable: {req.marker}") + messages.append(f"not applicable: {req.marker}") - if constraint_messages: - ignore_messages[raw_constraint] = constraint_messages - elif req.name in constrained_reqs: - ignore_messages[raw_constraint] = [ + if messages: + all_messages[raw_constraint] = messages + elif req.name in reqs: + all_messages[raw_constraint] = [ f"updated existing constraint for {req.name}" ] - constrained_reqs[req.name] = constrain_requirement(req, constrained_reqs) + reqs[req.name] = constrain_requirement(req, reqs) else: - constrained_reqs[req.name] = req + reqs[req.name] = req - return constrained_reqs, ignore_messages + return reqs, all_messages def constrain_requirement( requirement: Requirement, constrained_requirements: dict[str, Requirement] ) -> Requirement: - """Refine or replace a requirement from a set of constraints. + """Modify or replace a requirement based on a set of constraints. Parameters ---------- - requirement (list): - A list of PEP-508 dependency specs, expected to contain both a package - name and at least one speicifier. + requirement (Requirement): + A ``Requirement`` to constrain. + + constrained_requirements (dict): + A dictionary of ``Requirement`` objects, keyed by canonical name. Returns ------- - A 2-tuple of: - - a dictionary of ``Requirement`` objects, keyed by canonical name - - a dictionary of messages strings, keyed by constraint + A constrained ``Requirement``. """ # URLs cannot be merged if requirement.url: return requirement - as_constrained = constrained_requirements.get(canonicalize_name(requirement.name)) + constrained = constrained_requirements.get(canonicalize_name(requirement.name)) - if as_constrained: - if as_constrained.url: - requirement = as_constrained - else: - requirement.specifier = requirement.specifier & as_constrained.specifier + if constrained: + if constrained.url: + return constrained + requirement.specifier = requirement.specifier & constrained.specifier return requirement From 92baa16724b95b947ab9da8866a287c192578218 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Wed, 29 Jan 2025 08:01:40 -0600 Subject: [PATCH 18/19] skip an await if not needed, rework changelog entry --- CHANGELOG.md | 20 ++++++++------------ micropip/transaction.py | 2 +- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 001c8343..2526743a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,18 +6,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased -### Changed - -- `micropip.install` now installs wheels from PyPI or custom indexes before built-in - Pyodide packages, reversing the previous behavior. - [#181](https://github.com/pyodide/micropip/pull/181) - -### Added - -- Added support for constraining resolved requirements via - `micropip.install(..., constraints=[...])`. and `micropip.set_constraints([...])` - [#181](https://github.com/pyodide/micropip/pull/181) - ### Fixed - Fix a bug that prevented non-standard relative urls to be treated as such @@ -28,6 +16,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `micropip` now vendors `pypa/packaging` for better reliability. [#178](https://github.com/pyodide/micropip/pull/178) +- `micropip.install` adds optional `constraints`, similar to `pip install -c`, + which refine the version or direct URLs of requested packages and their + dependencies. This includes built-in packages, which are now installed after + any requested or constrained external packages. + [#181](https://github.com/pyodide/micropip/pull/181) +- `micropip.set_constraints` sets default constraints for later + calls to `micropip.install` that do not specify constraints. + [#181](https://github.com/pyodide/micropip/pull/181) ## [0.8.0] - 2024/12/15 diff --git a/micropip/transaction.py b/micropip/transaction.py index cfbd3aaf..7f358936 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -213,7 +213,7 @@ async def _add_requirement_from_pyodide_lock(self, req: Requirement) -> bool: self.pyodide_packages.append( PackageMetadata(name=req.name, version=str(version), source="pyodide") ) - if locked_package["depends"]: + if self.constraints and locked_package["depends"]: await self.gather_requirements(locked_package["depends"]) return True From 00e6e7ae7e5f954849943a67377b994f9da192f7 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Thu, 30 Jan 2025 12:50:48 -0600 Subject: [PATCH 19/19] try more url checking --- micropip/_utils.py | 2 +- micropip/transaction.py | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/micropip/_utils.py b/micropip/_utils.py index e213f8ed..e9c6c92b 100644 --- a/micropip/_utils.py +++ b/micropip/_utils.py @@ -316,7 +316,7 @@ def validate_constraints( if req.extras: messages.append("may not provide [extras]") - if not (req.url or req.specifier): + if not (req.url or len(req.specifier)): messages.append("no version or URL") if req.marker and not req.marker.evaluate(environment): diff --git a/micropip/transaction.py b/micropip/transaction.py index 7f358936..a13a54f6 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -75,22 +75,24 @@ async def add_requirement(self, req: str | Requirement) -> None: return await self.add_requirement_inner(req) try: - req = constrain_requirement(Requirement(req), self.constrained_reqs) - url = req.url + as_req = constrain_requirement(Requirement(req), self.constrained_reqs) except InvalidRequirement: - url = f"{req}" + as_req = None - if not (url and urlparse(url).path.endswith(".whl")): - return await self.add_requirement_inner( - req if isinstance(req, Requirement) else Requirement(req) - ) + if as_req: + if as_req.name and len(as_req.specifier): + return await self.add_requirement_inner(as_req) + if as_req.url: + req = as_req.url - if url: + if urlparse(req).path.endswith(".whl"): # custom download location - wheel = WheelInfo.from_url(url) + wheel = WheelInfo.from_url(req) check_compatible(wheel.filename) return await self.add_wheel(wheel, extras=set(), specifier="") + return await self.add_requirement_inner(Requirement(req)) + def check_version_satisfied(self, req: Requirement) -> tuple[bool, str]: ver = None try: