From 6854f9ea3589db0c24075367b21c8b180e2d6722 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Tue, 26 Nov 2024 09:48:19 +0100 Subject: [PATCH 01/12] First search in installed packages --- .gitignore | 4 +- findlibs/__init__.py | 150 +++++++++++++++++++++++++++---------------- 2 files changed, 98 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 152ddae..869b8ae 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,6 @@ __pycache__/ .vscode/*.code-snippets # Ignore code-workspaces -*.code-workspace \ No newline at end of file +*.code-workspace + +*swp diff --git a/findlibs/__init__.py b/findlibs/__init__.py index 519ced2..11f5f8e 100644 --- a/findlibs/__init__.py +++ b/findlibs/__init__.py @@ -10,6 +10,7 @@ import configparser import ctypes.util +import importlib import os import sys from pathlib import Path @@ -21,6 +22,50 @@ "win32": ".dll", } +def _find_in_package(lib_name: str, pkg_name: str) -> str|None: + """Tries to find the library in an installed python module `{pgk_name}libs`. + This is a convention used by, for example, by newly built binary-only ecmwf + packages, such as eckit dlibs in the "eckitlib" python module.""" + # NOTE we could have searched for relative location wrt __file__ -- but that + # breaks eg editable installs of findlibs, conda-venv combinations, etc. + # The price we pay is that the binary packages have to be importible, ie, + # the default output of auditwheel wont work + try: + module = importlib.import_module(pkg_name + "libs") + venv_wheel_lib = str((Path(module.__file__) / '..' / lib_name).resolve()) + if os.path.exists(venv_wheel_lib): + return venv_wheel_lib + except ImportError: + pass + return None + +def _find_in_python(lib_name: str, pkg_name: str) -> str|None: + """Tries to find the library installed directly to Conda/Python sys.prefix + libs""" + roots = [sys.prefix] + if "CONDA_PREFIX" in os.environ: + roots.append(os.environ["CONDA_PREFIX"]) + + for root in roots: + for lib in ("lib", "lib64"): + fullname = os.path.join(root, lib, lib_name) + if os.path.exists(fullname): + return fullname + return None + +def _find_in_home(lib_name: str, pkg_name: str) -> str|None: + env_prefixes = [pkg_name.upper(), pkg_name.lower()] + env_suffixes = ["HOME", "DIR"] + envs = ["{}_{}".format(x, y) for x in env_prefixes for y in env_suffixes] + + for env in envs: + if env in os.environ: + home = os.path.expanduser(os.environ[env]) + for lib in ("lib", "lib64"): + fullname = os.path.join(home, lib, lib_name) + if os.path.exists(fullname): + return fullname + return None def _get_paths_from_config(): locations = [ @@ -71,72 +116,27 @@ def _get_paths_from_config(): return paths - -def find(lib_name, pkg_name=None): - """Returns the path to the selected library, or None if not found. - - Arguments - --------- - lib_name : str - Library name without the `lib` prefix. The name of the library to - find is formed using ``lib_name`` and a platform specific suffix - (by default ".so"). E.g. when ``lib_name`` is "eccodes" the library - name will be "libeccodes.so" on Linux and "libeccodes.dylib" - on macOS. - pkg_name : str, optional - Package name if it differs from the library name. Defaults to None. - - Returns - -------- - str or None - Path to selected library - """ - pkg_name = pkg_name or lib_name - extension = EXTENSIONS.get(sys.platform, ".so") - libname = "lib{}{}".format(lib_name, extension) - - # sys.prefix/lib, $CONDA_PREFIX/lib has highest priority; - # otherwise, system library may mess up anaconda's virtual environment. - - roots = [sys.prefix] - if "CONDA_PREFIX" in os.environ: - roots.append(os.environ["CONDA_PREFIX"]) - - for root in roots: - for lib in ("lib", "lib64"): - fullname = os.path.join(root, lib, libname) - if os.path.exists(fullname): - return fullname - - env_prefixes = [pkg_name.upper(), pkg_name.lower()] - env_suffixes = ["HOME", "DIR"] - envs = ["{}_{}".format(x, y) for x in env_prefixes for y in env_suffixes] - - for env in envs: - if env in os.environ: - home = os.path.expanduser(os.environ[env]) - for lib in ("lib", "lib64"): - fullname = os.path.join(home, lib, libname) - if os.path.exists(fullname): - return fullname - - config_paths = _get_paths_from_config() - - for root in config_paths: +def _find_in_config_paths(lib_name: str, pkg_name: str) -> str|None: + paths = _get_paths_from_config() + for root in paths: for lib in ("lib", "lib64"): - filepath = root / lib / f"lib{lib_name}{extension}" + filepath = root / lib / lib_name if filepath.exists(): return str(filepath) + return None +def _find_in_ld_path(lib_name: str, pkg_name: str) -> str|None: for path in ( "LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH", ): for home in os.environ.get(path, "").split(":"): - fullname = os.path.join(home, libname) + fullname = os.path.join(home, lib_name) if os.path.exists(fullname): return fullname + return None +def _find_in_sys(lib_name: str, pkg_name: str) -> str|None: for root in ( "/", "/usr/", @@ -146,8 +146,48 @@ def find(lib_name, pkg_name=None): os.path.expanduser("~/.local"), ): for lib in ("lib", "lib64"): - fullname = os.path.join(root, lib, libname) + fullname = os.path.join(root, lib, lib_name) if os.path.exists(fullname): return fullname + return None +def _find_in_ctypes_util(lib_name: str, pkg_name: str) -> str|None: return ctypes.util.find_library(lib_name) + +def find(lib_name: str, pkg_name: str|None = None) -> str|None: + """Returns the path to the selected library, or None if not found. + + Arguments + --------- + lib_name : str + Library name without the `lib` prefix. The name of the library to + find is formed using ``lib_name`` and a platform specific suffix + (by default ".so"). E.g. when ``lib_name`` is "eccodes" the library + name will be "libeccodes.so" on Linux and "libeccodes.dylib" + on macOS. + pkg_name : str, optional + Package name if it differs from the library name. Defaults to None. + + Returns + -------- + str or None + Path to selected library + """ + pkg_name = pkg_name or lib_name + extension = EXTENSIONS.get(sys.platform, ".so") + lib_name = "lib{}{}".format(lib_name, extension) + + sources = [ + _find_in_package, + _find_in_python, + _find_in_home, + _find_in_config_paths, + _find_in_ld_path, + _find_in_sys, + _find_in_ctypes_util, + ] + + for source in sources: + if (result := source(lib_name, pkg_name)): + return result + return None From 4a06d95e175c158192898478bc2b4c26c5afbc38 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Fri, 6 Dec 2024 13:51:15 +0100 Subject: [PATCH 02/12] Enable fine-grained source disabling --- findlibs/__init__.py | 67 ++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/findlibs/__init__.py b/findlibs/__init__.py index 11f5f8e..438b2d9 100644 --- a/findlibs/__init__.py +++ b/findlibs/__init__.py @@ -22,7 +22,8 @@ "win32": ".dll", } -def _find_in_package(lib_name: str, pkg_name: str) -> str|None: + +def _find_in_package(lib_name: str, pkg_name: str) -> str | None: """Tries to find the library in an installed python module `{pgk_name}libs`. This is a convention used by, for example, by newly built binary-only ecmwf packages, such as eckit dlibs in the "eckitlib" python module.""" @@ -32,14 +33,15 @@ def _find_in_package(lib_name: str, pkg_name: str) -> str|None: # the default output of auditwheel wont work try: module = importlib.import_module(pkg_name + "libs") - venv_wheel_lib = str((Path(module.__file__) / '..' / lib_name).resolve()) + venv_wheel_lib = str((Path(module.__file__) / ".." / lib_name).resolve()) if os.path.exists(venv_wheel_lib): return venv_wheel_lib except ImportError: pass return None -def _find_in_python(lib_name: str, pkg_name: str) -> str|None: + +def _find_in_python(lib_name: str, pkg_name: str) -> str | None: """Tries to find the library installed directly to Conda/Python sys.prefix libs""" roots = [sys.prefix] @@ -53,7 +55,8 @@ def _find_in_python(lib_name: str, pkg_name: str) -> str|None: return fullname return None -def _find_in_home(lib_name: str, pkg_name: str) -> str|None: + +def _find_in_home(lib_name: str, pkg_name: str) -> str | None: env_prefixes = [pkg_name.upper(), pkg_name.lower()] env_suffixes = ["HOME", "DIR"] envs = ["{}_{}".format(x, y) for x in env_prefixes for y in env_suffixes] @@ -67,6 +70,7 @@ def _find_in_home(lib_name: str, pkg_name: str) -> str|None: return fullname return None + def _get_paths_from_config(): locations = [ Path(p).expanduser() @@ -116,7 +120,8 @@ def _get_paths_from_config(): return paths -def _find_in_config_paths(lib_name: str, pkg_name: str) -> str|None: + +def _find_in_config_paths(lib_name: str, pkg_name: str) -> str | None: paths = _get_paths_from_config() for root in paths: for lib in ("lib", "lib64"): @@ -125,7 +130,8 @@ def _find_in_config_paths(lib_name: str, pkg_name: str) -> str|None: return str(filepath) return None -def _find_in_ld_path(lib_name: str, pkg_name: str) -> str|None: + +def _find_in_ld_path(lib_name: str, pkg_name: str) -> str | None: for path in ( "LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH", @@ -136,7 +142,8 @@ def _find_in_ld_path(lib_name: str, pkg_name: str) -> str|None: return fullname return None -def _find_in_sys(lib_name: str, pkg_name: str) -> str|None: + +def _find_in_sys(lib_name: str, pkg_name: str) -> str | None: for root in ( "/", "/usr/", @@ -151,11 +158,24 @@ def _find_in_sys(lib_name: str, pkg_name: str) -> str|None: return fullname return None -def _find_in_ctypes_util(lib_name: str, pkg_name: str) -> str|None: + +def _find_in_ctypes_util(lib_name: str, pkg_name: str) -> str | None: return ctypes.util.find_library(lib_name) -def find(lib_name: str, pkg_name: str|None = None) -> str|None: + +def find(lib_name: str, pkg_name: str | None = None) -> str | None: """Returns the path to the selected library, or None if not found. + Searches over multiple sources in this order: + - importible python module ("PACKAGE") + - python's sys.prefix and conda's libs ("PYTHON") + - package's home like ECCODES_HOME ("HOME") + - findlibs config like .findlibs ("CONFIG_PATHS") + - ld library path ("LD_PATH") + - system's libraries ("SYS") + - invocation of ctypes.util ("CTYPES_UTIL") + each can be disabled via setting FINDLIBS_DISABLE_{method} to "yes", + so eg `export FINDLIBS_DISABLE_PACKAGE=yes`. Consult the code for each + individual method implementation and further configurability. Arguments --------- @@ -177,17 +197,22 @@ def find(lib_name: str, pkg_name: str|None = None) -> str|None: extension = EXTENSIONS.get(sys.platform, ".so") lib_name = "lib{}{}".format(lib_name, extension) - sources = [ - _find_in_package, - _find_in_python, - _find_in_home, - _find_in_config_paths, - _find_in_ld_path, - _find_in_sys, - _find_in_ctypes_util, - ] - - for source in sources: - if (result := source(lib_name, pkg_name)): + sources = ( + (_find_in_package, "PACKAGE"), + (_find_in_python, "PYTHON"), + (_find_in_home, "HOME"), + (_find_in_config_paths, "CONFIG_PATHS"), + (_find_in_ld_path, "LD_PATH"), + (_find_in_sys, "SYS"), + (_find_in_ctypes_util, "CTYPES_UTIL"), + ) + sources_filtered = ( + source_clb + for source_clb, source_name in sources + if os.environ.get(f"FINDLIBS_DISABLE_{source_name}", None) != "yes" + ) + + for source in sources_filtered: + if result := source(lib_name, pkg_name): return result return None From 35db5273e64dd123658edcafcf3928bc4ed21c2c Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Fri, 6 Dec 2024 14:56:47 +0100 Subject: [PATCH 03/12] Allow transitive dependency search --- findlibs/__init__.py | 69 +++++++++++++++++++++++---- tests/transitive/__init__.py | 0 tests/transitive/modAlibs/__init__.py | 1 + tests/transitive/modAlibs/libmodA.so | 0 tests/transitive/modBlibs/__init__.py | 0 tests/transitive/test_transitive.py | 24 ++++++++++ 6 files changed, 84 insertions(+), 10 deletions(-) create mode 100644 tests/transitive/__init__.py create mode 100644 tests/transitive/modAlibs/__init__.py create mode 100644 tests/transitive/modAlibs/libmodA.so create mode 100644 tests/transitive/modBlibs/__init__.py create mode 100644 tests/transitive/test_transitive.py diff --git a/findlibs/__init__.py b/findlibs/__init__.py index 438b2d9..0300efb 100644 --- a/findlibs/__init__.py +++ b/findlibs/__init__.py @@ -13,27 +13,72 @@ import importlib import os import sys +import warnings +from collections import defaultdict from pathlib import Path +from types import ModuleType __version__ = "0.0.5" -EXTENSIONS = { - "darwin": ".dylib", - "win32": ".dll", -} - - -def _find_in_package(lib_name: str, pkg_name: str) -> str | None: +EXTENSIONS = defaultdict( + lambda: ".so", + darwin=".dylib", + win32=".dll", +) +DYLIB_PATH = defaultdict( + lambda: "LD_LIBRARY_PATH", + darwin="DYLD_LIBRARY_PATH", + # win32? May be trickier +) + + +def _extend_dylib_path_with(p: str) -> None: + """See _find_in_package""" + current = os.environ.get(DYLIB_PATH[sys.platform], "") + if not current: + extended = p + else: + extended = f"{p}:{current}" + os.environ[DYLIB_PATH[sys.platform]] = extended + + +def _transitive_dylib_path_extension(module: ModuleType) -> None: + """See _find_in_package""" + # NOTE consider replacing hasattr with entrypoint-based declaration + # https://packaging.python.org/en/latest/specifications/entry-points/ + if hasattr(module, "findlibs_dependencies"): + for module_name in module.findlibs_dependencies: + try: + rec_into = importlib.import_module(module_name) + ext_path = str(Path(rec_into.__file__).parent) + _extend_dylib_path_with(ext_path) + _transitive_dylib_path_extension(rec_into) + except ImportError: + # NOTE we don't use ImportWarning here as thats off by default + warnings.warn( + f"unable to import {module_name} yet declared as dependency of {module.__name__}" + ) + + +def _find_in_package( + lib_name: str, pkg_name: str, trans_ext_dylib: bool = True +) -> str | None: """Tries to find the library in an installed python module `{pgk_name}libs`. This is a convention used by, for example, by newly built binary-only ecmwf - packages, such as eckit dlibs in the "eckitlib" python module.""" + packages, such as eckit dlibs in the "eckitlib" python module. + + If trans_ext_dylib is True, it additionally extends platform linker's dylib path + (LD_LIBRARY_PATH / DYLD_LIBRARY_PATH) with dependencies declared in the module's + init. This is needed if the `.so`s in the wheel don't have correct rpath""" # NOTE we could have searched for relative location wrt __file__ -- but that # breaks eg editable installs of findlibs, conda-venv combinations, etc. # The price we pay is that the binary packages have to be importible, ie, # the default output of auditwheel wont work try: module = importlib.import_module(pkg_name + "libs") - venv_wheel_lib = str((Path(module.__file__) / ".." / lib_name).resolve()) + if trans_ext_dylib: + _transitive_dylib_path_extension(module) + venv_wheel_lib = str((Path(module.__file__).parent / lib_name)) if os.path.exists(venv_wheel_lib): return venv_wheel_lib except ImportError: @@ -160,6 +205,10 @@ def _find_in_sys(lib_name: str, pkg_name: str) -> str | None: def _find_in_ctypes_util(lib_name: str, pkg_name: str) -> str | None: + # NOTE this is a bit unreliable function, as for some libraries/sources, + # it returns full path, in others just a filename. It still may be worth + # it as a fallback even in the filename-only case, to help troubleshoot some + # yet unknown source return ctypes.util.find_library(lib_name) @@ -194,7 +243,7 @@ def find(lib_name: str, pkg_name: str | None = None) -> str | None: Path to selected library """ pkg_name = pkg_name or lib_name - extension = EXTENSIONS.get(sys.platform, ".so") + extension = EXTENSIONS[sys.platform] lib_name = "lib{}{}".format(lib_name, extension) sources = ( diff --git a/tests/transitive/__init__.py b/tests/transitive/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/transitive/modAlibs/__init__.py b/tests/transitive/modAlibs/__init__.py new file mode 100644 index 0000000..539462a --- /dev/null +++ b/tests/transitive/modAlibs/__init__.py @@ -0,0 +1 @@ +findlibs_dependencies = ["modBlibs"] diff --git a/tests/transitive/modAlibs/libmodA.so b/tests/transitive/modAlibs/libmodA.so new file mode 100644 index 0000000..e69de29 diff --git a/tests/transitive/modBlibs/__init__.py b/tests/transitive/modBlibs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/transitive/test_transitive.py b/tests/transitive/test_transitive.py new file mode 100644 index 0000000..fe90176 --- /dev/null +++ b/tests/transitive/test_transitive.py @@ -0,0 +1,24 @@ +import os +import sys +from pathlib import Path + +from findlibs import DYLIB_PATH, find + + +def test_transitive() -> None: + """There is a module modAlibs in this directory that mocks expected bin wheel contract: + - modulename ends with 'libs' + - contains libmodA.so + - inside __init__ there is the findlibs_dependencies, containing modBlibs + This test checks that when such module is findlibs-found, it extend the platform's dylib + env var with the full path to the modBlibs module + """ + + sys.path.append(str(Path(__file__).parent)) + + found = find("modA") + expected_found = str(Path(__file__).parent / "modAlibs" / "libmodA.so") + assert found == expected_found + + expected_dylib = str(Path(__file__).parent / "modBlibs") + assert expected_dylib in os.environ[DYLIB_PATH[sys.platform]] From 3508f77e7bca0017bc15b4b67d066a76d013dd05 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Fri, 6 Dec 2024 17:08:42 +0100 Subject: [PATCH 04/12] Replace ld lib path exts with preloads --- findlibs/__init__.py | 44 ++++++++++++---------------- tests/transitive/modBlibs/libmodB.so | 0 tests/transitive/test_transitive.py | 21 +++++++++---- 3 files changed, 34 insertions(+), 31 deletions(-) create mode 100644 tests/transitive/modBlibs/libmodB.so diff --git a/findlibs/__init__.py b/findlibs/__init__.py index 0300efb..121d0ef 100644 --- a/findlibs/__init__.py +++ b/findlibs/__init__.py @@ -15,6 +15,7 @@ import sys import warnings from collections import defaultdict +from ctypes import CDLL from pathlib import Path from types import ModuleType @@ -25,24 +26,16 @@ darwin=".dylib", win32=".dll", ) -DYLIB_PATH = defaultdict( - lambda: "LD_LIBRARY_PATH", - darwin="DYLD_LIBRARY_PATH", - # win32? May be trickier -) -def _extend_dylib_path_with(p: str) -> None: +def _single_preload_deps(path: str) -> None: """See _find_in_package""" - current = os.environ.get(DYLIB_PATH[sys.platform], "") - if not current: - extended = p - else: - extended = f"{p}:{current}" - os.environ[DYLIB_PATH[sys.platform]] = extended + for lib in os.listdir(path): + if lib.endswith(".so"): + _ = CDLL(f"{path}/{lib}") -def _transitive_dylib_path_extension(module: ModuleType) -> None: +def _transitive_preload_deps(module: ModuleType) -> None: """See _find_in_package""" # NOTE consider replacing hasattr with entrypoint-based declaration # https://packaging.python.org/en/latest/specifications/entry-points/ @@ -51,8 +44,10 @@ def _transitive_dylib_path_extension(module: ModuleType) -> None: try: rec_into = importlib.import_module(module_name) ext_path = str(Path(rec_into.__file__).parent) - _extend_dylib_path_with(ext_path) - _transitive_dylib_path_extension(rec_into) + # NOTE we need *first* to evaluate recursive call, *then* preload, + # to ensure that dependencies are already in place + _transitive_preload_deps(rec_into) + _single_preload_deps(ext_path) except ImportError: # NOTE we don't use ImportWarning here as thats off by default warnings.warn( @@ -61,23 +56,22 @@ def _transitive_dylib_path_extension(module: ModuleType) -> None: def _find_in_package( - lib_name: str, pkg_name: str, trans_ext_dylib: bool = True + lib_name: str, pkg_name: str, preload_deps: bool = True ) -> str | None: """Tries to find the library in an installed python module `{pgk_name}libs`. This is a convention used by, for example, by newly built binary-only ecmwf packages, such as eckit dlibs in the "eckitlib" python module. - If trans_ext_dylib is True, it additionally extends platform linker's dylib path - (LD_LIBRARY_PATH / DYLD_LIBRARY_PATH) with dependencies declared in the module's - init. This is needed if the `.so`s in the wheel don't have correct rpath""" - # NOTE we could have searched for relative location wrt __file__ -- but that - # breaks eg editable installs of findlibs, conda-venv combinations, etc. - # The price we pay is that the binary packages have to be importible, ie, - # the default output of auditwheel wont work + If preload deps is True, it additionally opens all dylibs of this library and its + transitive dependencies This is needed if the `.so`s in the wheel don't have + correct rpath -- which is effectively impossible in non-trivial venvs. + + It would be tempting to just extend LD_LIBRARY_PATH -- alas, that won't have any + effect as the linker has been configured already by the time cpython is running""" try: module = importlib.import_module(pkg_name + "libs") - if trans_ext_dylib: - _transitive_dylib_path_extension(module) + if preload_deps: + _transitive_preload_deps(module) venv_wheel_lib = str((Path(module.__file__).parent / lib_name)) if os.path.exists(venv_wheel_lib): return venv_wheel_lib diff --git a/tests/transitive/modBlibs/libmodB.so b/tests/transitive/modBlibs/libmodB.so new file mode 100644 index 0000000..e69de29 diff --git a/tests/transitive/test_transitive.py b/tests/transitive/test_transitive.py index fe90176..c386ed1 100644 --- a/tests/transitive/test_transitive.py +++ b/tests/transitive/test_transitive.py @@ -1,11 +1,10 @@ -import os import sys from pathlib import Path -from findlibs import DYLIB_PATH, find +import findlibs -def test_transitive() -> None: +def test_transitive(monkeypatch) -> None: """There is a module modAlibs in this directory that mocks expected bin wheel contract: - modulename ends with 'libs' - contains libmodA.so @@ -14,11 +13,21 @@ def test_transitive() -> None: env var with the full path to the modBlibs module """ + # so that modAlibs and modBlibs are visible sys.path.append(str(Path(__file__).parent)) - found = find("modA") + # the files in test are not real .so, we thus just track what got loaded + loaded_libs = set() + + def libload_accumulator(path: str): + loaded_libs.add(path) + + monkeypatch.setattr(findlibs, "CDLL", libload_accumulator) + + # test + found = findlibs.find("modA") expected_found = str(Path(__file__).parent / "modAlibs" / "libmodA.so") assert found == expected_found - expected_dylib = str(Path(__file__).parent / "modBlibs") - assert expected_dylib in os.environ[DYLIB_PATH[sys.platform]] + expected_dylib = str(Path(__file__).parent / "modBlibs" / "libmodB.so") + assert loaded_libs == {expected_dylib} From 0c26e72b251fbb999ee743de4e4cd340a0f56b83 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 9 Dec 2024 11:29:43 +0100 Subject: [PATCH 05/12] Add logging basis --- findlibs/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/findlibs/__init__.py b/findlibs/__init__.py index 121d0ef..1698ca8 100644 --- a/findlibs/__init__.py +++ b/findlibs/__init__.py @@ -11,6 +11,7 @@ import configparser import ctypes.util import importlib +import logging import os import sys import warnings @@ -21,6 +22,8 @@ __version__ = "0.0.5" +logger = logging.getLogger(__name__) + EXTENSIONS = defaultdict( lambda: ".so", darwin=".dylib", @@ -256,6 +259,8 @@ def find(lib_name: str, pkg_name: str | None = None) -> str | None: ) for source in sources_filtered: + logger.debug(f"about to search for {lib_name}/{pkg_name} in {source}") if result := source(lib_name, pkg_name): + logger.debug(f"found {lib_name}/{pkg_name} in {source}") return result return None From 9ecc0854d5d2628c2765553436750e85af77e517 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 9 Dec 2024 15:45:59 +0100 Subject: [PATCH 06/12] Switch to rich binary wheels --- findlibs/__init__.py | 11 +++++++---- tests/transitive/modAlibs/__init__.py | 2 +- tests/transitive/modAlibs/{ => lib64}/libmodA.so | 0 tests/transitive/modBlibs/{ => lib64}/libmodB.so | 0 tests/transitive/test_transitive.py | 4 ++-- 5 files changed, 10 insertions(+), 7 deletions(-) rename tests/transitive/modAlibs/{ => lib64}/libmodA.so (100%) rename tests/transitive/modBlibs/{ => lib64}/libmodB.so (100%) diff --git a/findlibs/__init__.py b/findlibs/__init__.py index 1698ca8..e4b6c63 100644 --- a/findlibs/__init__.py +++ b/findlibs/__init__.py @@ -30,6 +30,8 @@ win32=".dll", ) +binary_module_name = lambda s: f"{s}libs" # noqa: E731 + def _single_preload_deps(path: str) -> None: """See _find_in_package""" @@ -45,11 +47,12 @@ def _transitive_preload_deps(module: ModuleType) -> None: if hasattr(module, "findlibs_dependencies"): for module_name in module.findlibs_dependencies: try: - rec_into = importlib.import_module(module_name) - ext_path = str(Path(rec_into.__file__).parent) + rec_into = importlib.import_module(binary_module_name(module_name)) # NOTE we need *first* to evaluate recursive call, *then* preload, # to ensure that dependencies are already in place _transitive_preload_deps(rec_into) + + ext_path = str(Path(rec_into.__file__).parent / "lib64") _single_preload_deps(ext_path) except ImportError: # NOTE we don't use ImportWarning here as thats off by default @@ -72,10 +75,10 @@ def _find_in_package( It would be tempting to just extend LD_LIBRARY_PATH -- alas, that won't have any effect as the linker has been configured already by the time cpython is running""" try: - module = importlib.import_module(pkg_name + "libs") + module = importlib.import_module(binary_module_name(pkg_name)) if preload_deps: _transitive_preload_deps(module) - venv_wheel_lib = str((Path(module.__file__).parent / lib_name)) + venv_wheel_lib = str((Path(module.__file__).parent / "lib64" / lib_name)) if os.path.exists(venv_wheel_lib): return venv_wheel_lib except ImportError: diff --git a/tests/transitive/modAlibs/__init__.py b/tests/transitive/modAlibs/__init__.py index 539462a..392565f 100644 --- a/tests/transitive/modAlibs/__init__.py +++ b/tests/transitive/modAlibs/__init__.py @@ -1 +1 @@ -findlibs_dependencies = ["modBlibs"] +findlibs_dependencies = ["modB"] diff --git a/tests/transitive/modAlibs/libmodA.so b/tests/transitive/modAlibs/lib64/libmodA.so similarity index 100% rename from tests/transitive/modAlibs/libmodA.so rename to tests/transitive/modAlibs/lib64/libmodA.so diff --git a/tests/transitive/modBlibs/libmodB.so b/tests/transitive/modBlibs/lib64/libmodB.so similarity index 100% rename from tests/transitive/modBlibs/libmodB.so rename to tests/transitive/modBlibs/lib64/libmodB.so diff --git a/tests/transitive/test_transitive.py b/tests/transitive/test_transitive.py index c386ed1..484488f 100644 --- a/tests/transitive/test_transitive.py +++ b/tests/transitive/test_transitive.py @@ -26,8 +26,8 @@ def libload_accumulator(path: str): # test found = findlibs.find("modA") - expected_found = str(Path(__file__).parent / "modAlibs" / "libmodA.so") + expected_found = str(Path(__file__).parent / "modAlibs" / "lib64" / "libmodA.so") assert found == expected_found - expected_dylib = str(Path(__file__).parent / "modBlibs" / "libmodB.so") + expected_dylib = str(Path(__file__).parent / "modBlibs" / "lib64" / "libmodB.so") assert loaded_libs == {expected_dylib} From e7d743113030dfff54f590a799bfcb4b48a4acc9 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Thu, 12 Dec 2024 10:21:19 +0100 Subject: [PATCH 07/12] Fix macos --- findlibs/__init__.py | 19 +++++++++++++------ tests/transitive/test_transitive.py | 1 + 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/findlibs/__init__.py b/findlibs/__init__.py index e4b6c63..96f0fb7 100644 --- a/findlibs/__init__.py +++ b/findlibs/__init__.py @@ -36,7 +36,7 @@ def _single_preload_deps(path: str) -> None: """See _find_in_package""" for lib in os.listdir(path): - if lib.endswith(".so"): + if lib.endswith(EXTENSIONS[sys.platform]): _ = CDLL(f"{path}/{lib}") @@ -52,8 +52,12 @@ def _transitive_preload_deps(module: ModuleType) -> None: # to ensure that dependencies are already in place _transitive_preload_deps(rec_into) - ext_path = str(Path(rec_into.__file__).parent / "lib64") - _single_preload_deps(ext_path) + for ext_path in ( + str(Path(rec_into.__file__).parent / "lib"), + str(Path(rec_into.__file__).parent / "lib64"), + ): + if os.path.exists(ext_path): + _single_preload_deps(ext_path) except ImportError: # NOTE we don't use ImportWarning here as thats off by default warnings.warn( @@ -78,9 +82,12 @@ def _find_in_package( module = importlib.import_module(binary_module_name(pkg_name)) if preload_deps: _transitive_preload_deps(module) - venv_wheel_lib = str((Path(module.__file__).parent / "lib64" / lib_name)) - if os.path.exists(venv_wheel_lib): - return venv_wheel_lib + for venv_wheel_lib in ( + str((Path(module.__file__).parent / "lib" / lib_name)), + str((Path(module.__file__).parent / "lib64" / lib_name)), + ): + if os.path.exists(venv_wheel_lib): + return venv_wheel_lib except ImportError: pass return None diff --git a/tests/transitive/test_transitive.py b/tests/transitive/test_transitive.py index 484488f..c455462 100644 --- a/tests/transitive/test_transitive.py +++ b/tests/transitive/test_transitive.py @@ -23,6 +23,7 @@ def libload_accumulator(path: str): loaded_libs.add(path) monkeypatch.setattr(findlibs, "CDLL", libload_accumulator) + findlibs.EXTENSIONS[sys.platform] = ".so" # test found = findlibs.find("modA") From ab59d12247ef3550b5108b4318fec057f4f46654 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 16 Dec 2024 13:04:13 +0100 Subject: [PATCH 08/12] Python package name consolidation --- findlibs/__init__.py | 20 ++++++++++--------- tests/transitive/modAlib/__init__.py | 1 + .../{modAlibs => modAlib}/lib64/libmodA.so | 0 tests/transitive/modAlibs/__init__.py | 1 - .../{modBlibs => modBlib}/__init__.py | 0 .../{modBlibs => modBlib}/lib64/libmodB.so | 0 tests/transitive/test_transitive.py | 11 +++++----- 7 files changed, 17 insertions(+), 16 deletions(-) create mode 100644 tests/transitive/modAlib/__init__.py rename tests/transitive/{modAlibs => modAlib}/lib64/libmodA.so (100%) delete mode 100644 tests/transitive/modAlibs/__init__.py rename tests/transitive/{modBlibs => modBlib}/__init__.py (100%) rename tests/transitive/{modBlibs => modBlib}/lib64/libmodB.so (100%) diff --git a/findlibs/__init__.py b/findlibs/__init__.py index 96f0fb7..b30fad5 100644 --- a/findlibs/__init__.py +++ b/findlibs/__init__.py @@ -30,8 +30,6 @@ win32=".dll", ) -binary_module_name = lambda s: f"{s}libs" # noqa: E731 - def _single_preload_deps(path: str) -> None: """See _find_in_package""" @@ -47,7 +45,7 @@ def _transitive_preload_deps(module: ModuleType) -> None: if hasattr(module, "findlibs_dependencies"): for module_name in module.findlibs_dependencies: try: - rec_into = importlib.import_module(binary_module_name(module_name)) + rec_into = importlib.import_module(module_name) # NOTE we need *first* to evaluate recursive call, *then* preload, # to ensure that dependencies are already in place _transitive_preload_deps(rec_into) @@ -68,9 +66,8 @@ def _transitive_preload_deps(module: ModuleType) -> None: def _find_in_package( lib_name: str, pkg_name: str, preload_deps: bool = True ) -> str | None: - """Tries to find the library in an installed python module `{pgk_name}libs`. - This is a convention used by, for example, by newly built binary-only ecmwf - packages, such as eckit dlibs in the "eckitlib" python module. + """Tries to find the library in an installed python module `{pgk_name}`. + Examples of packages with such expositions are `eckitlib` or `odclib`. If preload deps is True, it additionally opens all dylibs of this library and its transitive dependencies This is needed if the `.so`s in the wheel don't have @@ -79,7 +76,7 @@ def _find_in_package( It would be tempting to just extend LD_LIBRARY_PATH -- alas, that won't have any effect as the linker has been configured already by the time cpython is running""" try: - module = importlib.import_module(binary_module_name(pkg_name)) + module = importlib.import_module(pkg_name) if preload_deps: _transitive_preload_deps(module) for venv_wheel_lib in ( @@ -110,6 +107,9 @@ def _find_in_python(lib_name: str, pkg_name: str) -> str | None: def _find_in_home(lib_name: str, pkg_name: str) -> str | None: env_prefixes = [pkg_name.upper(), pkg_name.lower()] + if pkg_name.endswith("lib"): + # if eg "eckitlib" is pkg name, consider also "eckit" prefix + env_prefixes += [pkg_name.upper()[:-3], pkg_name.lower()[:-3]] env_suffixes = ["HOME", "DIR"] envs = ["{}_{}".format(x, y) for x in env_prefixes for y in env_suffixes] @@ -242,14 +242,16 @@ def find(lib_name: str, pkg_name: str | None = None) -> str | None: name will be "libeccodes.so" on Linux and "libeccodes.dylib" on macOS. pkg_name : str, optional - Package name if it differs from the library name. Defaults to None. + Package name if it differs from the library name. Defaults to None, + which sets it to f"{lib_name}lib". Used by python module import and + home sources, with the home source considering also `lib`-less name. Returns -------- str or None Path to selected library """ - pkg_name = pkg_name or lib_name + pkg_name = pkg_name or f"{lib_name}lib" extension = EXTENSIONS[sys.platform] lib_name = "lib{}{}".format(lib_name, extension) diff --git a/tests/transitive/modAlib/__init__.py b/tests/transitive/modAlib/__init__.py new file mode 100644 index 0000000..6749725 --- /dev/null +++ b/tests/transitive/modAlib/__init__.py @@ -0,0 +1 @@ +findlibs_dependencies = ["modBlib"] diff --git a/tests/transitive/modAlibs/lib64/libmodA.so b/tests/transitive/modAlib/lib64/libmodA.so similarity index 100% rename from tests/transitive/modAlibs/lib64/libmodA.so rename to tests/transitive/modAlib/lib64/libmodA.so diff --git a/tests/transitive/modAlibs/__init__.py b/tests/transitive/modAlibs/__init__.py deleted file mode 100644 index 392565f..0000000 --- a/tests/transitive/modAlibs/__init__.py +++ /dev/null @@ -1 +0,0 @@ -findlibs_dependencies = ["modB"] diff --git a/tests/transitive/modBlibs/__init__.py b/tests/transitive/modBlib/__init__.py similarity index 100% rename from tests/transitive/modBlibs/__init__.py rename to tests/transitive/modBlib/__init__.py diff --git a/tests/transitive/modBlibs/lib64/libmodB.so b/tests/transitive/modBlib/lib64/libmodB.so similarity index 100% rename from tests/transitive/modBlibs/lib64/libmodB.so rename to tests/transitive/modBlib/lib64/libmodB.so diff --git a/tests/transitive/test_transitive.py b/tests/transitive/test_transitive.py index c455462..0be912b 100644 --- a/tests/transitive/test_transitive.py +++ b/tests/transitive/test_transitive.py @@ -6,11 +6,10 @@ def test_transitive(monkeypatch) -> None: """There is a module modAlibs in this directory that mocks expected bin wheel contract: - - modulename ends with 'libs' + - modulename ends with 'lib' - contains libmodA.so - - inside __init__ there is the findlibs_dependencies, containing modBlibs - This test checks that when such module is findlibs-found, it extend the platform's dylib - env var with the full path to the modBlibs module + - inside __init__ there is the findlibs_dependencies, containing modBlib + This test checks that when such module is findlibs-found, the (mock) ld loaded the libmodB """ # so that modAlibs and modBlibs are visible @@ -27,8 +26,8 @@ def libload_accumulator(path: str): # test found = findlibs.find("modA") - expected_found = str(Path(__file__).parent / "modAlibs" / "lib64" / "libmodA.so") + expected_found = str(Path(__file__).parent / "modAlib" / "lib64" / "libmodA.so") assert found == expected_found - expected_dylib = str(Path(__file__).parent / "modBlibs" / "lib64" / "libmodB.so") + expected_dylib = str(Path(__file__).parent / "modBlib" / "lib64" / "libmodB.so") assert loaded_libs == {expected_dylib} From 3c7780ecb2a4fbbc4cf35132c0b84ef237c4a981 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Fri, 31 Jan 2025 10:56:45 +0100 Subject: [PATCH 09/12] More robust deps preload --- findlibs/__init__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/findlibs/__init__.py b/findlibs/__init__.py index b30fad5..c21b05b 100644 --- a/findlibs/__init__.py +++ b/findlibs/__init__.py @@ -13,6 +13,7 @@ import importlib import logging import os +import re import sys import warnings from collections import defaultdict @@ -29,13 +30,21 @@ darwin=".dylib", win32=".dll", ) +EXTENSIONS_RE = defaultdict( + lambda: r"^.*\.so(.[0-9]+)?$", + darwin=r"^.*\.dylib$", + win32=r"^.*\.dll$", +) def _single_preload_deps(path: str) -> None: """See _find_in_package""" for lib in os.listdir(path): - if lib.endswith(EXTENSIONS[sys.platform]): + print(f"considering {lib}") + if re.match(EXTENSIONS_RE[sys.platform], lib): + print(f"loading {lib}") _ = CDLL(f"{path}/{lib}") + print(f"loaded {lib}") def _transitive_preload_deps(module: ModuleType) -> None: From f102ba16ab7acfa3253458a61fbdf73f02b4efa2 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Mon, 3 Feb 2025 16:57:38 +0100 Subject: [PATCH 10/12] Fix debug logs --- findlibs/__init__.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/findlibs/__init__.py b/findlibs/__init__.py index c21b05b..0739221 100644 --- a/findlibs/__init__.py +++ b/findlibs/__init__.py @@ -39,12 +39,13 @@ def _single_preload_deps(path: str) -> None: """See _find_in_package""" + logger.debug(f"initiating recursive search at {path}") for lib in os.listdir(path): - print(f"considering {lib}") + logger.debug(f"considering {lib}") if re.match(EXTENSIONS_RE[sys.platform], lib): - print(f"loading {lib}") + logger.debug(f"loading {lib} at {path}") _ = CDLL(f"{path}/{lib}") - print(f"loaded {lib}") + logger.debug(f"loaded {lib}") def _transitive_preload_deps(module: ModuleType) -> None: @@ -53,6 +54,7 @@ def _transitive_preload_deps(module: ModuleType) -> None: # https://packaging.python.org/en/latest/specifications/entry-points/ if hasattr(module, "findlibs_dependencies"): for module_name in module.findlibs_dependencies: + logger.debug(f"consider transitive dependency preload of {module_name}") try: rec_into = importlib.import_module(module_name) # NOTE we need *first* to evaluate recursive call, *then* preload, @@ -67,9 +69,9 @@ def _transitive_preload_deps(module: ModuleType) -> None: _single_preload_deps(ext_path) except ImportError: # NOTE we don't use ImportWarning here as thats off by default - warnings.warn( - f"unable to import {module_name} yet declared as dependency of {module.__name__}" - ) + m = f"unable to import {module_name} yet declared as dependency of {module.__name__}" + warnings.warn(m) + logger.debug(m) def _find_in_package( @@ -86,6 +88,7 @@ def _find_in_package( effect as the linker has been configured already by the time cpython is running""" try: module = importlib.import_module(pkg_name) + logger.debug(f"found package {pkg_name}") if preload_deps: _transitive_preload_deps(module) for venv_wheel_lib in ( From 9ac85b993e431cedcee5adb113b4c3b2f0530542 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Tue, 4 Feb 2025 10:19:56 +0100 Subject: [PATCH 11/12] Turn off recursive preload on macos due to incompat --- findlibs/__init__.py | 6 ++++-- tests/transitive/test_transitive.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/findlibs/__init__.py b/findlibs/__init__.py index 0739221..6f8dfb1 100644 --- a/findlibs/__init__.py +++ b/findlibs/__init__.py @@ -75,7 +75,7 @@ def _transitive_preload_deps(module: ModuleType) -> None: def _find_in_package( - lib_name: str, pkg_name: str, preload_deps: bool = True + lib_name: str, pkg_name: str, preload_deps: bool|None = None ) -> str | None: """Tries to find the library in an installed python module `{pgk_name}`. Examples of packages with such expositions are `eckitlib` or `odclib`. @@ -86,9 +86,11 @@ def _find_in_package( It would be tempting to just extend LD_LIBRARY_PATH -- alas, that won't have any effect as the linker has been configured already by the time cpython is running""" + if preload_deps is None: + preload_deps = sys.platform != "darwin" # NOTE dyld doesnt seem to coop with ctypes.CDLL of weak-deps try: module = importlib.import_module(pkg_name) - logger.debug(f"found package {pkg_name}") + logger.debug(f"found package {pkg_name}; with {preload_deps=}") if preload_deps: _transitive_preload_deps(module) for venv_wheel_lib in ( diff --git a/tests/transitive/test_transitive.py b/tests/transitive/test_transitive.py index 0be912b..6108970 100644 --- a/tests/transitive/test_transitive.py +++ b/tests/transitive/test_transitive.py @@ -22,7 +22,7 @@ def libload_accumulator(path: str): loaded_libs.add(path) monkeypatch.setattr(findlibs, "CDLL", libload_accumulator) - findlibs.EXTENSIONS[sys.platform] = ".so" + monkeypatch.setattr(sys, "platform", "tests") # this makes the test behave like linux default # test found = findlibs.find("modA") From f2fbdfcccda44f1ccc761583b29a920e5ea15836 Mon Sep 17 00:00:00 2001 From: Iain Russell Date: Fri, 7 Feb 2025 13:27:08 +0000 Subject: [PATCH 12/12] Balck formatting --- findlibs/__init__.py | 6 ++++-- tests/transitive/test_transitive.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/findlibs/__init__.py b/findlibs/__init__.py index 6f8dfb1..376b7a2 100644 --- a/findlibs/__init__.py +++ b/findlibs/__init__.py @@ -75,7 +75,7 @@ def _transitive_preload_deps(module: ModuleType) -> None: def _find_in_package( - lib_name: str, pkg_name: str, preload_deps: bool|None = None + lib_name: str, pkg_name: str, preload_deps: bool | None = None ) -> str | None: """Tries to find the library in an installed python module `{pgk_name}`. Examples of packages with such expositions are `eckitlib` or `odclib`. @@ -87,7 +87,9 @@ def _find_in_package( It would be tempting to just extend LD_LIBRARY_PATH -- alas, that won't have any effect as the linker has been configured already by the time cpython is running""" if preload_deps is None: - preload_deps = sys.platform != "darwin" # NOTE dyld doesnt seem to coop with ctypes.CDLL of weak-deps + preload_deps = ( + sys.platform != "darwin" + ) # NOTE dyld doesnt seem to coop with ctypes.CDLL of weak-deps try: module = importlib.import_module(pkg_name) logger.debug(f"found package {pkg_name}; with {preload_deps=}") diff --git a/tests/transitive/test_transitive.py b/tests/transitive/test_transitive.py index 6108970..2b3e913 100644 --- a/tests/transitive/test_transitive.py +++ b/tests/transitive/test_transitive.py @@ -22,7 +22,9 @@ def libload_accumulator(path: str): loaded_libs.add(path) monkeypatch.setattr(findlibs, "CDLL", libload_accumulator) - monkeypatch.setattr(sys, "platform", "tests") # this makes the test behave like linux default + monkeypatch.setattr( + sys, "platform", "tests" + ) # this makes the test behave like linux default # test found = findlibs.find("modA")