Fix usage of deprecated libraries

.github/workflows/run-tests.yml

@@ -68,7 +68,7 @@ jobs:
           pip install -r requirements-dev.txt
       - name: Build hivemind
         run: |
-          pip install . --global-option=build_py --global-option="--buildgo" --no-use-pep517
+          pip install . --config-settings=build_py.buildgo=true
       - name: Test
         run: |
           cd tests
@@ -98,7 +98,7 @@ jobs:
           pip install bitsandbytes==0.45.2
       - name: Build hivemind
         run: |
-          pip install -e . --no-use-pep517
+          pip install -e .
       - name: Test
         run: |
           export HIVEMIND_MEMORY_SHARING_STRATEGY=file_descriptor

hivemind/p2p/p2p_daemon.py

@@ -8,7 +8,7 @@
 from contextlib import closing, suppress
 from dataclasses import dataclass
 from datetime import datetime
-from importlib.resources import path
+from importlib.resources import files
 from typing import Any, AsyncIterator, Awaitable, Callable, List, Optional, Sequence, Tuple, Type, TypeVar, Union
 
 from google.protobuf.message import Message
@@ -158,8 +158,7 @@ async def create(
             )
 
         self = cls()
-        with path(cli, P2PD_FILENAME) as p:
-            p2pd_path = p
+        p2pd_path = files(cli) / P2PD_FILENAME
 
         socket_uid = secrets.token_urlsafe(8)
         self._daemon_listen_maddr = Multiaddr(cls._UNIX_SOCKET_PREFIX + f"p2pd-{socket_uid}.sock")

modal_ci.py

@@ -72,12 +72,10 @@ def setup_environment(*, build_p2pd=False):
             "install",
             "--no-cache-dir",
             ".",
-            "--global-option=build_py",
-            "--global-option=--buildgo",
-            "--no-use-pep517",
+            "--config-settings=build_py.buildgo=true",
         ]
     else:
-        install_cmd = ["pip", "install", "-e", ".", "--no-use-pep517"]
+        install_cmd = ["pip", "install", "-e", "."]
 
     subprocess.run(install_cmd, check=True)
 

pyproject.toml

@@ -1,3 +1,58 @@
+[build-system]
+requires = ["setuptools>=64", "grpcio-tools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hivemind"
+description = "Decentralized deep learning in PyTorch"
+readme = "README.md"
+authors = [{name = "Learning@home & contributors", email = "[email protected]"}]
+license = "MIT"
+requires-python = ">=3.9"
+keywords = ["pytorch", "deep learning", "machine learning", "gpu", "distributed computing", "volunteer computing", "dht"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Scientific/Engineering :: Mathematics",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dynamic = ["version", "dependencies"]
+
+[project.optional-dependencies]
+bitsandbytes = ["bitsandbytes~=0.45.2"]
+
+[project.scripts]
+hivemind-dht = "hivemind.hivemind_cli.run_dht:main"
+hivemind-server = "hivemind.hivemind_cli.run_server:main"
+
+[project.urls]
+Homepage = "https://github.com/learning-at-home/hivemind"
+Repository = "https://github.com/learning-at-home/hivemind"
+
+[tool.setuptools]
+packages = {find = {}}
+include-package-data = true
+
+[tool.setuptools.dynamic]
+version = {attr = "hivemind.__version__"}
+dependencies = {file = ["requirements.txt"]}
+optional-dependencies.dev = {file = ["requirements-dev.txt"]}
+optional-dependencies.docs = {file = ["requirements-docs.txt"]}
+optional-dependencies.all = {file = ["requirements-all.txt"]}
+
+[tool.setuptools.package-data]
+hivemind = ["proto/*", "hivemind_cli/*"]
+
 [tool.coverage.run]
 concurrency = ["thread", "multiprocessing"]
 omit = ["hivemind/proto/*"]

requirements.txt

@@ -7,7 +7,7 @@ msgpack>=0.5.6
 sortedcontainers
 uvloop>=0.14.0
 grpcio-tools>=1.33.2
-protobuf>=5.29.0
+protobuf>=6.31.0
 configargparse>=1.2.3
 py-multihash>=0.2.3
 cryptography>=3.4.6

setup.py

@@ -1,4 +1,3 @@
-import codecs
 import glob
 import hashlib
 import os
@@ -9,8 +8,7 @@
 import tempfile
 import urllib.request
 
-from pkg_resources import parse_requirements, parse_version
-from setuptools import find_packages, setup
+from setuptools import setup
 from setuptools.command.build_py import build_py
 from setuptools.command.develop import develop
 
@@ -64,9 +62,11 @@ def build_p2p_daemon():
 
     if m is None:
         raise FileNotFoundError("Could not find golang installation")
-    version = parse_version(m.group(1))
-    if version < parse_version("1.13"):
-        raise OSError(f"Newer version of go required: must be >= 1.13, found {version}")
+
+    go_version_str = m.group(1)
+    go_version_parts = [int(x) for x in go_version_str.split(".")]
+    if go_version_parts[0] < 1 or (go_version_parts[0] == 1 and go_version_parts[1] < 13):
+        raise OSError(f"Newer version of go required: must be >= 1.13, found {go_version_str}")
 
     with tempfile.TemporaryDirectory() as tempdir:
         dest = os.path.join(tempdir, "libp2p-daemon.tar.gz")
@@ -122,6 +122,12 @@ def initialize_options(self):
         super().initialize_options()
         self.buildgo = False
 
+    def finalize_options(self):
+        super().finalize_options()
+        # Convert string to boolean if needed
+        if isinstance(self.buildgo, str):
+            self.buildgo = self.buildgo.lower() in ('true', '1', 'yes', 'on')
+
     def run(self):
         if self.buildgo:
             build_p2p_daemon()
@@ -140,66 +146,6 @@ def run(self):
         super().run()
 
 
-with open("requirements.txt") as requirements_file:
-    install_requires = list(map(str, parse_requirements(requirements_file)))
-
-# loading version from setup.py
-with codecs.open(os.path.join(here, "hivemind/__init__.py"), encoding="utf-8") as init_file:
-    version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", init_file.read(), re.MULTILINE)
-    version_string = version_match.group(1)
-
-extras = {}
-
-with open("requirements-dev.txt") as dev_requirements_file:
-    extras["dev"] = list(map(str, parse_requirements(dev_requirements_file)))
-
-with open("requirements-docs.txt") as docs_requirements_file:
-    extras["docs"] = list(map(str, parse_requirements(docs_requirements_file)))
-
-extras["bitsandbytes"] = ["bitsandbytes~=0.45.2"]
-
-extras["all"] = extras["dev"] + extras["docs"] + extras["bitsandbytes"]
-
 setup(
-    name="hivemind",
-    version=version_string,
     cmdclass={"build_py": BuildPy, "develop": Develop},
-    description="Decentralized deep learning in PyTorch",
-    long_description="Decentralized deep learning in PyTorch. Built to train models on thousands of volunteers "
-    "across the world.",
-    author="Learning@home & contributors",
-    author_email="[email protected]",
-    url="https://github.com/learning-at-home/hivemind",
-    packages=find_packages(exclude=["tests"]),
-    package_data={"hivemind": ["proto/*", "hivemind_cli/*"]},
-    include_package_data=True,
-    license="MIT",
-    setup_requires=["grpcio-tools"],
-    install_requires=install_requires,
-    extras_require=extras,
-    classifiers=[
-        "Development Status :: 4 - Beta",
-        "Intended Audience :: Developers",
-        "Intended Audience :: Science/Research",
-        "License :: OSI Approved :: MIT License",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-        "Programming Language :: Python :: 3.12",
-        "Topic :: Scientific/Engineering",
-        "Topic :: Scientific/Engineering :: Mathematics",
-        "Topic :: Scientific/Engineering :: Artificial Intelligence",
-        "Topic :: Software Development",
-        "Topic :: Software Development :: Libraries",
-        "Topic :: Software Development :: Libraries :: Python Modules",
-    ],
-    entry_points={
-        "console_scripts": [
-            "hivemind-dht = hivemind.hivemind_cli.run_dht:main",
-            "hivemind-server = hivemind.hivemind_cli.run_server:main",
-        ]
-    },
-    # What does your project relate to?
-    keywords="pytorch, deep learning, machine learning, gpu, distributed computing, volunteer computing, dht",
 )

tests/test_auth.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta
+from datetime import UTC, datetime, timedelta
 from typing import Optional
 
 import pytest
@@ -31,7 +31,7 @@ async def get_token(self) -> AccessToken:
         token = AccessToken(
             username=self._username,
             public_key=self.local_public_key.to_bytes(),
-            expiration_time=str(datetime.utcnow() + timedelta(minutes=1)),
+            expiration_time=str(datetime.now(UTC) + timedelta(minutes=1)),
         )
         token.signature = MockAuthorizer._authority_private_key.sign(self._token_to_bytes(token))
         return token
@@ -52,7 +52,7 @@ def is_token_valid(self, access_token: AccessToken) -> bool:
         if expiration_time.tzinfo is not None:
             logger.exception(f"Expected to have no timezone for expiration time: {access_token.expiration_time}")
             return False
-        if expiration_time < datetime.utcnow():
+        if expiration_time < datetime.now(UTC):
             logger.exception("Access token has expired")
             return False
 
@@ -62,7 +62,7 @@ def is_token_valid(self, access_token: AccessToken) -> bool:
 
     def does_token_need_refreshing(self, access_token: AccessToken) -> bool:
         expiration_time = datetime.fromisoformat(access_token.expiration_time)
-        return expiration_time < datetime.utcnow() + self._MAX_LATENCY
+        return expiration_time < datetime.now(UTC) + self._MAX_LATENCY
 
     @staticmethod
     def _token_to_bytes(access_token: AccessToken) -> bytes:

tests/test_utils/p2p_daemon.py

@@ -5,17 +5,16 @@
 import time
 import uuid
 from contextlib import asynccontextmanager, suppress
+from importlib.resources import files
 from typing import NamedTuple
 
-from pkg_resources import resource_filename
-
 from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client
 from hivemind.utils.multiaddr import Multiaddr, protocols
 
 from test_utils.networking import get_free_port
 
 TIMEOUT_DURATION = 5  # seconds
-P2PD_PATH = resource_filename("hivemind", "hivemind_cli/p2pd")
+P2PD_PATH = str(files("hivemind").joinpath("hivemind_cli/p2pd"))
 
 
 async def try_until_success(coro_func, timeout=TIMEOUT_DURATION):