diff --git a/.github/workflows/run-tests-on-modal.yml b/.github/workflows/run-tests-on-modal.yml index b96f2e8ff..9cf5fe333 100644 --- a/.github/workflows/run-tests-on-modal.yml +++ b/.github/workflows/run-tests-on-modal.yml @@ -39,7 +39,7 @@ jobs: - name: Install build dependencies run: | python -m pip install --upgrade pip - pip install modal==0.73.32 + pip install modal==1.1.0 - name: Run tests run: | @@ -74,7 +74,7 @@ jobs: - name: Install build dependencies run: | python -m pip install --upgrade pip - pip install modal==0.73.32 + pip install modal==1.1.0 - name: Measure and upload coverage run: | @@ -105,7 +105,7 @@ jobs: - name: Install build dependencies run: | python -m pip install --upgrade pip - pip install modal==0.73.32 + pip install modal==1.1.0 - name: Run p2pd tests run: | diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index d1c3fe3ff..704b36a40 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -68,7 +68,7 @@ jobs: pip install -r requirements-dev.txt - name: Build hivemind run: | - pip install . --global-option=build_py --global-option="--buildgo" --no-use-pep517 + pip install . --config-settings=build_py.buildgo=true - name: Test run: | cd tests @@ -98,7 +98,7 @@ jobs: pip install bitsandbytes==0.45.2 - name: Build hivemind run: | - pip install -e . --no-use-pep517 + pip install -e . - name: Test run: | export HIVEMIND_MEMORY_SHARING_STRATEGY=file_descriptor diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index a6c9484c9..3840efc3b 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -8,7 +8,7 @@ from contextlib import closing, suppress from dataclasses import dataclass from datetime import datetime -from importlib.resources import path +from importlib.resources import files from typing import Any, AsyncIterator, Awaitable, Callable, List, Optional, Sequence, Tuple, Type, TypeVar, Union from google.protobuf.message import Message @@ -158,8 +158,7 @@ async def create( ) self = cls() - with path(cli, P2PD_FILENAME) as p: - p2pd_path = p + p2pd_path = files(cli) / P2PD_FILENAME socket_uid = secrets.token_urlsafe(8) self._daemon_listen_maddr = Multiaddr(cls._UNIX_SOCKET_PREFIX + f"p2pd-{socket_uid}.sock") diff --git a/modal_ci.py b/modal_ci.py index 43b977ee7..4d2c3c7a1 100644 --- a/modal_ci.py +++ b/modal_ci.py @@ -15,13 +15,6 @@ "cd bitsandbytes && cmake -DCOMPUTE_BACKEND=cpu -S . && make && pip --no-cache install . ", ] ) - .add_local_dir("hivemind", remote_path="/root/hivemind/hivemind") - .add_local_file("requirements.txt", remote_path="/root/hivemind/requirements.txt") - .add_local_file("requirements-dev.txt", remote_path="/root/hivemind/requirements-dev.txt") - .add_local_file("requirements-docs.txt", remote_path="/root/hivemind/requirements-docs.txt") - .add_local_file("setup.py", remote_path="/root/hivemind/setup.py") - .add_local_file("pyproject.toml", remote_path="/root/hivemind/pyproject.toml") - .add_local_dir("tests", remote_path="/root/hivemind/tests") ) # Create an image with golang and other system dependencies @@ -41,13 +34,6 @@ "cd bitsandbytes && cmake -DCOMPUTE_BACKEND=cpu -S . && make && pip --no-cache install . ", ] ) - .add_local_dir("hivemind", remote_path="/root/hivemind/hivemind") - .add_local_file("requirements.txt", remote_path="/root/hivemind/requirements.txt") - .add_local_file("requirements-dev.txt", remote_path="/root/hivemind/requirements-dev.txt") - .add_local_file("requirements-docs.txt", remote_path="/root/hivemind/requirements-docs.txt") - .add_local_file("setup.py", remote_path="/root/hivemind/setup.py") - .add_local_file("pyproject.toml", remote_path="/root/hivemind/pyproject.toml") - .add_local_dir("tests", remote_path="/root/hivemind/tests") ) @@ -72,12 +58,10 @@ def setup_environment(*, build_p2pd=False): "install", "--no-cache-dir", ".", - "--global-option=build_py", - "--global-option=--buildgo", - "--no-use-pep517", + "--config-settings=build_py.buildgo=true", ] else: - install_cmd = ["pip", "install", "-e", ".", "--no-use-pep517"] + install_cmd = ["pip", "install", "-e", "."] subprocess.run(install_cmd, check=True) @@ -87,7 +71,12 @@ def setup_environment(*, build_p2pd=False): return environment -@app.function(image=image, timeout=600, cpu=8, memory=8192) +@app.function( + image=image.add_local_dir(".", remote_path="/root/hivemind"), + timeout=1200, + cpu=8, + memory=8192, +) def run_tests(): environment = setup_environment(build_p2pd=False) @@ -106,7 +95,13 @@ def run_tests(): ) -@app.function(image=image, timeout=900, cpu=8, memory=8192, secrets=[codecov_secret]) +@app.function( + image=image.add_local_dir(".", remote_path="/root/hivemind"), + timeout=900, + cpu=8, + memory=8192, + secrets=[codecov_secret], +) def run_codecov(): environment = setup_environment(build_p2pd=False) @@ -147,7 +142,12 @@ def run_codecov(): ) -@app.function(image=image_with_golang, timeout=600, cpu=1, memory=4096) +@app.function( + image=image_with_golang.add_local_dir(".", remote_path="/root/hivemind"), + timeout=600, + cpu=1, + memory=4096, +) def build_and_test_p2pd(): environment = setup_environment(build_p2pd=True) diff --git a/pyproject.toml b/pyproject.toml index 1bef4011b..6d521f16f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,58 @@ +[build-system] +requires = ["setuptools>=64", "grpcio-tools"] +build-backend = "setuptools.build_meta" + +[project] +name = "hivemind" +description = "Decentralized deep learning in PyTorch" +readme = "README.md" +authors = [{name = "Learning@home & contributors", email = "hivemind-team@hotmail.com"}] +license = "MIT" +requires-python = ">=3.9" +keywords = ["pytorch", "deep learning", "machine learning", "gpu", "distributed computing", "volunteer computing", "dht"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dynamic = ["version", "dependencies"] + +[project.optional-dependencies] +bitsandbytes = ["bitsandbytes~=0.45.2"] + +[project.scripts] +hivemind-dht = "hivemind.hivemind_cli.run_dht:main" +hivemind-server = "hivemind.hivemind_cli.run_server:main" + +[project.urls] +Homepage = "https://github.com/learning-at-home/hivemind" +Repository = "https://github.com/learning-at-home/hivemind" + +[tool.setuptools] +packages = {find = {}} +include-package-data = true + +[tool.setuptools.dynamic] +version = {attr = "hivemind.__version__"} +dependencies = {file = ["requirements.txt"]} +optional-dependencies.dev = {file = ["requirements-dev.txt"]} +optional-dependencies.docs = {file = ["requirements-docs.txt"]} +optional-dependencies.all = {file = ["requirements-all.txt"]} + +[tool.setuptools.package-data] +hivemind = ["proto/*", "hivemind_cli/*"] + [tool.coverage.run] concurrency = ["thread", "multiprocessing"] omit = ["hivemind/proto/*"] diff --git a/requirements.txt b/requirements.txt index 78aab21bd..84117fbcd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ msgpack>=0.5.6 sortedcontainers uvloop>=0.14.0 grpcio-tools>=1.33.2 -protobuf>=5.29.0 +protobuf>=6.31.0 configargparse>=1.2.3 py-multihash>=0.2.3 cryptography>=3.4.6 diff --git a/setup.py b/setup.py index e1f90b274..eae904911 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -import codecs import glob import hashlib import os @@ -9,10 +8,10 @@ import tempfile import urllib.request -from pkg_resources import parse_requirements, parse_version -from setuptools import find_packages, setup +from setuptools import setup from setuptools.command.build_py import build_py from setuptools.command.develop import develop +from setuptools.command.egg_info import egg_info P2PD_VERSION = "v0.5.0.hivemind1" @@ -64,9 +63,11 @@ def build_p2p_daemon(): if m is None: raise FileNotFoundError("Could not find golang installation") - version = parse_version(m.group(1)) - if version < parse_version("1.13"): - raise OSError(f"Newer version of go required: must be >= 1.13, found {version}") + + go_version_str = m.group(1) + go_version_parts = [int(x) for x in go_version_str.split(".")] + if go_version_parts[0] < 1 or (go_version_parts[0] == 1 and go_version_parts[1] < 13): + raise OSError(f"Newer version of go required: must be >= 1.13, found {go_version_str}") with tempfile.TemporaryDirectory() as tempdir: dest = os.path.join(tempdir, "libp2p-daemon.tar.gz") @@ -122,6 +123,12 @@ def initialize_options(self): super().initialize_options() self.buildgo = False + def finalize_options(self): + super().finalize_options() + # Convert string to boolean if needed + if isinstance(self.buildgo, str): + self.buildgo = self.buildgo.lower() in ("true", "1", "yes", "on") + def run(self): if self.buildgo: build_p2p_daemon() @@ -140,66 +147,13 @@ def run(self): super().run() -with open("requirements.txt") as requirements_file: - install_requires = list(map(str, parse_requirements(requirements_file))) - -# loading version from setup.py -with codecs.open(os.path.join(here, "hivemind/__init__.py"), encoding="utf-8") as init_file: - version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", init_file.read(), re.MULTILINE) - version_string = version_match.group(1) - -extras = {} - -with open("requirements-dev.txt") as dev_requirements_file: - extras["dev"] = list(map(str, parse_requirements(dev_requirements_file))) - -with open("requirements-docs.txt") as docs_requirements_file: - extras["docs"] = list(map(str, parse_requirements(docs_requirements_file))) - -extras["bitsandbytes"] = ["bitsandbytes~=0.45.2"] +class EggInfo(egg_info): + def run(self): + # Generate protobuf files early in the build process + proto_compile(os.path.join(here, "hivemind", "proto")) + super().run() -extras["all"] = extras["dev"] + extras["docs"] + extras["bitsandbytes"] setup( - name="hivemind", - version=version_string, - cmdclass={"build_py": BuildPy, "develop": Develop}, - description="Decentralized deep learning in PyTorch", - long_description="Decentralized deep learning in PyTorch. Built to train models on thousands of volunteers " - "across the world.", - author="Learning@home & contributors", - author_email="hivemind-team@hotmail.com", - url="https://github.com/learning-at-home/hivemind", - packages=find_packages(exclude=["tests"]), - package_data={"hivemind": ["proto/*", "hivemind_cli/*"]}, - include_package_data=True, - license="MIT", - setup_requires=["grpcio-tools"], - install_requires=install_requires, - extras_require=extras, - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Mathematics", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Software Development", - "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - entry_points={ - "console_scripts": [ - "hivemind-dht = hivemind.hivemind_cli.run_dht:main", - "hivemind-server = hivemind.hivemind_cli.run_server:main", - ] - }, - # What does your project relate to? - keywords="pytorch, deep learning, machine learning, gpu, distributed computing, volunteer computing, dht", + cmdclass={"build_py": BuildPy, "develop": Develop, "egg_info": EggInfo}, ) diff --git a/tests/test_auth.py b/tests/test_auth.py index 75b647995..c402ab2b8 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from typing import Optional import pytest @@ -31,7 +31,7 @@ async def get_token(self) -> AccessToken: token = AccessToken( username=self._username, public_key=self.local_public_key.to_bytes(), - expiration_time=str(datetime.utcnow() + timedelta(minutes=1)), + expiration_time=str(datetime.now(UTC) + timedelta(minutes=1)), ) token.signature = MockAuthorizer._authority_private_key.sign(self._token_to_bytes(token)) return token @@ -52,7 +52,7 @@ def is_token_valid(self, access_token: AccessToken) -> bool: if expiration_time.tzinfo is not None: logger.exception(f"Expected to have no timezone for expiration time: {access_token.expiration_time}") return False - if expiration_time < datetime.utcnow(): + if expiration_time < datetime.now(UTC): logger.exception("Access token has expired") return False @@ -62,7 +62,7 @@ def is_token_valid(self, access_token: AccessToken) -> bool: def does_token_need_refreshing(self, access_token: AccessToken) -> bool: expiration_time = datetime.fromisoformat(access_token.expiration_time) - return expiration_time < datetime.utcnow() + self._MAX_LATENCY + return expiration_time < datetime.now(UTC) + self._MAX_LATENCY @staticmethod def _token_to_bytes(access_token: AccessToken) -> bytes: diff --git a/tests/test_utils/p2p_daemon.py b/tests/test_utils/p2p_daemon.py index db208cdd5..ab87d96e3 100644 --- a/tests/test_utils/p2p_daemon.py +++ b/tests/test_utils/p2p_daemon.py @@ -5,17 +5,16 @@ import time import uuid from contextlib import asynccontextmanager, suppress +from importlib.resources import files from typing import NamedTuple -from pkg_resources import resource_filename - from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client from hivemind.utils.multiaddr import Multiaddr, protocols from test_utils.networking import get_free_port TIMEOUT_DURATION = 5 # seconds -P2PD_PATH = resource_filename("hivemind", "hivemind_cli/p2pd") +P2PD_PATH = str(files("hivemind").joinpath("hivemind_cli/p2pd")) async def try_until_success(coro_func, timeout=TIMEOUT_DURATION):