diff --git a/Pipfile b/Pipfile index 495559a..2eaa8a2 100644 --- a/Pipfile +++ b/Pipfile @@ -16,6 +16,8 @@ colorama = "*" verboselogs = "*" crcmod = "*" tinyaes = "*" +requests = "*" +urllib3 = "*" pywin32 = {version = "*", sys_platform = "== 'win32'"} pefile = {version = "*", sys_platform = "== 'win32'"} "pywin32-ctypes" = {version = "*", sys_platform = "== 'win32'"} diff --git a/pbpy/pbhttp.py b/pbpy/pbhttp.py new file mode 100644 index 0000000..c98deec --- /dev/null +++ b/pbpy/pbhttp.py @@ -0,0 +1,25 @@ +import requests +from requests.adapters import HTTPAdapter +from urllib3.util import Retry + +def setup(): + retry_strategy = Retry( + total=5, # Maximum number of retries + status_forcelist=[429, 500, 502, 503, 504], # Retry on these status codes + backoff_factor=1 # Wait 1 sec before retrying, then increase by 1 sec each retry + ) + + adapter = HTTPAdapter(max_retries=retry_strategy) + http = requests.Session() + http.mount("https://", adapter) + +def download(url: str, downloaded_file: str): + setup() + response = http.get(url, stream=True) + + try: + with open(downloaded_file, 'wb') as file: + for chunk in response.iter_content(chunk_size=10 * 1024): + file.write(chunk) + except OSError as e: # For file I/O errors (e.g., disk full, permission denied) + pblog.error(f"File I/O error: {e}") diff --git a/pbpy/pbsteamcmd.py b/pbpy/pbsteamcmd.py index 5e04882..f312c8f 100644 --- a/pbpy/pbsteamcmd.py +++ b/pbpy/pbsteamcmd.py @@ -3,14 +3,13 @@ import shutil import time import traceback -import urllib.request from pathlib import Path import gevent import steam.protobufs.steammessages_partnerapps_pb2 # don't remove from steam.client import SteamClient -from pbpy import pbconfig, pbinfo, pblog, pbtools +from pbpy import pbconfig, pbhttp, pbinfo, pblog, pbtools drm_upload_regex = re.compile( r"https:\/\/partnerupload\.steampowered\.com\/upload\/(\d+)" @@ -212,10 +211,7 @@ def handle_drm_file(): ) url = resp.body.download_url if url: - with urllib.request.urlopen(url) as response, open( - str(drm_output), "wb" - ) as out_file: - shutil.copyfileobj(response, out_file) + pbhttp.download(url, str(drm_output)) steamclient.close() if not drm_output.exists() and drm_download_failed: diff --git a/pbpy/pbunreal.py b/pbpy/pbunreal.py index 90fcb96..4879af5 100644 --- a/pbpy/pbunreal.py +++ b/pbpy/pbunreal.py @@ -7,14 +7,13 @@ import re import shutil import time -import urllib.request import zipfile from functools import lru_cache from pathlib import Path from shutil import disk_usage, move, rmtree from urllib.parse import urlparse -from pbpy import pbconfig, pbgit, pbinfo, pblog, pbtools, pbuac +from pbpy import pbconfig, pbgit, pbhttp, pbinfo, pblog, pbtools, pbuac # Those variable values are not likely to be changed in the future, it's safe to keep them hardcoded uev_prefix = "uev:" @@ -1299,10 +1298,7 @@ def inspect_source(all=False): if not zip_path.exists(): pblog.info(f"Downloading Resharper {version}") url = f"https://download-cdn.jetbrains.com/resharper/dotUltimate.{version}/{zip_name}" - with urllib.request.urlopen(url) as response, open( - str(zip_path), "wb" - ) as out_file: - shutil.copyfileobj(response, out_file) + pbhttp.download(url, str(zip_path)) resharper_dir = saved_dir / Path("ResharperCLI") pblog.info(f"Unpacking Resharper {version}") shutil.unpack_archive(str(zip_path), str(resharper_dir)) @@ -1499,3 +1495,4 @@ def build_installed_build(): if not set_engine_version(version): pbtools.error_state("Error while updating engine version in .uproject file") pblog.info(f"Successfully changed engine version as {str(version)}") + diff --git a/pbsync/__main__.py b/pbsync/__main__.py index 26a89bc..9c5799d 100644 --- a/pbsync/__main__.py +++ b/pbsync/__main__.py @@ -2,7 +2,10 @@ import multiprocessing import os import os.path +import platform import sys +import shutil +import subprocess import threading import time import webbrowser @@ -15,6 +18,7 @@ pbdispatch, pbgh, pbgit, + pbhttp, pblog, pbpy_version, pbsteamcmd, @@ -31,6 +35,52 @@ default_config_name = "PBSync.xml" +def check_gh_cli(): + try: + result = subprocess.run(["gh", "--version"], capture_output=True, text=True, check=True) + pblog.info(result.stdout.strip()) + return True + except FileNotFoundError: + return False + +def install_gh_cli(): + pblog.info("GitHub CLI not found. Installing...") + gh_ver="2.63.0" + # Download GitHub CLI + match sys.platform: + case "win32": + url = f"https://github.com/cli/cli/releases/download/v{gh_ver}/gh_{gh_ver}_windows_amd64.msi" + downloaded_file = 'gh.msi' + case "darwin": + url = f"https://github.com/cli/cli/releases/download/v{gh_ver}/gh_{gh_ver}_macOS_universal.pkg" + downloaded_file = 'gh.pkg' + case _: + pblog.error("Your operation system is not supported") + return None + + pbhttp.download(url) + + # Install GitHub CLI + try: + match sys.platform: + case "win32": subprocess.run(['msiexec', '/i', downloaded_file, '/passive'], check=True) + case "darwin": subprocess.run(['sudo', 'installer', '-pkg', downloaded_file, '-target', '/'], check=True) + except subprocess.CalledProcessError as e: + pblog.error(f"Command failed with return code {e.returncode}") + + pblog.info("GitHub CLI installed.") + result = subprocess.run(['gh', '--version'], capture_output=True, text=True, check=True) + pblog.info(result.stdout.strip()) + # Delete installation file + os.remove(downloaded_file) + + try: + subprocess.run(['gh', 'auth', 'login'], check=True) + except: + pass + +if not check_gh_cli(): + install_gh_cli() def config_handler(config_var, config_parser_func): if not pbconfig.generate_config(config_var, config_parser_func): @@ -43,7 +93,6 @@ def config_handler(config_var, config_parser_func): def sync_handler(sync_val: str, repository_val=None): - sync_val = sync_val.lower() if sync_val == "all" or sync_val == "force" or sync_val == "partial": @@ -55,76 +104,30 @@ def sync_handler(sync_val: str, repository_val=None): detected_git_version = pbgit.get_git_version() supported_git_version = pbconfig.get("supported_git_version") - needs_git_update = False if detected_git_version == supported_git_version: pblog.info(f"Current Git version: {detected_git_version}") else: - pblog.warning("Git is not updated to the supported version in your system") - pblog.warning( - f"Supported Git Version: {pbconfig.get('supported_git_version')}" - ) - pblog.warning(f"Current Git Version: {detected_git_version}") - needs_git_update = True - repo = "microsoft/git" - version = f"v{supported_git_version}" - if ( - "vfs" in detected_git_version - and sys.platform == "win32" - or sys.platform == "darwin" - ): - pblog.info("Auto-updating Git...") - if sys.platform == "win32": - directory = "Saved/PBSyncDownloads" - download = f"Git-{supported_git_version}-64-bit.exe" - if ( - pbgh.download_release_file( - version, - download, - directory=directory, - repo=f"https://github.com/{repo}", - ) - != 0 - ): - pblog.error( - "Git auto-update failed, please download and install manually." - ) - webbrowser.open( - f"https://github.com/{repo}/releases/download/{version}/{download}" - ) - else: - download_path = f"Saved\\PBSyncDownloads\\{download}" - proc = pbtools.run([download_path]) - if proc.returncode: - pblog.error("Git auto-update failed. Please try manually:") - webbrowser.open( - f"https://github.com/{repo}/releases/download/{version}/{download}" - ) - else: - needs_git_update = False - # reconfigure credential manager to make sure we have the proper path - pbtools.run([*pbgit.get_gcm_executable(), "configure"]) - os.remove(download_path) - else: - proc = pbtools.run( - [pbgit.get_git_executable(), "update-microsoft-git"] - ) - # if non-zero, error out - if proc.returncode: - pblog.error( - "Git auto-update failed, please download and install manually." - ) - else: - needs_git_update = False - input( - "Launching Git update, please press enter when done installing. " - ) - if needs_git_update: - pblog.error( - f"Please install the supported Git version from https://github.com/{repo}/releases/tag/{version}" - ) - pblog.error( - f"Visit {pbconfig.get('git_instructions')} for installation instructions" - ) + try: + match sys.platform: + case "win32": subprocess.run(['gh', 'release', 'download', supported_git_version, '-p', 'Git*.exe', '-R', 'microsoft/git'], check=True) + case "darwin": subprocess.run(['gh', 'release', 'download', supported_git_version, '-p', 'git*.pkg', '-R', 'microsoft/git'], check=True) + except subprocess.CalledProcessError as e: + pblog.error(f"Command failed with return code {e.returncode}") + + git_installer = [file for file in os.listdir() if file.startswith("Git") or file.startswith("git")][0] + + # Install Git + try: + match sys.platform: + case "win32": subprocess.run([git_installer], '/VERYSILENT', check=True) + case "darwin": subprocess.run(['sudo', 'installer', '-pkg', git_installer, '-target', '/'], check=True) + pblog.info(f'Installing Git {supported_git_version}...') + except subprocess.CalledProcessError as e: + pblog.error(f"Command failed with return code {e.returncode}") + + pblog.info(f'Git {supported_git_version} installed successfully.') + # Delete installation file + os.remove(git_installer) if ( os.name == "nt" @@ -195,55 +198,38 @@ def sync_handler(sync_val: str, repository_val=None): if detected_lfs_version == supported_lfs_version: pblog.info(f"Current Git LFS version: {detected_lfs_version}") else: - pblog.warning( - "Git LFS is not updated to the supported version in your system" - ) - pblog.warning(f"Supported Git LFS Version: {supported_lfs_version}") - pblog.warning(f"Current Git LFS Version: {detected_lfs_version}") - version = f"v{supported_lfs_version}" - needs_git_update = True - repo = "git-lfs/git-lfs" - if os.name == "nt": - pblog.info("Auto-updating Git LFS...") - directory = "Saved/PBSyncDownloads" - download = f"git-lfs-windows-{version}.exe" - result = pbgh.download_release_file( - version, - download, - directory=directory, - repo=f"https://github.com/{repo}", - ) - if result != 0: - pblog.error( - "Git LFS auto-update failed, please download and install manually." - ) - webbrowser.open( - f"https://github.com/{repo}/releases/download/{version}/{download}" - ) - else: - download_path = f"Saved\\PBSyncDownloads\\{download}" - proc = pbtools.run([download_path]) - if proc.returncode: - pblog.error( - "Git LFS auto-update failed, please download and install manually." - ) - webbrowser.open( - f"https://github.com/{repo}/releases/download/{version}/{download}" - ) - else: - # install LFS for the user - current_drive = Path().resolve() - current_drive = current_drive.drive or current_drive.root - pbtools.run( - [pbgit.get_lfs_executable(), "install"], cwd=current_drive - ) - needs_git_update = False - os.remove(download_path) - - if needs_git_update: - pblog.error( - f"Please install the supported Git LFS version from https://github.com/{repo}/releases/tag/{version}" - ) + # Download Git LFS + try: + match sys.platform: + case "win32": subprocess.run(['gh', 'release', 'download', supported_lfs_version, '-p', '*.exe', '-R', 'git-lfs/git-lfs'], check=True) + case "darwin": + if platform.machine() == "AMD64": + subprocess.run(['gh', 'release', 'download', supported_lfs_version, '-p', '*darwin-amd64*', '-R', 'git-lfs/git-lfs'], check=True) + else: + subprocess.run(['gh', 'release', 'download', supported_lfs_version, '-p', '*darwin-arm64*', '-R', 'git-lfs/git-lfs'], check=True) + except subprocess.CalledProcessError as e: + pblog.error(f"Command failed with return code {e.returncode}") + + # Find the downloaded .exe (Windows) or folder (Mac OS) + lfs_installer = [file for file in os.listdir() if file.startswith("git-lfs")][0] + + try: + match sys.platform: + case "win32": subprocess.run([lfs_installer], '/VERYSILENT', check=True) + case "darwin": + subprocess.run(['unzip', lfs_installer], check=True) + subprocess.run(['sudo', f"./{lfs_installer}/install.sh"], check=True) + pblog.info(f"Installing Git LFS {supported_lfs_version}...") + except subprocess.CalledProcessError as e: + pblog.error(f"Command failed with return code {e.returncode}") + + pblog.info(f'Git LFS {supported_lfs_version} installed successfully.') + + # Delete installation file/folder + if path.isfile(lfs_installer): + os.remove(lfs_installer) + elif path.isdir(lfs_installer): + shutil.rmtree(lfs_installer) # check if Git LFS was installed to a different path if os.name == "nt" and pbgit.get_lfs_executable() == "git-lfs": @@ -978,3 +964,4 @@ def pbsync_config_parser_func(root): # Working directory fix for scripts calling PBSync from Scripts folder os.chdir("..") main(sys.argv[1:]) +