From d23a17a8bd31a8b407b6e8458801e7b387dcd49e Mon Sep 17 00:00:00 2001 From: Johannes <24914225+Johannes11833@users.noreply.github.com> Date: Mon, 7 Jul 2025 21:06:08 +0200 Subject: [PATCH 1/9] fix: empty error message during failed transfer operation --- rclone_python/utils.py | 11 ++++++++--- tests/test_copy.py | 12 ++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/rclone_python/utils.py b/rclone_python/utils.py index 15d5d3c..6f9df89 100644 --- a/rclone_python/utils.py +++ b/rclone_python/utils.py @@ -17,7 +17,8 @@ class RcloneException(ChildProcessError): - def __init__(self, description, error_msg): + + def __init__(self, description: str, error_msg: str): self.description = description self.error_msg = error_msg super().__init__(f"{description}. Error message: \n{error_msg}") @@ -189,11 +190,15 @@ def extract_rclone_progress(line: str) -> Tuple[bool, Union[Dict[str, Any], None Returns: Tuple[bool, Union[Dict[str, Any], None]]: The retrieved update Dictionary or error message. """ + stats = None try: log_item: Dict = json.loads(line) - if log_item.get("level", None) == "error": - return False, log_item + level = log_item.get("level", None) + if level != "info": + if level in ("error", "critical"): + # return error message + return False, log_item else: # stats updates use the "info" level stats = log_item.get("stats", None) diff --git a/tests/test_copy.py b/tests/test_copy.py index 842a9fe..b64e1e8 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -198,3 +198,15 @@ def test_progress_listener(tmp_remote_folder, tmp_local_folder, show_progress, m assert len(file_2_progress) > 0 assert file_2_progress[0] == pytest.approx(0, abs=0.1) assert file_2_progress[-1] == pytest.approx(1) + + +def test_rclone_transfer_operation_error_message(default_test_setup, tmp_local_folder): + faulty_remote_name = default_test_setup.remote_name + "s:" + + try: + rclone.copy(faulty_remote_name, tmp_local_folder) + assert False + except RcloneException as exception: + # check that a rclone exception message is set + assert len(exception.description) > 0 + assert len(exception.error_msg) > 0 From 63795d765693651249ac5a847b8e5f7968235466 Mon Sep 17 00:00:00 2001 From: Johannes <24914225+Johannes11833@users.noreply.github.com> Date: Mon, 7 Jul 2025 21:21:32 +0200 Subject: [PATCH 2/9] fix test_extract_rclone_progress_normal_update failing --- rclone_python/utils.py | 1 - tests/test_copy.py | 2 +- tests/test_utils.py | 4 +++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/rclone_python/utils.py b/rclone_python/utils.py index 6f9df89..d4a8413 100644 --- a/rclone_python/utils.py +++ b/rclone_python/utils.py @@ -17,7 +17,6 @@ class RcloneException(ChildProcessError): - def __init__(self, description: str, error_msg: str): self.description = description self.error_msg = error_msg diff --git a/tests/test_copy.py b/tests/test_copy.py index b64e1e8..817bc04 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -203,7 +203,7 @@ def test_progress_listener(tmp_remote_folder, tmp_local_folder, show_progress, m def test_rclone_transfer_operation_error_message(default_test_setup, tmp_local_folder): faulty_remote_name = default_test_setup.remote_name + "s:" - try: + try: rclone.copy(faulty_remote_name, tmp_local_folder) assert False except RcloneException as exception: diff --git a/tests/test_utils.py b/tests/test_utils.py index 95395b3..5dbaccf 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -46,7 +46,9 @@ def test_extract_rclone_progress_normal_update(valid_rclone_stats_update): # valid input where the total file size is already known input = valid_rclone_stats_update - valid, output = extract_rclone_progress(json.dumps({"stats": input})) + valid, output = extract_rclone_progress( + json.dumps({"stats": input, "level": "info"}) + ) assert valid # validate task summary assert output["total"] == pytest.approx(input["totalBytes"]) From 775d6e597599d0e73d74448ea4e73bb9dc1d078b Mon Sep 17 00:00:00 2001 From: Johannes <24914225+Johannes11833@users.noreply.github.com> Date: Sat, 12 Jul 2025 15:04:43 +0200 Subject: [PATCH 3/9] add scripts to launch test server --- launch_test_server.bat | 22 ++++++++++++++++++++++ launch_test_server.sh | 24 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 launch_test_server.bat create mode 100644 launch_test_server.sh diff --git a/launch_test_server.bat b/launch_test_server.bat new file mode 100644 index 0000000..85ff8ce --- /dev/null +++ b/launch_test_server.bat @@ -0,0 +1,22 @@ +@echo off +echo Starting MinIO server with Docker... + +docker run --rm -d --name minio ^ + -p 9000:9000 ^ + -p 9001:9001 ^ + -e MINIO_ROOT_USER=minioadmin ^ + -e MINIO_ROOT_PASSWORD=minioadmin ^ + quay.io/minio/minio server /data --console-address ":9001" + +echo MinIO should now be running on: +echo - S3 Endpoint: http://localhost:9000 +echo - Console UI : http://localhost:9001 + + +rclone config create test_server_s3 s3 ^ + provider Minio ^ + env_auth false ^ + access_key_id minioadmin ^ + secret_access_key minioadmin ^ + endpoint http://localhost:9000 ^ + region us-east-1 \ No newline at end of file diff --git a/launch_test_server.sh b/launch_test_server.sh new file mode 100644 index 0000000..30c1d80 --- /dev/null +++ b/launch_test_server.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +echo "Starting MinIO server with Docker..." + +docker run --rm -d --name minio \ + -p 9000:9000 \ + -p 9001:9001 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + quay.io/minio/minio server /data --console-address ":9001" + +echo "MinIO should now be running on:" +echo "- S3 Endpoint: http://localhost:9000" +echo "- Console UI : http://localhost:9001" + +echo "Configuring rclone..." + +rclone config create test_server_s3 s3 \ + provider Minio \ + env_auth false \ + access_key_id minioadmin \ + secret_access_key minioadmin \ + endpoint http://localhost:9000 \ + region us-east-1 \ No newline at end of file From a157d223d92018789da172d3cef4eba1b3d18039 Mon Sep 17 00:00:00 2001 From: Johannes <24914225+Johannes11833@users.noreply.github.com> Date: Sat, 12 Jul 2025 15:05:05 +0200 Subject: [PATCH 4/9] adjust tests to work with s3 server --- rclone_python/rclone.py | 4 +--- tests/conftest.py | 4 ++-- tests/test_about.py | 10 +++------- tests/test_copy.py | 9 ++++----- tests/test_mkdir.py | 6 +++--- 5 files changed, 13 insertions(+), 20 deletions(-) diff --git a/rclone_python/rclone.py b/rclone_python/rclone.py index b0242d9..853f0c0 100644 --- a/rclone_python/rclone.py +++ b/rclone_python/rclone.py @@ -50,12 +50,10 @@ def is_installed() -> bool: def about(remote_name: str) -> Dict: """ Executes the rclone about command and returns the retrieved json as a dictionary. + All sizes are in number of bytes. :param remote_name: The name of the remote to examine. :return: Dictionary with remote properties. """ - if not remote_name.endswith(":"): - # if the remote name missed the colon manually add it. - remote_name += ":" stdout, _ = utils.run_rclone_cmd(f'about "{remote_name}" --json') diff --git a/tests/conftest.py b/tests/conftest.py index 5473582..1bd1bd4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,8 +22,8 @@ def __init__( @pytest.fixture(scope="session") def default_test_setup(): return TestSetup( - remote_name="box", - remote_test_dir_rel="test_dir", + remote_name="test_server_s3", + remote_test_dir_rel="testdir", local_test_txt_file="tests/data/lorem.txt", ) diff --git a/tests/test_about.py b/tests/test_about.py index 08a6c89..1235ea0 100644 --- a/tests/test_about.py +++ b/tests/test_about.py @@ -1,15 +1,11 @@ from rclone_python import rclone -def test_about(default_test_setup): +def test_about(tmp_local_folder): + # NOTE: rclone about does not work on s3 --> do locally instead expected_fields = set(["total", "used", "free"]) - output = rclone.about(default_test_setup.remote_name) - assert expected_fields.issubset(set(output.keys())) - for field in expected_fields: - assert isinstance(output[field], (int, float)) - - output = rclone.about(default_test_setup.remote_name + ":") + output = rclone.about(tmp_local_folder) assert expected_fields.issubset(set(output.keys())) for field in expected_fields: assert isinstance(output[field], (int, float)) diff --git a/tests/test_copy.py b/tests/test_copy.py index 817bc04..5e7d5a4 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -96,12 +96,9 @@ def test_copy(default_test_setup, tmp_remote_folder, tmp_local_folder, command): ) assert (download_path / tmp_local_file.name).is_file() - # download with errors (no such directory) + # upload with errors (no such directory) with pytest.raises(RcloneException, match="directory not found") as e_info: - command( - tmp_remote_folder + "_1", - download_path, - ) + command(str(tmp_local_folder) + "_foo", tmp_remote_folder) def test_sync(default_test_setup, tmp_remote_folder, tmp_local_folder): @@ -168,6 +165,8 @@ def test_progress_listener(tmp_remote_folder, tmp_local_folder, show_progress, m tmp_remote_folder, listener=recorder.update, show_progress=show_progress, + # limit the bandwidth to see progress updates + args=["--bwlimit 10M"], ) # check that all fields are there diff --git a/tests/test_mkdir.py b/tests/test_mkdir.py index fb0aac9..c5d0f89 100644 --- a/tests/test_mkdir.py +++ b/tests/test_mkdir.py @@ -1,11 +1,11 @@ from rclone_python import rclone -def test_mkdir(tmp_remote_folder): +def test_mkdir(tmp_local_folder): folder_name = "dir1 v2" - rclone.mkdir(f"{tmp_remote_folder}/{folder_name}") + rclone.mkdir(f"{tmp_local_folder}/{folder_name}") - output = rclone.ls(tmp_remote_folder) + output = rclone.ls(tmp_local_folder) assert len(output) == 1 assert output[0]["Path"] == folder_name From 5b568e4c3138de2813fb3de928b3f97909f40c76 Mon Sep 17 00:00:00 2001 From: Johannes Gundlach <24914225+Johannes11833@users.noreply.github.com> Date: Sat, 12 Jul 2025 15:36:02 +0200 Subject: [PATCH 5/9] make launch_test_server.sh executable --- launch_test_server.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 launch_test_server.sh diff --git a/launch_test_server.sh b/launch_test_server.sh old mode 100644 new mode 100755 From e2bb69d9b29c9dd3e0094a08cb322f4b7b073ba5 Mon Sep 17 00:00:00 2001 From: Johannes <24914225+Johannes11833@users.noreply.github.com> Date: Sat, 12 Jul 2025 15:37:30 +0200 Subject: [PATCH 6/9] update pytest GitHub action --- .github/workflows/run_tests.yml | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index a8a890d..93d3793 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -11,7 +11,6 @@ jobs: build: runs-on: ubuntu-latest env: - rclone_config: ${{ secrets.RCLONE_CONFIG }} python_version: "3.10" timezone: "Europe/Berlin" steps: @@ -22,11 +21,9 @@ jobs: python-version: ${{ env.python_version }} - name: Install the rclone software run: sudo -v ; curl https://rclone.org/install.sh | sudo bash - - name: Setup rclone config file + - name: Setup s3 test-server and rclone run: | - mkdir -p ~/.config/rclone - echo "$rclone_config" > ~/.config/rclone/rclone.conf - rclone listremotes + ./launch_test_server.sh - name: Set timezone run: | sudo timedatectl set-timezone ${{ env.timezone }} @@ -38,11 +35,4 @@ jobs: pip install pytest - name: Run test suite run: | - pytest -v - - name: Update rclone config file secret - run: cat ~/.config/rclone/rclone.conf | gh secret set RCLONE_CONFIG - env: - rclone_config_secret_name: RCLONE_CONFIG - GITHUB_TOKEN: ${{ secrets.GH_PAT }} - GH_REPO: ${{ github.repository }} - + pytest -v \ No newline at end of file From d97bda81c10307fa86500bc35b3505bed7d3bc19 Mon Sep 17 00:00:00 2001 From: Johannes <24914225+Johannes11833@users.noreply.github.com> Date: Sat, 12 Jul 2025 15:44:56 +0200 Subject: [PATCH 7/9] decrease bandwidth in test_progress_listener test --- tests/test_copy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_copy.py b/tests/test_copy.py index 5e7d5a4..6d212da 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -166,7 +166,7 @@ def test_progress_listener(tmp_remote_folder, tmp_local_folder, show_progress, m listener=recorder.update, show_progress=show_progress, # limit the bandwidth to see progress updates - args=["--bwlimit 10M"], + args=["--bwlimit 5M"], ) # check that all fields are there From cd6d6de0868d3a72ec79b4c23c2cbabadb8bbc01 Mon Sep 17 00:00:00 2001 From: Johannes <24914225+Johannes11833@users.noreply.github.com> Date: Sat, 12 Jul 2025 15:57:50 +0200 Subject: [PATCH 8/9] add progress thresholds in test_progress_listener test --- tests/test_copy.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/test_copy.py b/tests/test_copy.py index 6d212da..4ded5fa 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -166,7 +166,7 @@ def test_progress_listener(tmp_remote_folder, tmp_local_folder, show_progress, m listener=recorder.update, show_progress=show_progress, # limit the bandwidth to see progress updates - args=["--bwlimit 5M"], + args=["--bwlimit 10M"], ) # check that all fields are there @@ -181,22 +181,24 @@ def test_progress_listener(tmp_remote_folder, tmp_local_folder, show_progress, m } assert expected_fields.issubset(unique_fields) + THRESHOLD = 0.25 + # check summary progress assert len(recorder.history) > 0 - assert recorder.get_summary_stats("progress")[0] == pytest.approx(0, abs=0.1) - assert recorder.get_summary_stats("progress")[-1] == pytest.approx(1) + assert recorder.get_summary_stats("progress")[0] == pytest.approx(0, abs=THRESHOLD) + assert recorder.get_summary_stats("progress")[-1] == pytest.approx(1, abs=THRESHOLD) # check file_1 progress file_1_progress = recorder.get_subtask_stats("progress", tmp_file_1.name) assert len(file_1_progress) > 0 - assert file_1_progress[0] == pytest.approx(0, abs=0.1) - assert file_1_progress[-1] == pytest.approx(1) + assert file_1_progress[0] == pytest.approx(0, abs=THRESHOLD) + assert file_1_progress[-1] == pytest.approx(1, abs=THRESHOLD) # check file_2 progress file_2_progress = recorder.get_subtask_stats("progress", tmp_file_2.name) assert len(file_2_progress) > 0 - assert file_2_progress[0] == pytest.approx(0, abs=0.1) - assert file_2_progress[-1] == pytest.approx(1) + assert file_2_progress[0] == pytest.approx(0, abs=THRESHOLD) + assert file_2_progress[-1] == pytest.approx(1, abs=THRESHOLD) def test_rclone_transfer_operation_error_message(default_test_setup, tmp_local_folder): From 62189a1a56e56e59da03f0d552cada099ecca59c Mon Sep 17 00:00:00 2001 From: Johannes <24914225+Johannes11833@users.noreply.github.com> Date: Sat, 12 Jul 2025 16:05:14 +0200 Subject: [PATCH 9/9] update about command --- rclone_python/rclone.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rclone_python/rclone.py b/rclone_python/rclone.py index 853f0c0..da399b4 100644 --- a/rclone_python/rclone.py +++ b/rclone_python/rclone.py @@ -47,15 +47,15 @@ def is_installed() -> bool: @__check_installed -def about(remote_name: str) -> Dict: +def about(path: str) -> Dict: """ Executes the rclone about command and returns the retrieved json as a dictionary. - All sizes are in number of bytes. + All sizes are in number of bytes. This command can be run with local and remote paths. :param remote_name: The name of the remote to examine. :return: Dictionary with remote properties. """ - stdout, _ = utils.run_rclone_cmd(f'about "{remote_name}" --json') + stdout, _ = utils.run_rclone_cmd(f'about "{path}" --json') return json.loads(stdout)