diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index a8a890d..93d3793 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -11,7 +11,6 @@ jobs: build: runs-on: ubuntu-latest env: - rclone_config: ${{ secrets.RCLONE_CONFIG }} python_version: "3.10" timezone: "Europe/Berlin" steps: @@ -22,11 +21,9 @@ jobs: python-version: ${{ env.python_version }} - name: Install the rclone software run: sudo -v ; curl https://rclone.org/install.sh | sudo bash - - name: Setup rclone config file + - name: Setup s3 test-server and rclone run: | - mkdir -p ~/.config/rclone - echo "$rclone_config" > ~/.config/rclone/rclone.conf - rclone listremotes + ./launch_test_server.sh - name: Set timezone run: | sudo timedatectl set-timezone ${{ env.timezone }} @@ -38,11 +35,4 @@ jobs: pip install pytest - name: Run test suite run: | - pytest -v - - name: Update rclone config file secret - run: cat ~/.config/rclone/rclone.conf | gh secret set RCLONE_CONFIG - env: - rclone_config_secret_name: RCLONE_CONFIG - GITHUB_TOKEN: ${{ secrets.GH_PAT }} - GH_REPO: ${{ github.repository }} - + pytest -v \ No newline at end of file diff --git a/launch_test_server.bat b/launch_test_server.bat new file mode 100644 index 0000000..85ff8ce --- /dev/null +++ b/launch_test_server.bat @@ -0,0 +1,22 @@ +@echo off +echo Starting MinIO server with Docker... + +docker run --rm -d --name minio ^ + -p 9000:9000 ^ + -p 9001:9001 ^ + -e MINIO_ROOT_USER=minioadmin ^ + -e MINIO_ROOT_PASSWORD=minioadmin ^ + quay.io/minio/minio server /data --console-address ":9001" + +echo MinIO should now be running on: +echo - S3 Endpoint: http://localhost:9000 +echo - Console UI : http://localhost:9001 + + +rclone config create test_server_s3 s3 ^ + provider Minio ^ + env_auth false ^ + access_key_id minioadmin ^ + secret_access_key minioadmin ^ + endpoint http://localhost:9000 ^ + region us-east-1 \ No newline at end of file diff --git a/launch_test_server.sh b/launch_test_server.sh new file mode 100755 index 0000000..30c1d80 --- /dev/null +++ b/launch_test_server.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +echo "Starting MinIO server with Docker..." + +docker run --rm -d --name minio \ + -p 9000:9000 \ + -p 9001:9001 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + quay.io/minio/minio server /data --console-address ":9001" + +echo "MinIO should now be running on:" +echo "- S3 Endpoint: http://localhost:9000" +echo "- Console UI : http://localhost:9001" + +echo "Configuring rclone..." + +rclone config create test_server_s3 s3 \ + provider Minio \ + env_auth false \ + access_key_id minioadmin \ + secret_access_key minioadmin \ + endpoint http://localhost:9000 \ + region us-east-1 \ No newline at end of file diff --git a/rclone_python/rclone.py b/rclone_python/rclone.py index b0242d9..853f0c0 100644 --- a/rclone_python/rclone.py +++ b/rclone_python/rclone.py @@ -50,12 +50,10 @@ def is_installed() -> bool: def about(remote_name: str) -> Dict: """ Executes the rclone about command and returns the retrieved json as a dictionary. + All sizes are in number of bytes. :param remote_name: The name of the remote to examine. :return: Dictionary with remote properties. """ - if not remote_name.endswith(":"): - # if the remote name missed the colon manually add it. - remote_name += ":" stdout, _ = utils.run_rclone_cmd(f'about "{remote_name}" --json') diff --git a/rclone_python/utils.py b/rclone_python/utils.py index 15d5d3c..d4a8413 100644 --- a/rclone_python/utils.py +++ b/rclone_python/utils.py @@ -17,7 +17,7 @@ class RcloneException(ChildProcessError): - def __init__(self, description, error_msg): + def __init__(self, description: str, error_msg: str): self.description = description self.error_msg = error_msg super().__init__(f"{description}. Error message: \n{error_msg}") @@ -189,11 +189,15 @@ def extract_rclone_progress(line: str) -> Tuple[bool, Union[Dict[str, Any], None Returns: Tuple[bool, Union[Dict[str, Any], None]]: The retrieved update Dictionary or error message. """ + stats = None try: log_item: Dict = json.loads(line) - if log_item.get("level", None) == "error": - return False, log_item + level = log_item.get("level", None) + if level != "info": + if level in ("error", "critical"): + # return error message + return False, log_item else: # stats updates use the "info" level stats = log_item.get("stats", None) diff --git a/tests/conftest.py b/tests/conftest.py index 5473582..1bd1bd4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,8 +22,8 @@ def __init__( @pytest.fixture(scope="session") def default_test_setup(): return TestSetup( - remote_name="box", - remote_test_dir_rel="test_dir", + remote_name="test_server_s3", + remote_test_dir_rel="testdir", local_test_txt_file="tests/data/lorem.txt", ) diff --git a/tests/test_about.py b/tests/test_about.py index 08a6c89..1235ea0 100644 --- a/tests/test_about.py +++ b/tests/test_about.py @@ -1,15 +1,11 @@ from rclone_python import rclone -def test_about(default_test_setup): +def test_about(tmp_local_folder): + # NOTE: rclone about does not work on s3 --> do locally instead expected_fields = set(["total", "used", "free"]) - output = rclone.about(default_test_setup.remote_name) - assert expected_fields.issubset(set(output.keys())) - for field in expected_fields: - assert isinstance(output[field], (int, float)) - - output = rclone.about(default_test_setup.remote_name + ":") + output = rclone.about(tmp_local_folder) assert expected_fields.issubset(set(output.keys())) for field in expected_fields: assert isinstance(output[field], (int, float)) diff --git a/tests/test_copy.py b/tests/test_copy.py index 842a9fe..4ded5fa 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -96,12 +96,9 @@ def test_copy(default_test_setup, tmp_remote_folder, tmp_local_folder, command): ) assert (download_path / tmp_local_file.name).is_file() - # download with errors (no such directory) + # upload with errors (no such directory) with pytest.raises(RcloneException, match="directory not found") as e_info: - command( - tmp_remote_folder + "_1", - download_path, - ) + command(str(tmp_local_folder) + "_foo", tmp_remote_folder) def test_sync(default_test_setup, tmp_remote_folder, tmp_local_folder): @@ -168,6 +165,8 @@ def test_progress_listener(tmp_remote_folder, tmp_local_folder, show_progress, m tmp_remote_folder, listener=recorder.update, show_progress=show_progress, + # limit the bandwidth to see progress updates + args=["--bwlimit 10M"], ) # check that all fields are there @@ -182,19 +181,33 @@ def test_progress_listener(tmp_remote_folder, tmp_local_folder, show_progress, m } assert expected_fields.issubset(unique_fields) + THRESHOLD = 0.25 + # check summary progress assert len(recorder.history) > 0 - assert recorder.get_summary_stats("progress")[0] == pytest.approx(0, abs=0.1) - assert recorder.get_summary_stats("progress")[-1] == pytest.approx(1) + assert recorder.get_summary_stats("progress")[0] == pytest.approx(0, abs=THRESHOLD) + assert recorder.get_summary_stats("progress")[-1] == pytest.approx(1, abs=THRESHOLD) # check file_1 progress file_1_progress = recorder.get_subtask_stats("progress", tmp_file_1.name) assert len(file_1_progress) > 0 - assert file_1_progress[0] == pytest.approx(0, abs=0.1) - assert file_1_progress[-1] == pytest.approx(1) + assert file_1_progress[0] == pytest.approx(0, abs=THRESHOLD) + assert file_1_progress[-1] == pytest.approx(1, abs=THRESHOLD) # check file_2 progress file_2_progress = recorder.get_subtask_stats("progress", tmp_file_2.name) assert len(file_2_progress) > 0 - assert file_2_progress[0] == pytest.approx(0, abs=0.1) - assert file_2_progress[-1] == pytest.approx(1) + assert file_2_progress[0] == pytest.approx(0, abs=THRESHOLD) + assert file_2_progress[-1] == pytest.approx(1, abs=THRESHOLD) + + +def test_rclone_transfer_operation_error_message(default_test_setup, tmp_local_folder): + faulty_remote_name = default_test_setup.remote_name + "s:" + + try: + rclone.copy(faulty_remote_name, tmp_local_folder) + assert False + except RcloneException as exception: + # check that a rclone exception message is set + assert len(exception.description) > 0 + assert len(exception.error_msg) > 0 diff --git a/tests/test_mkdir.py b/tests/test_mkdir.py index fb0aac9..c5d0f89 100644 --- a/tests/test_mkdir.py +++ b/tests/test_mkdir.py @@ -1,11 +1,11 @@ from rclone_python import rclone -def test_mkdir(tmp_remote_folder): +def test_mkdir(tmp_local_folder): folder_name = "dir1 v2" - rclone.mkdir(f"{tmp_remote_folder}/{folder_name}") + rclone.mkdir(f"{tmp_local_folder}/{folder_name}") - output = rclone.ls(tmp_remote_folder) + output = rclone.ls(tmp_local_folder) assert len(output) == 1 assert output[0]["Path"] == folder_name diff --git a/tests/test_utils.py b/tests/test_utils.py index 95395b3..5dbaccf 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -46,7 +46,9 @@ def test_extract_rclone_progress_normal_update(valid_rclone_stats_update): # valid input where the total file size is already known input = valid_rclone_stats_update - valid, output = extract_rclone_progress(json.dumps({"stats": input})) + valid, output = extract_rclone_progress( + json.dumps({"stats": input, "level": "info"}) + ) assert valid # validate task summary assert output["total"] == pytest.approx(input["totalBytes"])