Feature/local s3 test server

.github/workflows/run_tests.yml

@@ -11,7 +11,6 @@ jobs:
   build:  
     runs-on: ubuntu-latest
     env:
-      rclone_config: ${{ secrets.RCLONE_CONFIG }}
       python_version: "3.10"
       timezone: "Europe/Berlin"
     steps:  
@@ -22,11 +21,9 @@ jobs:
           python-version: ${{ env.python_version }}    
       - name: Install the rclone software
         run: sudo -v ; curl https://rclone.org/install.sh | sudo bash
-      - name: Setup rclone config file
+      - name: Setup s3 test-server and rclone
         run: |
-            mkdir -p ~/.config/rclone
-            echo "$rclone_config" > ~/.config/rclone/rclone.conf
-            rclone listremotes
+            ./launch_test_server.sh
       - name: Set timezone
         run: |
             sudo timedatectl set-timezone ${{ env.timezone }}
@@ -38,11 +35,4 @@ jobs:
           pip install pytest
       - name: Run test suite  
         run: |  
-          pytest -v
-      - name: Update rclone config file secret
-        run:  cat ~/.config/rclone/rclone.conf | gh secret set RCLONE_CONFIG
-        env:
-            rclone_config_secret_name: RCLONE_CONFIG
-            GITHUB_TOKEN: ${{ secrets.GH_PAT }}
-            GH_REPO: ${{ github.repository }}
-
+          pytest -v

launch_test_server.bat

@@ -0,0 +1,22 @@
+@echo off
+echo Starting MinIO server with Docker...
+
+docker run --rm -d --name minio ^
+  -p 9000:9000 ^
+  -p 9001:9001 ^
+  -e MINIO_ROOT_USER=minioadmin ^
+  -e MINIO_ROOT_PASSWORD=minioadmin ^
+  quay.io/minio/minio server /data --console-address ":9001"
+
+echo MinIO should now be running on:
+echo - S3 Endpoint: http://localhost:9000
+echo - Console UI : http://localhost:9001
+
+
+rclone config create test_server_s3 s3 ^
+  provider Minio ^
+  env_auth false ^
+  access_key_id minioadmin ^
+  secret_access_key minioadmin ^
+  endpoint http://localhost:9000 ^
+  region us-east-1

launch_test_server.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+echo "Starting MinIO server with Docker..."
+
+docker run --rm -d --name minio \
+  -p 9000:9000 \
+  -p 9001:9001 \
+  -e MINIO_ROOT_USER=minioadmin \
+  -e MINIO_ROOT_PASSWORD=minioadmin \
+  quay.io/minio/minio server /data --console-address ":9001"
+
+echo "MinIO should now be running on:"
+echo "- S3 Endpoint: http://localhost:9000"
+echo "- Console UI : http://localhost:9001"
+
+echo "Configuring rclone..."
+
+rclone config create test_server_s3 s3 \
+  provider Minio \
+  env_auth false \
+  access_key_id minioadmin \
+  secret_access_key minioadmin \
+  endpoint http://localhost:9000 \
+  region us-east-1

rclone_python/rclone.py

@@ -50,12 +50,10 @@ def is_installed() -> bool:
 def about(remote_name: str) -> Dict:
     """
     Executes the rclone about command and returns the retrieved json as a dictionary.
+    All sizes are in number of bytes.
     :param remote_name: The name of the remote to examine.
     :return: Dictionary with remote properties.
     """
-    if not remote_name.endswith(":"):
-        # if the remote name missed the colon manually add it.
-        remote_name += ":"
 
     stdout, _ = utils.run_rclone_cmd(f'about "{remote_name}" --json')
 

rclone_python/utils.py

@@ -17,7 +17,7 @@
 
 
 class RcloneException(ChildProcessError):
-    def __init__(self, description, error_msg):
+    def __init__(self, description: str, error_msg: str):
         self.description = description
         self.error_msg = error_msg
         super().__init__(f"{description}. Error message: \n{error_msg}")
@@ -189,11 +189,15 @@ def extract_rclone_progress(line: str) -> Tuple[bool, Union[Dict[str, Any], None
     Returns:
         Tuple[bool, Union[Dict[str, Any], None]]: The retrieved update Dictionary or error message.
     """
+    stats = None
 
     try:
         log_item: Dict = json.loads(line)
-        if log_item.get("level", None) == "error":
-            return False, log_item
+        level = log_item.get("level", None)
+        if level != "info":
+            if level in ("error", "critical"):
+                # return error message
+                return False, log_item
         else:
             # stats updates use the "info" level
             stats = log_item.get("stats", None)

tests/conftest.py

@@ -22,8 +22,8 @@ def __init__(
 @pytest.fixture(scope="session")
 def default_test_setup():
     return TestSetup(
-        remote_name="box",
-        remote_test_dir_rel="test_dir",
+        remote_name="test_server_s3",
+        remote_test_dir_rel="testdir",
         local_test_txt_file="tests/data/lorem.txt",
     )
 

tests/test_about.py

@@ -1,15 +1,11 @@
 from rclone_python import rclone
 
 
-def test_about(default_test_setup):
+def test_about(tmp_local_folder):
+    # NOTE: rclone about does not work on s3 --> do locally instead
     expected_fields = set(["total", "used", "free"])
 
-    output = rclone.about(default_test_setup.remote_name)
-    assert expected_fields.issubset(set(output.keys()))
-    for field in expected_fields:
-        assert isinstance(output[field], (int, float))
-
-    output = rclone.about(default_test_setup.remote_name + ":")
+    output = rclone.about(tmp_local_folder)
     assert expected_fields.issubset(set(output.keys()))
     for field in expected_fields:
         assert isinstance(output[field], (int, float))

tests/test_copy.py

@@ -96,12 +96,9 @@ def test_copy(default_test_setup, tmp_remote_folder, tmp_local_folder, command):
     )
     assert (download_path / tmp_local_file.name).is_file()
 
-    # download with errors (no such directory)
+    # upload with errors (no such directory)
     with pytest.raises(RcloneException, match="directory not found") as e_info:
-        command(
-            tmp_remote_folder + "_1",
-            download_path,
-        )
+        command(str(tmp_local_folder) + "_foo", tmp_remote_folder)
 
 
 def test_sync(default_test_setup, tmp_remote_folder, tmp_local_folder):
@@ -168,6 +165,8 @@ def test_progress_listener(tmp_remote_folder, tmp_local_folder, show_progress, m
         tmp_remote_folder,
         listener=recorder.update,
         show_progress=show_progress,
+        # limit the bandwidth to see progress updates
+        args=["--bwlimit 10M"],
     )
 
     # check that all fields are there
@@ -182,19 +181,33 @@ def test_progress_listener(tmp_remote_folder, tmp_local_folder, show_progress, m
     }
     assert expected_fields.issubset(unique_fields)
 
+    THRESHOLD = 0.25
+
     # check summary progress
     assert len(recorder.history) > 0
-    assert recorder.get_summary_stats("progress")[0] == pytest.approx(0, abs=0.1)
-    assert recorder.get_summary_stats("progress")[-1] == pytest.approx(1)
+    assert recorder.get_summary_stats("progress")[0] == pytest.approx(0, abs=THRESHOLD)
+    assert recorder.get_summary_stats("progress")[-1] == pytest.approx(1, abs=THRESHOLD)
 
     # check file_1 progress
     file_1_progress = recorder.get_subtask_stats("progress", tmp_file_1.name)
     assert len(file_1_progress) > 0
-    assert file_1_progress[0] == pytest.approx(0, abs=0.1)
-    assert file_1_progress[-1] == pytest.approx(1)
+    assert file_1_progress[0] == pytest.approx(0, abs=THRESHOLD)
+    assert file_1_progress[-1] == pytest.approx(1, abs=THRESHOLD)
 
     # check file_2 progress
     file_2_progress = recorder.get_subtask_stats("progress", tmp_file_2.name)
     assert len(file_2_progress) > 0
-    assert file_2_progress[0] == pytest.approx(0, abs=0.1)
-    assert file_2_progress[-1] == pytest.approx(1)
+    assert file_2_progress[0] == pytest.approx(0, abs=THRESHOLD)
+    assert file_2_progress[-1] == pytest.approx(1, abs=THRESHOLD)
+
+
+def test_rclone_transfer_operation_error_message(default_test_setup, tmp_local_folder):
+    faulty_remote_name = default_test_setup.remote_name + "s:"
+
+    try:
+        rclone.copy(faulty_remote_name, tmp_local_folder)
+        assert False
+    except RcloneException as exception:
+        # check that a rclone exception message is set
+        assert len(exception.description) > 0
+        assert len(exception.error_msg) > 0

tests/test_mkdir.py

@@ -1,11 +1,11 @@
 from rclone_python import rclone
 
 
-def test_mkdir(tmp_remote_folder):
+def test_mkdir(tmp_local_folder):
     folder_name = "dir1  v2"
-    rclone.mkdir(f"{tmp_remote_folder}/{folder_name}")
+    rclone.mkdir(f"{tmp_local_folder}/{folder_name}")
 
-    output = rclone.ls(tmp_remote_folder)
+    output = rclone.ls(tmp_local_folder)
 
     assert len(output) == 1
     assert output[0]["Path"] == folder_name

tests/test_utils.py

@@ -46,7 +46,9 @@ def test_extract_rclone_progress_normal_update(valid_rclone_stats_update):
     # valid input where the total file size is already known
     input = valid_rclone_stats_update
 
-    valid, output = extract_rclone_progress(json.dumps({"stats": input}))
+    valid, output = extract_rclone_progress(
+        json.dumps({"stats": input, "level": "info"})
+    )
     assert valid
     # validate task summary
     assert output["total"] == pytest.approx(input["totalBytes"])