diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index a8a890d..93d3793 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -11,7 +11,6 @@ jobs: build: runs-on: ubuntu-latest env: - rclone_config: ${{ secrets.RCLONE_CONFIG }} python_version: "3.10" timezone: "Europe/Berlin" steps: @@ -22,11 +21,9 @@ jobs: python-version: ${{ env.python_version }} - name: Install the rclone software run: sudo -v ; curl https://rclone.org/install.sh | sudo bash - - name: Setup rclone config file + - name: Setup s3 test-server and rclone run: | - mkdir -p ~/.config/rclone - echo "$rclone_config" > ~/.config/rclone/rclone.conf - rclone listremotes + ./launch_test_server.sh - name: Set timezone run: | sudo timedatectl set-timezone ${{ env.timezone }} @@ -38,11 +35,4 @@ jobs: pip install pytest - name: Run test suite run: | - pytest -v - - name: Update rclone config file secret - run: cat ~/.config/rclone/rclone.conf | gh secret set RCLONE_CONFIG - env: - rclone_config_secret_name: RCLONE_CONFIG - GITHUB_TOKEN: ${{ secrets.GH_PAT }} - GH_REPO: ${{ github.repository }} - + pytest -v \ No newline at end of file diff --git a/launch_test_server.bat b/launch_test_server.bat new file mode 100644 index 0000000..85ff8ce --- /dev/null +++ b/launch_test_server.bat @@ -0,0 +1,22 @@ +@echo off +echo Starting MinIO server with Docker... + +docker run --rm -d --name minio ^ + -p 9000:9000 ^ + -p 9001:9001 ^ + -e MINIO_ROOT_USER=minioadmin ^ + -e MINIO_ROOT_PASSWORD=minioadmin ^ + quay.io/minio/minio server /data --console-address ":9001" + +echo MinIO should now be running on: +echo - S3 Endpoint: http://localhost:9000 +echo - Console UI : http://localhost:9001 + + +rclone config create test_server_s3 s3 ^ + provider Minio ^ + env_auth false ^ + access_key_id minioadmin ^ + secret_access_key minioadmin ^ + endpoint http://localhost:9000 ^ + region us-east-1 \ No newline at end of file diff --git a/launch_test_server.sh b/launch_test_server.sh new file mode 100755 index 0000000..30c1d80 --- /dev/null +++ b/launch_test_server.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +echo "Starting MinIO server with Docker..." + +docker run --rm -d --name minio \ + -p 9000:9000 \ + -p 9001:9001 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + quay.io/minio/minio server /data --console-address ":9001" + +echo "MinIO should now be running on:" +echo "- S3 Endpoint: http://localhost:9000" +echo "- Console UI : http://localhost:9001" + +echo "Configuring rclone..." + +rclone config create test_server_s3 s3 \ + provider Minio \ + env_auth false \ + access_key_id minioadmin \ + secret_access_key minioadmin \ + endpoint http://localhost:9000 \ + region us-east-1 \ No newline at end of file diff --git a/rclone_python/rclone.py b/rclone_python/rclone.py index b0242d9..da399b4 100644 --- a/rclone_python/rclone.py +++ b/rclone_python/rclone.py @@ -47,17 +47,15 @@ def is_installed() -> bool: @__check_installed -def about(remote_name: str) -> Dict: +def about(path: str) -> Dict: """ Executes the rclone about command and returns the retrieved json as a dictionary. + All sizes are in number of bytes. This command can be run with local and remote paths. :param remote_name: The name of the remote to examine. :return: Dictionary with remote properties. """ - if not remote_name.endswith(":"): - # if the remote name missed the colon manually add it. - remote_name += ":" - stdout, _ = utils.run_rclone_cmd(f'about "{remote_name}" --json') + stdout, _ = utils.run_rclone_cmd(f'about "{path}" --json') return json.loads(stdout) diff --git a/tests/conftest.py b/tests/conftest.py index 5473582..1bd1bd4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,8 +22,8 @@ def __init__( @pytest.fixture(scope="session") def default_test_setup(): return TestSetup( - remote_name="box", - remote_test_dir_rel="test_dir", + remote_name="test_server_s3", + remote_test_dir_rel="testdir", local_test_txt_file="tests/data/lorem.txt", ) diff --git a/tests/test_about.py b/tests/test_about.py index 08a6c89..1235ea0 100644 --- a/tests/test_about.py +++ b/tests/test_about.py @@ -1,15 +1,11 @@ from rclone_python import rclone -def test_about(default_test_setup): +def test_about(tmp_local_folder): + # NOTE: rclone about does not work on s3 --> do locally instead expected_fields = set(["total", "used", "free"]) - output = rclone.about(default_test_setup.remote_name) - assert expected_fields.issubset(set(output.keys())) - for field in expected_fields: - assert isinstance(output[field], (int, float)) - - output = rclone.about(default_test_setup.remote_name + ":") + output = rclone.about(tmp_local_folder) assert expected_fields.issubset(set(output.keys())) for field in expected_fields: assert isinstance(output[field], (int, float)) diff --git a/tests/test_copy.py b/tests/test_copy.py index 817bc04..4ded5fa 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -96,12 +96,9 @@ def test_copy(default_test_setup, tmp_remote_folder, tmp_local_folder, command): ) assert (download_path / tmp_local_file.name).is_file() - # download with errors (no such directory) + # upload with errors (no such directory) with pytest.raises(RcloneException, match="directory not found") as e_info: - command( - tmp_remote_folder + "_1", - download_path, - ) + command(str(tmp_local_folder) + "_foo", tmp_remote_folder) def test_sync(default_test_setup, tmp_remote_folder, tmp_local_folder): @@ -168,6 +165,8 @@ def test_progress_listener(tmp_remote_folder, tmp_local_folder, show_progress, m tmp_remote_folder, listener=recorder.update, show_progress=show_progress, + # limit the bandwidth to see progress updates + args=["--bwlimit 10M"], ) # check that all fields are there @@ -182,22 +181,24 @@ def test_progress_listener(tmp_remote_folder, tmp_local_folder, show_progress, m } assert expected_fields.issubset(unique_fields) + THRESHOLD = 0.25 + # check summary progress assert len(recorder.history) > 0 - assert recorder.get_summary_stats("progress")[0] == pytest.approx(0, abs=0.1) - assert recorder.get_summary_stats("progress")[-1] == pytest.approx(1) + assert recorder.get_summary_stats("progress")[0] == pytest.approx(0, abs=THRESHOLD) + assert recorder.get_summary_stats("progress")[-1] == pytest.approx(1, abs=THRESHOLD) # check file_1 progress file_1_progress = recorder.get_subtask_stats("progress", tmp_file_1.name) assert len(file_1_progress) > 0 - assert file_1_progress[0] == pytest.approx(0, abs=0.1) - assert file_1_progress[-1] == pytest.approx(1) + assert file_1_progress[0] == pytest.approx(0, abs=THRESHOLD) + assert file_1_progress[-1] == pytest.approx(1, abs=THRESHOLD) # check file_2 progress file_2_progress = recorder.get_subtask_stats("progress", tmp_file_2.name) assert len(file_2_progress) > 0 - assert file_2_progress[0] == pytest.approx(0, abs=0.1) - assert file_2_progress[-1] == pytest.approx(1) + assert file_2_progress[0] == pytest.approx(0, abs=THRESHOLD) + assert file_2_progress[-1] == pytest.approx(1, abs=THRESHOLD) def test_rclone_transfer_operation_error_message(default_test_setup, tmp_local_folder): diff --git a/tests/test_mkdir.py b/tests/test_mkdir.py index fb0aac9..c5d0f89 100644 --- a/tests/test_mkdir.py +++ b/tests/test_mkdir.py @@ -1,11 +1,11 @@ from rclone_python import rclone -def test_mkdir(tmp_remote_folder): +def test_mkdir(tmp_local_folder): folder_name = "dir1 v2" - rclone.mkdir(f"{tmp_remote_folder}/{folder_name}") + rclone.mkdir(f"{tmp_local_folder}/{folder_name}") - output = rclone.ls(tmp_remote_folder) + output = rclone.ls(tmp_local_folder) assert len(output) == 1 assert output[0]["Path"] == folder_name