Merge branch 'dev' into 224-prevent-use-of-run-update-methods-when-the-run-instance-has-not-been-initialised

Author: kzscisoft

.github/workflows/test_client_macos.yml

@@ -15,10 +15,10 @@ jobs:
     runs-on: macos-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.11
+    - name: Set up Python 3.12
       uses: actions/setup-python@v3
       with:
-        python-version: "3.11"
+        python-version: "3.12"
     - name: Install dependencies
       run: |
         rm poetry.lock

.github/workflows/test_client_ubuntu.yml

@@ -19,10 +19,10 @@ jobs:
     timeout-minutes: 20
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.11
+    - name: Set up Python 3.12
       uses: actions/setup-python@v3
       with:
-        python-version: "3.11"
+        python-version: "3.12"
     - name: Install dependencies
       run: python -m pip install poetry
     - name: Test with pytest

.github/workflows/test_client_windows.yml

@@ -15,10 +15,10 @@ jobs:
     timeout-minutes: 20
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.11
+    - name: Set up Python 3.12
       uses: actions/setup-python@v3
       with:
-        python-version: "3.11"
+        python-version: "3.12"
     - name: Install dependencies
       run: |
         rm poetry.lock

.github/workflows/test_multiple_python.yml

@@ -0,0 +1,35 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Simvue Client (Python Versions)
+
+on:
+  push:
+    branches: [ "main", "dev", "hotfix/update-ci" ]
+  pull_request:
+    branches: [ "main", "dev", "hotfix/update-ci" ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+        matrix:
+            python-version: ['3.9', '3.10', '3.11']
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: "${{ matrix.python-version }}"
+    - name: Install dependencies
+      run: python -m pip install poetry
+    - name: Test with pytest
+      run: |
+        export SIMVUE_URL=${{ secrets.SIMVUE_URL }}
+        export SIMVUE_TOKEN=${{ secrets.SIMVUE_TOKEN }}
+        poetry install --all-extras
+        poetry run pytest --cov --cov-report=xml tests/unit/ tests/refactor/

.pre-commit-config.yaml

@@ -23,7 +23,7 @@ repos:
         args: [--branch, main, --branch, dev]
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.3
+    rev: v0.4.4
     hooks:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix, "--ignore=C901" ]

README.md

@@ -12,6 +12,7 @@ Collect metadata, metrics and artifacts from simulations, processing and AI/ML t
 
 <div align="center">
 <a href="https://github.com/simvue-io/client/blob/main/LICENSE" target="_blank"><img src="https://img.shields.io/github/license/simvue-io/client"/></a>
+<img src="https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue">
 <a href="https://pypi.org/project/simvue/" target="_blank"><img src="https://img.shields.io/pypi/v/simvue.svg"/></a>
 <a href="https://pepy.tech/project/simvue"><img src="https://static.pepy.tech/badge/simvue"/></a>
 <p></p>

poetry.lock

@@ -43,8 +43,11 @@ pandas = {version = "^2.2.0", optional = true}
 plotly = {version = "^5.18.0", optional = true}
 numpy = {version = "^1.26.3", optional = true}
 matplotlib = {version = "^3.8.2", optional = true}
+typing_extensions = { version = "^4.11.0", python = "<3.10" }
 toml = "^0.10.2"
 click = "^8.1.7"
+humanfriendly = "^10.0"
+tabulate = "^0.9.0"
 
 [tool.poetry.extras]
 dataset = ["pandas", "numpy"]

pyproject.toml

@@ -258,10 +258,12 @@ def get_runs(
         alerts: bool = False,
         metadata: bool = False,
         format: typing.Literal["dict", "dataframe"] = "dict",
+        count: int = 100,
+        start_index: int = 0,
     ) -> typing.Union[
         "DataFrame", list[dict[str, typing.Union[int, str, float, None]]], None
     ]:
-        """Retrieve all runs matching filters
+        """Retrieve all runs matching filters.
 
         Parameters
         ----------
@@ -280,6 +282,10 @@ def get_runs(
         format : str ('dict' | 'dataframe'), optional
             the structure of the response, either a dictionary or a dataframe.
             Default is 'dict'. Pandas must be installed for 'dataframe'.
+        count : int, optional
+            maximum number of entries to return. Default is 100.
+        start_index : int, optional
+            the index from which to count entries. Default is 0.
 
         Returns
         -------
@@ -301,6 +307,8 @@ def get_runs(
             "return_alerts": alerts,
             "return_system": system,
             "return_metadata": metadata,
+            "count": count,
+            "start": start_index,
         }
 
         response = requests.get(
@@ -778,14 +786,21 @@ def get_folder(self, folder_id: str) -> typing.Optional[dict[str, typing.Any]]:
         return _folders[0]
 
     def get_folders(
-        self, filters: typing.Optional[list[str]] = None
+        self,
+        filters: typing.Optional[list[str]] = None,
+        count: int = 100,
+        start_index: int = 0,
     ) -> list[dict[str, typing.Any]]:
         """Retrieve folders from the server
 
         Parameters
         ----------
         filters : list[str] | None
             set of filters to apply to the search
+        count : int, optional
+            maximum number of entries to return. Default is 100.
+        start_index : int, optional
+            the index from which to count entries. Default is 0.
 
         Returns
         -------
@@ -797,7 +812,11 @@ def get_folders(
         RuntimeError
             if there was a failure retrieving data from the server
         """
-        params: dict[str, str] = {"filters": json.dumps(filters or [])}
+        params: dict[str, typing.Union[str, int]] = {
+            "filters": json.dumps(filters or []),
+            "count": count,
+            "start": start_index,
+        }
 
         response: requests.Response = requests.get(
             f"{self._url}/api/folders", headers=self._headers, params=params
@@ -873,7 +892,7 @@ def _get_run_metrics_from_server(
             "xaxis": xaxis,
             "max_points": max_points,
         }
-        print(params)
+
         metrics_response: requests.Response = requests.get(
             f"{self._url}/api/metrics", headers=self._headers, params=params
         )

simvue/client.py

@@ -11,6 +11,8 @@
 __date__ = "2023-11-15"
 
 import logging
+import multiprocessing.synchronize
+import sys
 import multiprocessing
 import os
 import subprocess
@@ -23,6 +25,48 @@
 logger = logging.getLogger(__name__)
 
 
+def _execute_process(
+    proc_id: str,
+    command: typing.List[str],
+    runner_name: str,
+    exit_status_dict: typing.Dict[str, int],
+    std_err: typing.Dict[str, str],
+    std_out: typing.Dict[str, str],
+    run_on_exit: typing.Optional[typing.Callable[[int, int, str], None]],
+    trigger: typing.Optional[multiprocessing.synchronize.Event],
+    environment: typing.Optional[typing.Dict[str, str]],
+) -> None:
+    with open(f"{runner_name}_{proc_id}.err", "w") as err:
+        with open(f"{runner_name}_{proc_id}.out", "w") as out:
+            _result = subprocess.Popen(
+                command,
+                stdout=out,
+                stderr=err,
+                universal_newlines=True,
+                env=environment,
+            )
+
+    _status_code = _result.wait()
+
+    with open(f"{runner_name}_{proc_id}.err") as err:
+        std_err[proc_id] = err.read()
+
+    with open(f"{runner_name}_{proc_id}.out") as out:
+        std_out[proc_id] = out.read()
+
+    exit_status_dict[proc_id] = _status_code
+
+    if run_on_exit:
+        run_on_exit(
+            status_code=exit_status_dict[proc_id],
+            std_out=std_out[proc_id],
+            std_err=std_err[proc_id],
+        )
+
+    if trigger:
+        trigger.set()
+
+
 class Executor:
     """Command Line command executor
 
@@ -62,6 +106,7 @@ def add_process(
         completion_callback: typing.Optional[
             typing.Callable[[int, str, str], None]
         ] = None,
+        completion_trigger: multiprocessing.synchronize.Event,
         **kwargs,
     ) -> None:
         """Add a process to be executed to the executor.
@@ -91,6 +136,8 @@ def callback_function(status_code: int, std_out: str, std_err: str) -> None:
             ...
         ```
 
+        Note `completion_callback` is not supported on Windows operating systems.
+
         Parameters
         ----------
         identifier : str
@@ -107,55 +154,24 @@ def callback_function(status_code: int, std_out: str, std_err: str) -> None:
         env : typing.Dict[str, str], optional
             environment variables for process
         completion_callback : typing.Callable | None, optional
-            callback to run when process terminates
+            callback to run when process terminates (not supported on Windows)
+        completion_trigger : multiprocessing.Event | None, optional
+            this trigger event is set when the processes completes
         """
         _pos_args = list(args)
 
+        if sys.platform == "win32" and completion_callback:
+            logger.warning(
+                "Completion callback for 'add_process' may fail on Windows due to "
+                "due to function pickling restrictions"
+            )
+
         if script:
             self._runner.save(filename=script, category="code")
 
         if input_file:
             self._runner.save(filename=input_file, category="input")
 
-        def _exec_process(
-            proc_id: str,
-            command: typing.List[str],
-            runner: "simvue.Run",
-            exit_status_dict: typing.Dict[str, int],
-            std_err: typing.Dict[str, str],
-            std_out: typing.Dict[str, str],
-            run_on_exit: typing.Optional[
-                typing.Callable[[int, int, str], None]
-            ] = completion_callback,
-            environment: typing.Optional[typing.Dict[str, str]] = env,
-        ) -> None:
-            with open(f"{runner.name}_{proc_id}.err", "w") as err:
-                with open(f"{runner.name}_{proc_id}.out", "w") as out:
-                    _result = subprocess.Popen(
-                        command,
-                        stdout=out,
-                        stderr=err,
-                        universal_newlines=True,
-                        env=environment,
-                    )
-
-            _status_code = _result.wait()
-
-            with open(f"{runner.name}_{proc_id}.err") as err:
-                std_err[proc_id] = err.read()
-
-            with open(f"{runner.name}_{proc_id}.out") as out:
-                std_out[proc_id] = out.read()
-
-            exit_status_dict[proc_id] = _status_code
-
-            if run_on_exit:
-                run_on_exit(
-                    status_code=exit_status_dict[proc_id],
-                    std_out=std_out[proc_id],
-                    std_err=std_err[proc_id],
-                )
-
         _command: typing.List[str] = []
 
         if executable:
@@ -189,14 +205,17 @@ def _exec_process(
         self._command_str[identifier] = " ".join(_command)
 
         self._processes[identifier] = multiprocessing.Process(
-            target=_exec_process,
+            target=_execute_process,
             args=(
                 identifier,
                 _command,
-                self._runner,
+                self._runner.name,
                 self._exit_codes,
                 self._std_err,
                 self._std_out,
+                completion_callback,
+                completion_trigger,
+                env,
             ),
         )
         logger.debug(f"Executing process: {' '.join(_command)}")
@@ -297,7 +316,8 @@ def _clear_cache_files(self) -> None:
     def wait_for_completion(self) -> None:
         """Wait for all processes to finish then perform tidy up and upload"""
         for process in self._processes.values():
-            process.join()
+            if process.is_alive():
+                process.join()
         self._log_events()
         self._save_output()