Merge branch 'dev' into 296-ensure-all-examples-are-up-to-date-and-verified

Author: kzscisoft

.pre-commit-config.yaml

@@ -23,7 +23,7 @@ repos:
         args: [--branch, main, --branch, dev]
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.5
+    rev: v0.4.7
     hooks:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix, "--ignore=C901" ]

poetry.lock

@@ -83,19 +83,22 @@ def post(
     if response.status_code == 422:
         _parsed_response = parse_validation_response(response.json())
         raise ValueError(
-            f"Validation error [{response.status_code}]:\n{_parsed_response}"
+            f"Validation error for '{url}' [{response.status_code}]:\n{_parsed_response}"
         )
 
     if response.status_code not in (200, 201, 409):
-        raise RuntimeError(f"HTTP error [{response.status_code}]: {response.text}")
+        raise RuntimeError(
+            f"HTTP error for '{url}' [{response.status_code}]: {response.text}"
+        )
 
     return response
 
 
 @retry(
     wait=wait_exponential(multiplier=RETRY_MULTIPLIER, min=RETRY_MIN, max=RETRY_MAX),
-    # retry=retry_if_not_exception_message(match="Validation error"), # if validation failed no point in retrying
+    retry=retry_if_exception_type(RuntimeError),
     stop=stop_after_attempt(RETRY_STOP),
+    reraise=True,
 )
 def put(
     url: str,
@@ -104,7 +107,7 @@ def put(
     is_json: bool = True,
     timeout: int = DEFAULT_API_TIMEOUT,
 ) -> requests.Response:
-    """HTTP POST with retries
+    """HTTP PUT with retries
 
     Parameters
     ----------

simvue/api.py

@@ -47,7 +47,6 @@ def _execute_process(
             )
 
     _status_code = _result.wait()
-
     with open(f"{runner_name}_{proc_id}.err") as err:
         std_err[proc_id] = err.read()
 

simvue/executor.py

@@ -203,7 +203,7 @@ def send_heartbeat(self) -> typing.Optional[dict[str, typing.Any]]:
         logger.debug(
             f"Creating heartbeat file: {os.path.join(self._directory, 'heartbeat')}"
         )
-        pathlib.Path(os.path.join(self._directory, "heartbeat"), "a").touch()
+        pathlib.Path(os.path.join(self._directory, "heartbeat")).touch()
         return {"success": True}
 
     @skip_if_failed("_aborted", "_suppress_errors", False)

simvue/factory/proxy/offline.py

@@ -99,6 +99,10 @@ def _serialize_plotly_figure(data: typing.Any) -> typing.Optional[tuple[str, str
         return None
     mimetype = "application/vnd.plotly.v1+json"
     data = plotly.io.to_json(data, engine="json")
+    mfile = BytesIO()
+    mfile.write(data.encode())
+    mfile.seek(0)
+    data = mfile.read()
     return data, mimetype
 
 
@@ -110,6 +114,10 @@ def _serialize_matplotlib(data: typing.Any) -> typing.Optional[tuple[str, str]]:
         return None
     mimetype = "application/vnd.plotly.v1+json"
     data = plotly.io.to_json(plotly.tools.mpl_to_plotly(data.gcf()), engine="json")
+    mfile = BytesIO()
+    mfile.write(data.encode())
+    mfile.seek(0)
+    data = mfile.read()
     return data, mimetype
 
 
@@ -121,6 +129,10 @@ def _serialize_matplotlib_figure(data: typing.Any) -> typing.Optional[tuple[str,
         return None
     mimetype = "application/vnd.plotly.v1+json"
     data = plotly.io.to_json(plotly.tools.mpl_to_plotly(data), engine="json")
+    mfile = BytesIO()
+    mfile.write(data.encode())
+    mfile.seek(0)
+    data = mfile.read()
     return data, mimetype
 
 
@@ -161,8 +173,11 @@ def _serialize_torch_tensor(data: typing.Any) -> typing.Optional[tuple[str, str]
 def _serialize_json(data: typing.Any) -> typing.Optional[tuple[str, str]]:
     mimetype = "application/json"
     try:
-        data = json.dumps(data)
-    except TypeError:
+        mfile = BytesIO()
+        mfile.write(json.dumps(data).encode())
+        mfile.seek(0)
+        data = mfile.read()
+    except (TypeError, json.JSONDecodeError):
         return None
     return data, mimetype
 

simvue/serialization.py

@@ -4,6 +4,7 @@
 import sys
 import tempfile
 import pathlib
+import os
 import multiprocessing
 
 
@@ -37,13 +38,31 @@ def test_executor_add_process(
         with pytest.raises(SystemExit):
             run.close()
 
-    time.sleep(1)
-    client = simvue.Client()
-    _events = client.get_events(
-        run._id,
-        message_contains="successfully" if successful else "non-zero exit",
-    )
-    assert len(_events) == 1
+
+@pytest.mark.executor
+def test_executor_multiprocess(request: pytest.FixtureRequest) -> None:
+    with tempfile.TemporaryDirectory() as tempd:
+        with simvue.Run() as run:
+            run.init(
+                "test_executor_multiprocess",
+                folder="/simvue_unit_testing",
+                tags=["simvue_client_tests", request.node.name]
+            )
+
+            for i in range(10):
+                out_file = pathlib.Path(tempd).joinpath(f"out_file_{i}.dat")
+                run.add_process(
+                    f"cmd_{i}",
+                    executable="bash",
+                    c="for i in {0..10}; do sleep 0.5; echo $i >> "+ f"{out_file}; done"
+                )
+            time.sleep(1)
+            for i in range(10):
+                out_file = pathlib.Path(tempd).joinpath(f"out_file_{i}.dat")
+                assert out_file.exists()
+    for i in range(10):
+        os.remove(f"test_executor_multiprocess_cmd_{i}.err")
+        os.remove(f"test_executor_multiprocess_cmd_{i}.out")
 
 
 @pytest.mark.executor