Attempt JSON serialization before pickling and fix bug with save_all

Author: kzscisoft

README.md

@@ -69,13 +69,13 @@ if __name__ == "__main__":
                                description='This is part 1 of a test') # Description
 
         # Upload the code
-        run.save('training.py', 'code')
+        run.save_file('training.py', 'code')
 
         # Upload an input file
-        run.save('params.in', 'input')
+        run.save_file('params.in', 'input')
 
         # Add an alert (the alert definition will be created if necessary)
-        run.add_alert(name='loss-too-high',   # Name
+        run.create_alert(name='loss-too-high',   # Name
                       source='metrics',       # Source
                       rule='is above',        # Rule
                       metric='loss',          # Metric
@@ -96,7 +96,7 @@ if __name__ == "__main__":
             ...
 
         # Upload an output file
-        run.save('output.cdf', 'output')
+        run.save_file('output.cdf', 'output')
 
         # If we weren't using a context manager we'd need to end the run
         # run.close()

examples/GeometryOptimisation/bluemira_simvue_geometry_optimisation.py

@@ -171,5 +171,5 @@ def my_minimise_length(vector, grad, parameterisation, ad_args=None):
 
 # Here we're minimising the length, within the bounds of our PrincetonD parameterisation,
 # so we'd expect that x1 goes to its upper bound, and x2 goes to its lower bound.
-run.save("bluemira_simvue_geometry_optimisation.py", "code")
+run.save_file("bluemira_simvue_geometry_optimisation.py", "code")
 run.close()

examples/PyTorch/main.py

@@ -205,7 +205,7 @@ def main():
         scheduler.step()
 
     if args.save_model:
-        run.save(model.state_dict(), "output", name="mnist_cnn.pt")
+        run.save_file(model.state_dict(), "output", name="mnist_cnn.pt")
 
     run.close()
 

examples/SU2/SU2.py

@@ -56,7 +56,7 @@
         filetype = None
         if input_file.endswith(".cfg"):
             filetype = "text/plain"
-        run.save(input_file, "input", filetype)
+        run.save_file(input_file, "input", filetype)
 
     running = True
     latest = []
@@ -106,6 +106,6 @@
 
     # Save output files
     for output_file in OUTPUT_FILES:
-        run.save(output_file, "output")
+        run.save_file(output_file, "output")
 
     run.close()

examples/Tensorflow/dynamic_rnn.py

@@ -45,7 +45,7 @@
             "computation over sequences with variable length. This example is using a toy dataset to "
             "classify linear sequences. The generated sequences have variable length.",
         )
-        run.save("dynamic_rnn.py", "code")
+        run.save_file("dynamic_rnn.py", "code")
 
         # ====================
         #  TOY DATA GENERATOR

simvue/executor.py

@@ -167,10 +167,10 @@ def callback_function(status_code: int, std_out: str, std_err: str) -> None:
             )
 
         if script:
-            self._runner.save(filename=script, category="code")
+            self._runner.save_file(filename=script, category="code")
 
         if input_file:
-            self._runner.save(filename=input_file, category="input")
+            self._runner.save_file(filename=input_file, category="input")
 
         _command: typing.List[str] = []
 
@@ -284,11 +284,11 @@ def _save_output(self) -> None:
         for proc_id in self._exit_codes.keys():
             # Only save the file if the contents are not empty
             if self._std_err[proc_id]:
-                self._runner.save(
+                self._runner.save_file(
                     f"{self._runner.name}_{proc_id}.err", category="output"
                 )
             if self._std_out[proc_id]:
-                self._runner.save(
+                self._runner.save_file(
                     f"{self._runner.name}_{proc_id}.out", category="output"
                 )
 

simvue/run.py

@@ -1142,7 +1142,7 @@ def save_all(
 
         for item in items:
             if item.is_file():
-                save_file = self.save(f"{item}", category, filetype, preserve_path)
+                save_file = self.save_file(item, category, filetype, preserve_path)
             elif item.is_dir():
                 save_file = self.save_directory(item, category, filetype, preserve_path)
             else:

simvue/serialization.py

@@ -8,6 +8,7 @@
 import typing
 import pickle
 import pandas
+import json
 import numpy
 
 from io import BytesIO
@@ -82,6 +83,8 @@ def serialize_object(
                 return _serialize_matplotlib(data)
         except ImportError:
             pass
+    elif serialized := _serialize_json(data):
+        return serialized
 
     if allow_pickle:
         return _serialize_pickle(data)
@@ -155,6 +158,15 @@ def _serialize_torch_tensor(data: typing.Any) -> typing.Optional[tuple[str, str]
     return data, mimetype
 
 
+def _serialize_json(data: typing.Any) -> typing.Optional[tuple[str, str]]:
+    mimetype = "application/json"
+    try:
+        data = json.dumps(data)
+    except TypeError:
+        return None
+    return data, mimetype
+
+
 def _serialize_pickle(data: typing.Any) -> typing.Optional[tuple[str, str]]:
     mimetype = "application/octet-stream"
     data = pickle.dumps(data)