Make offline mode work

Author: wk9874

examples/detailed_integration.py

@@ -23,6 +23,9 @@
 import simvue_tensorflow.plugin as sv_tf
 
 def tensorflow_example(run_folder, offline=False):
+    # Delete results from previous run, if they exist:
+    pathlib.Path(__file__).parent.joinpath("results").unlink(missing_ok=True)
+    
     # Load the training and test data
     (img_train, label_train), (img_test, label_test) = keras.datasets.fashion_mnist.load_data()
 
@@ -43,8 +46,8 @@ def tensorflow_example(run_folder, offline=False):
 
     # Can use the ModelCheckpoint callback, which is built into Tensorflow, to save a model after each Epoch
     # Providing the model_checkpoint_filepath in the TensorVue callback means it will automatically upload checkpoints to the Epoch runs
-    temp_dir = tempfile.TemporaryDirectory()
-    checkpoint_filepath = str(pathlib.Path(temp_dir.name).joinpath("checkpoint.model.keras"))
+    results_dir = pathlib.Path(__file__).parent.joinpath("results")
+    checkpoint_filepath = str(pathlib.Path(results_dir.name).joinpath("checkpoint.model.keras"))
     model_checkpoint_callback = ModelCheckpoint(
         filepath=checkpoint_filepath, save_best_only=False, verbose=1
     )
@@ -85,7 +88,7 @@ def tensorflow_example(run_folder, offline=False):
         evaluation_target=0.99,
 
         # Choose where the final model is saved
-        model_final_filepath=str(pathlib.Path(temp_dir.name).joinpath("tf_fashion_model.keras"))
+        model_final_filepath=str(pathlib.Path(results_dir.name).joinpath("tf_fashion_model.keras"))
     )
 
     # Fit and evaluate the model, including the tensorvue callback:

examples/example.py

@@ -0,0 +1,55 @@
+"""Create Alerts.
+
+Function for creating alerts based on definitions provided by user
+
+"""
+
+import typing
+
+import simvue
+
+
+def create_alerts(
+    alert_name: str, alert_definition: dict[str, typing.Any], run: simvue.Run
+) -> None:
+    """Create alerts from their definitions provided in to TensorVue.
+
+    Parameters
+    ----------
+    alert_name : str
+        Name of the alert to create
+    alert_definition : dict[str, typing.Any]
+        Definition of the alert, passed into the relevant Run method as kwargs
+    run : simvue.Run
+        The run to add the alerts to
+
+    Raises
+    ------
+    RuntimeError
+        Raised if a valid source could not be deduced from alert definition
+
+    """
+    alert_definition = alert_definition.copy()
+    _source = alert_definition.pop("source")
+    if _source == "events":
+        _alert_id = run.create_event_alert(
+            name=alert_name,
+            **alert_definition,
+        )
+    elif _source == "metrics" and alert_definition.get("threshold"):
+        _alert_id = run.create_metric_threshold_alert(
+            name=alert_name,
+            **alert_definition,
+        )
+    elif _source == "metrics":
+        _alert_id = run.create_metric_range_alert(
+            name=alert_name,
+            **alert_definition,
+        )
+    elif _source == "user":
+        _alert_id = run.create_user_alert(
+            name=alert_name,
+            **alert_definition,
+        )
+    else:
+        raise RuntimeError(f"{alert_name} has unknown source type '{_source}'")

simvue_tensorflow/extras/create_alerts.py

@@ -1,6 +1,6 @@
-"""Common.
+"""Operators.
 
-Classses which could be used in the construction of multiple adapters.
+Operators used in the definition of a TensorVue instance
 """
 # ruff: noqa: DOC201
 

simvue_tensorflow/extras/operators.py

@@ -17,6 +17,7 @@
 from tensorflow.keras.callbacks import Callback
 
 import simvue_tensorflow.extras.operators as operators
+from simvue_tensorflow.extras.create_alerts import create_alerts
 
 
 class TensorVue(Callback):
@@ -110,8 +111,6 @@ def __init__(
             Raised if the ML Optimisation framework is not enabled and no run name was provided
         KeyError
             Raised if attempted to add an alert to a run which was not defined
-        RuntimeError
-            Raised if a valid source could not be deduced from alert definition
 
         """
         if not optimisation_framework and not run_name:
@@ -125,6 +124,7 @@ def __init__(
         self.run_tags = run_tags or []
         self.run_metadata = run_metadata or {}
         self.run_mode = run_mode
+        self.alert_definitions = alert_definitions
         self.script_filepath = script_filepath
         self.model_checkpoint_filepath = model_checkpoint_filepath
         self.model_final_filepath = model_final_filepath
@@ -136,32 +136,14 @@ def __init__(
         self.simulation_run = simulation_run
         self.eval_run = evaluation_run
 
-        self.alerts = {}
+        # Create alerts in a disabled run up front, to validate that they have been defined accurately
         if alert_definitions:
             with simvue.Run(mode="disabled") as temp_run:
-                for alert_name, alert_definition in alert_definitions.items():
-                    _source = alert_definition.pop("source")
-                    if _source == "events":
-                        _alert_id = temp_run.create_event_alert(
-                            name=alert_name, **alert_definition, attach_to_run=False
-                        )
-                    elif _source == "metrics" and alert_definition.get("threshold"):
-                        _alert_id = temp_run.create_metric_threshold_alert(
-                            name=alert_name, **alert_definition, attach_to_run=False
-                        )
-                    elif _source == "metrics":
-                        _alert_id = temp_run.create_metric_range_alert(
-                            name=alert_name, **alert_definition, attach_to_run=False
-                        )
-                    elif _source == "user":
-                        _alert_id = temp_run.create_user_alert(
-                            name=alert_name, **alert_definition, attach_to_run=False
-                        )
-                    else:
-                        raise RuntimeError(
-                            f"{alert_name} has unknown source type '{_source}'"
-                        )
-                    self.alerts[alert_name] = _alert_id
+                temp_run._user_config.run.mode == self.run_mode
+                [
+                    create_alerts(alert_name, alert_definition, temp_run)
+                    for alert_name, alert_definition in alert_definitions.items()
+                ]
 
         self.manifest_alerts = manifest_alerts or []
         self.simulation_alerts = simulation_alerts or []
@@ -175,7 +157,7 @@ def __init__(
             + self.evaluation_alerts
             + self.manifest_alerts
         ):
-            if alert_name not in self.alerts.keys():
+            if alert_name not in self.alert_definitions.keys():
                 raise KeyError(
                     f"Alert name {alert_name} not present in alert definitions."
                 )
@@ -202,10 +184,10 @@ def create_manifest_run(self) -> simvue.Run:
             description=self.run_description,
             metadata=self.run_metadata,
         )
-        if self.manifest_alerts:
-            manifest_run.add_alerts(
-                [self.alerts[alert_name] for alert_name in self.manifest_alerts]
-            )
+        [
+            create_alerts(alert_name, self.alert_definitions[alert_name], manifest_run)
+            for alert_name in self.manifest_alerts
+        ]
 
         if self.script_filepath:
             manifest_run.save_file(
@@ -262,10 +244,12 @@ def on_train_begin(self, logs: dict):
 
         self.simulation_run.update_metadata(self.params)
 
-        if self.simulation_alerts:
-            self.simulation_run.add_alerts(
-                [self.alerts[alert_name] for alert_name in self.simulation_alerts]
+        [
+            create_alerts(
+                alert_name, self.alert_definitions[alert_name], self.simulation_run
             )
+            for alert_name in self.simulation_alerts
+        ]
 
         if self.script_filepath:
             self.simulation_run.save_file(
@@ -336,9 +320,12 @@ def on_epoch_begin(self, epoch: int, logs: dict) -> None:
         )
 
         if self.epoch_alerts and epoch + 1 >= self.start_alerts_from_epoch:
-            self.epoch_run.add_alerts(
-                [self.alerts[alert_name] for alert_name in self.epoch_alerts]
-            )
+            [
+                create_alerts(
+                    alert_name, self.alert_definitions[alert_name], self.epoch_run
+                )
+                for alert_name in self.epoch_alerts
+            ]
 
         if epoch > 0:
             self.epoch_run.log_event("Accuracy and Loss values before epoch training:")
@@ -534,10 +521,12 @@ def on_test_begin(self, logs: dict):
                         "evaluation",
                     ]
                 )
-            if self.evaluation_alerts:
-                self.eval_run.add_alerts(
-                    [self.alerts[alert_name] for alert_name in self.evaluation_alerts]
+            [
+                create_alerts(
+                    alert_name, self.alert_definitions[alert_name], self.eval_run
                 )
+                for alert_name in self.evaluation_alerts
+            ]
 
             if self.script_filepath:
                 self.eval_run.save_file(

simvue_tensorflow/plugin.py

@@ -8,8 +8,7 @@
 @pytest.mark.parametrize("offline", (True, False), ids=("offline", "online"))
 def test_tensorflow_connector(folder_setup, offline):
 
-    run_name = tensorflow_example(folder_setup)
-
+    run_name = tensorflow_example(folder_setup, offline)
     if offline:
         _id_mapping = sender()