Merge branch 'dev' into feature/toml-config

Author: kzscisoft

.github/workflows/citation.yml

@@ -0,0 +1,23 @@
+name: Citation File Validation
+
+on:
+  push:
+    branches: [ "main", "dev", "hotfix/update-ci" ]
+  pull_request:
+    branches: [ "main", "dev", "hotfix/update-ci" ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v3
+    - name: Install Checker
+      run: python3 -m pip install cffconvert
+    - name: Check Citation File
+      run: cffconvert --validate

.github/workflows/test_client_macos.yml

@@ -11,7 +11,7 @@ permissions:
 
 jobs:
   build:
-    timeout-minutes: 20
+    timeout-minutes: 30
     runs-on: macos-latest
     steps:
     - uses: actions/checkout@v3
@@ -28,4 +28,4 @@ jobs:
         export SIMVUE_URL=${{ secrets.SIMVUE_URL }}
         export SIMVUE_TOKEN=${{ secrets.SIMVUE_TOKEN }}
         poetry install --all-extras
-        poetry run pytest tests/unit/ tests/refactor/
+        poetry run pytest tests/unit/ tests/refactor/ -m 'not scenario'

.github/workflows/test_client_ubuntu.yml

@@ -16,7 +16,7 @@ jobs:
   build:
 
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 30
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python 3.12
@@ -30,7 +30,7 @@ jobs:
         export SIMVUE_URL=${{ secrets.SIMVUE_URL }}
         export SIMVUE_TOKEN=${{ secrets.SIMVUE_TOKEN }}
         poetry install --all-extras
-        poetry run pytest --cov --cov-report=xml tests/unit/ tests/refactor/
+        poetry run pytest --cov --cov-report=xml tests/unit/ tests/refactor/ -m 'not scenario'
     - name: Upload coverage reports to Codecov
       run: |
         curl -Os https://uploader.codecov.io/latest/linux/codecov

.github/workflows/test_client_windows.yml

@@ -12,7 +12,7 @@ permissions:
 jobs:
   build:
     runs-on: windows-latest
-    timeout-minutes: 20
+    timeout-minutes: 30
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python 3.12
@@ -29,4 +29,4 @@ jobs:
         export SIMVUE_URL=${{ secrets.SIMVUE_URL }}
         export SIMVUE_TOKEN=${{ secrets.SIMVUE_TOKEN }}
         poetry install --all-extras
-        poetry run pytest tests/unit/ tests/refactor/
+        poetry run pytest tests/unit/ tests/refactor/ -m 'not scenario'

.github/workflows/test_multiple_python.yml

@@ -5,17 +5,16 @@ name: Simvue Client (Python Versions)
 
 on:
   push:
-    branches: [ "main", "dev", "hotfix/update-ci" ]
-  pull_request:
-    branches: [ "main", "dev", "hotfix/update-ci" ]
+    tags:
+      - 'v*-*-rc*'
 
 permissions:
   contents: read
 
 jobs:
   build:
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 30
     strategy:
         matrix:
             python-version: ['3.9', '3.10', '3.11']
@@ -32,4 +31,4 @@ jobs:
         export SIMVUE_URL=${{ secrets.SIMVUE_URL }}
         export SIMVUE_TOKEN=${{ secrets.SIMVUE_TOKEN }}
         poetry install --all-extras
-        poetry run pytest --cov --cov-report=xml tests/unit/ tests/refactor/
+        poetry run pytest tests/unit/ tests/refactor/ -m 'not scenario'

.pre-commit-config.yaml

@@ -23,7 +23,7 @@ repos:
         args: [--branch, main, --branch, dev]
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.4
+    rev: v0.4.7
     hooks:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix, "--ignore=C901" ]

CHANGELOG.md

@@ -1,125 +1,130 @@
 # Change log
 
-## v0.15.0
+## Unreleased
 
-* Support v2 server.
+* Refactor and re-write of codebase to align with latest developments in version 2 of the Simvue server.
+* Added `Executor` to Simvue runs allowing users to start shell based processes as part of a run and handle termination of these.
+* Removal of obsolete functions due to server change, and renaming of functions and parameters (see [documentation](https://docs.simvue.io)).
+* Added pre-request validation to both `Client` and `Run` class methods via Pydantic.
+* Separation of save functionality into `save_file` and `save_object`.
+* Fixed issue whereby metrics would still have to wait for the next iteration of dispatch before being sent to the server, even if the queue was not full.
+* Added support for `'user'` alerts.
 
-## v0.14.3
+## [v0.14.3](https://github.com/simvue-io/client/releases/tag/v0.14.3) - 2023-06-29
 
 * Ensure import of the `requests` module is only done if actually used.
 
-## v0.14.0
+## [v0.14.0](https://github.com/simvue-io/client/releases/tag/v0.14.0) - 2023-04-04
 
 * Added a method to the `Client` class for retrieving events.
 
-## v0.13.3
+## [v0.13.3](https://github.com/simvue-io/client/releases/tag/v0.13.3) - 2023-04-04
 
 * Allow files (`input` and `code` only) to be saved for runs in the `created` state.
 * Allow metadata and tags to be updated for runs in the `created` state.
 
-## v0.13.2
+## [v0.13.2](https://github.com/simvue-io/client/releases/tag/v0.13.2) - 2023-04-04
 
 * Added `plot_metrics` method to the `Client` class to simplify plotting metrics.
 * (Bug fix) `reconnect` works without a uuid being specified when `offline` mode isn't being used.
 * (Bug fix) Restrict version of Pydantic to prevent V2 from accidently being used.
 
-## v0.13.1
+## [v0.13.1](https://github.com/simvue-io/client/releases/tag/v0.13.1) - 2023-03-28
 
 * Set `sample_by` to 0 by default (no sampling) in `get_metrics_multiple`.
 
-## v0.13.0
+## [v0.13.0](https://github.com/simvue-io/client/releases/tag/v0.13.0) - 2023-03-28
 
 * Added methods to the `Client` class for retrieving metrics.
 * CPU architecture and processor obtained on Apple hardware.
 * Client now reports to server when files have been successfully uploaded.
 * `User-Agent` header now included in HTTP requests.
 
-## v0.12.0
+## [v0.12.0](https://github.com/simvue-io/client/releases/tag/v0.12.0) - 2023-03-13
 
 * Add methods to the `Client` class for deleting runs and folders.
 * Confusing messages about `process no longer exists` or `NVML Shared Library Not Found` no longer displayed.
 
-## v0.11.4
+## [v0.11.4](https://github.com/simvue-io/client/releases/tag/v0.11.4) - 2023-03-13
 
 * (Bug fix) Ensure `simvue_sender` can be run when installed from PyPI.
 * (Bug fix) Runs created in `offline` mode using a context manager weren't automatically closed.
 
-## v0.11.3
+## [v0.11.3](https://github.com/simvue-io/client/releases/tag/v0.11.3) - 2023-03-07
 
 * Added logging messages for debugging when debug level set to `debug`.
 
-## v0.11.2
-
+## [v0.11.2](https://github.com/simvue-io/client/releases/tag/v0.11.2) - 2023-03-06
 * Raise exceptions in `Client` class methods if run does not exist or artifact does not exist.
 * (Bug fix) `list_artifacts` optional category restriction now works.
 
-## v0.11.1
+## [v0.11.1](https://github.com/simvue-io/client/releases/tag/v0.11.1) - 2023-03-05
 
 * Support different runs having different metadata in `get_runs` dataframe output.
 * (Bug fix) Error message when creating a duplicate run is now more clear.
 * (Bug fix) Correction to stopping the worker thread in situations where the run never started.
 
-## v0.11.0
+## [v0.11.0](https://github.com/simvue-io/client/releases/tag/v0.11.0) - 2023-03-04
 
 * Support optional dataframe output from `get_runs`.
 
-## v0.10.1
+## [v0.10.1](https://github.com/simvue-io/client/releases/tag/v0.10.1) - 2023-03-03
 
 * The worker process now no longer gives a long delay when a run has finished (now at most ~1 second).
 * The worker process ends when the `Run()` context ends or `close` is called, rather than only when the main process exits.
 
-## v0.10.0
+## [v0.10.0](https://github.com/simvue-io/client/releases/tag/v0.10.0) - 2023-02-07
 
 * The `client` class can now be used to retrieve runs.
 
-## v0.9.1
+## [v0.9.1](https://github.com/simvue-io/client/releases/tag/v0.9.1) - 2023-01-25
 
 * (Bug fix) Retries in POST/PUTs to REST APIs didn't happen.
 * Warn users if `allow_pickle=True` is required.
 
-## v0.9.0
+## [v0.9.0](https://github.com/simvue-io/client/releases/tag/v0.9.0) - 2023-01-25
 
 * Set status to `failed` or `terminated` if the context manager is used and there is an exception.
 
-## v0.8.0
+## [v0.8.0](https://github.com/simvue-io/client/releases/tag/v0.8.0) - 2023-01-23
 
 * Support NumPy arrays, PyTorch tensors, Matplotlib and Plotly plots and picklable Python objects as artifacts.
 * (Bug fix) Events in offline mode didn't work.
 
-## v0.7.2
+## [v0.7.2](https://github.com/simvue-io/client/releases/tag/v0.7.2) - 2023-01-08
 
 * Pydantic model is used for input validation.
 * Support NaN, -inf and inf in metadata and metrics.
 
-## v0.7.0
+## [v0.7.0](https://github.com/simvue-io/client/releases/tag/v0.7.0) - 2022-12-05
 
 * Collect CPU, GPU and memory resource metrics.
 * Automatically delete temporary files used in offline mode once runs have entered a terminal state.
 * Warn users if their access token has expired.
 * Remove dependency on the randomname module, instead handle name generation server side.
 
-## v0.6.0
+## [v0.6.0](https://github.com/simvue-io/client/releases/tag/v0.6.0) - 2022-11-07
 
 * `offline` and `disabled` options replaced with single `mode` flag.
 
-## v0.5.0
+## [v0.5.0](https://github.com/simvue-io/client/releases/tag/v0.5.0) - 2022-11-03
 
 * Added option to disable all monitoring.
 
-## v0.4.0
+## [v0.4.0](https://github.com/simvue-io/client/releases/tag/v0.4.0) - 2022-11-03
 
 * Offline mode added, enabling tracking of simulations running on worker nodes without outgoing network access.
 * Argument to `init` enabling runs to be left in the `created` state changed from `status="created"` to `running=True`.
 * Improvements to error handling.
 
-## v0.3.0
+## [v0.3.0](https://github.com/simvue-io/client/releases/tag/v0.3.0) - 2022-10-31
 
 * Update `add_alert` method to support either metrics or events based alerts.
 
-## v0.2.0
+## [v0.2.0](https://github.com/simvue-io/client/releases/tag/v0.2.0) - 2022-10-26
 
 * The previous `Simvue` class has been split into `Run` and `Client`. When creating a run use the new `Run` class rather than `Simvue`.
 
-## v0.1.0
+## [v0.1.0](https://github.com/simvue-io/client/releases/tag/v0.1.0) - 2022-10-25
 
 * First release.

CITATION.cff

@@ -0,0 +1,44 @@
+# This CITATION.cff file was generated with cffinit.
+# Visit https://bit.ly/cffinit to generate yours today!
+
+cff-version: 1.2.0
+title: Simvue
+message: >-
+  If you use this software, please cite it using the
+  metadata from this file.
+type: software
+authors:
+  - given-names: Andrew
+    family-names: Lahiff
+    affiliation: United Kingdom Atomic Energy Authority
+    orcid: 'https://orcid.org/0000-0002-2785-4116'
+  - given-names: Aby
+    family-names: Abraham
+    affiliation: United Kingdom Atomic Energy Authority
+  - given-names: Vignesh
+    family-names: Gopakumar
+    orcid: 'https://orcid.org/0000-0002-6773-1049'
+    affiliation: United Kingdom Atomic Energy Authority
+  - given-names: Kristian
+    family-names: Zarebski
+    affiliation: United Kingdom Atomic Energy Authority
+    orcid: 'https://orcid.org/0000-0002-6773-1049'
+  - given-names: Matthew
+    family-names: Field
+    affiliation: United Kingdom Atomic Energy Authority
+  - given-names: James
+    family-names: Panayis
+    affiliation: United Kingdom Atomic Energy Authority
+repository-code: 'https://github.com/simvue-io/client'
+url: 'https://simvue.io/'
+abstract: >-
+  Simvue allows users to organise and gain insights in simulations,
+  processing tasks and AI/ML training by capturing metadata and
+  data combined with real-time monitoring, logging and alerting.
+keywords:
+  - tracking
+  - monitoring
+  - metrics
+  - alerting
+  - simulation
+license: Apache-2.0

README.md

@@ -69,13 +69,13 @@ if __name__ == "__main__":
                                description='This is part 1 of a test') # Description
 
         # Upload the code
-        run.save('training.py', 'code')
+        run.save_file('training.py', 'code')
 
         # Upload an input file
-        run.save('params.in', 'input')
+        run.save_file('params.in', 'input')
 
         # Add an alert (the alert definition will be created if necessary)
-        run.add_alert(name='loss-too-high',   # Name
+        run.create_alert(name='loss-too-high',   # Name
                       source='metrics',       # Source
                       rule='is above',        # Rule
                       metric='loss',          # Metric
@@ -91,12 +91,12 @@ if __name__ == "__main__":
             ...
 
             # Send metrics inside main application loop
-            run.log({'loss': 0.5, 'density': 34.4})
+            run.log_metrics({'loss': 0.5, 'density': 34.4})
 
             ...
 
         # Upload an output file
-        run.save('output.cdf', 'output')
+        run.save_file('output.cdf', 'output')
 
         # If we weren't using a context manager we'd need to end the run
         # run.close()

examples/GeometryOptimisation/bluemira_simvue_geometry_optimisation.py

@@ -171,5 +171,5 @@ def my_minimise_length(vector, grad, parameterisation, ad_args=None):
 
 # Here we're minimising the length, within the bounds of our PrincetonD parameterisation,
 # so we'd expect that x1 goes to its upper bound, and x2 goes to its lower bound.
-run.save("bluemira_simvue_geometry_optimisation.py", "code")
+run.save_file("bluemira_simvue_geometry_optimisation.py", "code")
 run.close()