Merge pull request #21 from simvue-io/offline_mode

Add offline mode

Author: alahiff

bin/simvue_sender

@@ -0,0 +1,29 @@
+"""Send runs to server"""
+import getpass
+import os
+import logging
+import sys
+
+from simvue.sender import sender
+from simvue.utilities import create_file, remove_file
+
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+
+handler = logging.StreamHandler(sys.stdout)
+handler.setLevel(logging.INFO)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+
+if __name__ == "__main__":
+    lockfile = f"/tmp/simvue-{getpass.getuser()}.lock"
+    if not os.path.isfile(lockfile):
+        create_file(lockfile)
+
+        try:
+            sender()
+        except Exception as err:
+            logger.error('Exception running sender: %s', str(err))
+
+        remove_file(lockfile)

setup.py

@@ -12,13 +12,14 @@
     version=version,
     author="Andrew Lahiff",
     author_email="[email protected]",
-    description="Simulation tracking",
+    description="Simulation tracking and monitoring",
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/simvue-io/client",
     platforms=["any"],
     install_requires=["requests", "randomname", "msgpack", "tenacity"],
     package_dir={'': '.'},
-    packages=['simvue'],
+    packages=["simvue"],
     package_data={"": ["README.md"]},
+    scripts=["bin/simvue_sender"]
 )

simvue/client.py

@@ -2,7 +2,7 @@
 import os
 import requests
 
-from .utilities import get_config
+from .utilities import get_auth
 
 CONCURRENT_DOWNLOADS = 10
 DOWNLOAD_CHUNK_SIZE = 8192
@@ -31,7 +31,7 @@ class Client(object):
     Class for querying Simvue
     """
     def __init__(self):
-        self._url, self._token = get_config()
+        self._url, self._token = get_auth()
         self._headers = {"Authorization": f"Bearer {self._token}"}
 
     def list_artifacts(self, run, category=None):

simvue/offline.py

@@ -0,0 +1,101 @@
+import json
+import logging
+import os
+import time
+
+from .utilities import get_offline_directory, get_directory_name, create_file
+
+logger = logging.getLogger(__name__)
+
+class Offline(object):
+    """
+    Class for offline runs
+    """
+    def __init__(self, name, suppress_errors=False):
+        self._name = name
+        self._directory = os.path.join(get_offline_directory(), get_directory_name(name))
+        self._suppress_errors = suppress_errors
+
+    def _error(self, message):
+        """
+        Raise an exception if necessary and log error
+        """
+        if not self._suppress_errors:
+            raise RuntimeError(message)
+        else:
+            logger.error(message)
+
+    def _write_json(self, filename, data):
+        """
+        Write JSON to file
+        """
+        try:
+            with open(filename, 'w') as fh:
+                json.dump(data, fh)
+        except Exception as err:
+            self._error(f"Unable to write file {filename} due to {str(err)}")
+
+    def create_run(self, data):
+        """
+        Create a run
+        """
+        try:
+            os.makedirs(self._directory, exist_ok=True)
+        except Exception as err:
+            logger.error('Unable to create directory %s due to: %s', self._directory, str(err))
+        
+        filename = f"{self._directory}/run.json"
+        self._write_json(filename, data)
+
+        status = data['status']
+        filename = f"{self._directory}/{status}"
+        create_file(filename)
+
+        return True
+
+    def update(self, data):
+        """
+        Update metadata, tags or status
+        """
+        unique_id = time.time()
+        filename = f"{self._directory}/update-{unique_id}.json"
+        self._write_json(filename, data)
+
+        if 'status' in data:
+            status = data['status']
+            filename = f"{self._directory}/{status}"
+            create_file(filename)
+
+            if status == 'completed':
+                status_running = f"{self._directory}/running"
+                if os.path.isfile(status_running):
+                    os.remove(status_running)
+
+        return True
+
+    def set_folder_details(self, data):
+        """
+        Set folder details
+        """
+        unique_id = time.time()
+        filename = f"{self._directory}/folder-{unique_id}.json"
+        self._write_json(filename, data)
+        return True
+
+    def save_file(self, data):
+        """
+        Save file
+        """
+        unique_id = time.time()
+        filename = f"{self._directory}/file-{unique_id}.json"
+        self._write_json(filename, data)
+        return True
+
+    def add_alert(self, data):
+        """
+        Add an alert
+        """
+        unique_id = time.time()
+        filename = f"{self._directory}/alert-{unique_id}.json"
+        self._write_json(filename, data)
+        return True

simvue/remote.py

@@ -0,0 +1,171 @@
+import logging
+import requests
+
+from .utilities import get_auth
+
+logger = logging.getLogger(__name__)
+
+UPLOAD_TIMEOUT = 30
+DEFAULT_API_TIMEOUT = 10
+
+class Remote(object):
+    """
+    Class which interacts with Simvue REST API
+    """
+    def __init__(self, name, suppress_errors=False):
+        self._name = name
+        self._suppress_errors = suppress_errors
+        self._url, self._token = get_auth()
+        self._headers = {"Authorization": f"Bearer {self._token}"}
+        self._headers_mp = self._headers.copy()
+        self._headers_mp['Content-Type'] = 'application/msgpack'
+
+    def _error(self, message):
+        """
+        Raise an exception if necessary and log error
+        """
+        if not self._suppress_errors:
+            raise RuntimeError(message)
+        else:
+            logger.error(message)
+
+    def create_run(self, data):
+        """
+        Create a run
+        """
+        try:
+            response = requests.post(f"{self._url}/api/runs",
+                                     headers=self._headers,
+                                     json=data,
+                                     timeout=DEFAULT_API_TIMEOUT)
+        except requests.exceptions.RequestException:
+            return False
+
+        if response.status_code != 200:
+            self._error('Unable to reconnect to run')
+            return False
+
+        return True
+
+    def update(self, data):
+        """
+        Update metadata, tags or status
+        """
+        try:
+            response = requests.put(f"{self._url}/api/runs",
+                                    headers=self._headers,
+                                    json=data,
+                                    timeout=DEFAULT_API_TIMEOUT)
+        except requests.exceptions.RequestException:
+            pass
+
+        if response.status_code == 200:
+            return True
+
+        return False
+
+    def set_folder_details(self, data):
+        """
+        Set folder details
+        """
+        try:
+            response = requests.put(f"{self._url}/api/folders",
+                                    headers=self._headers,
+                                    json=data,
+                                    timeout=DEFAULT_API_TIMEOUT)
+        except requests.exceptions.RequestException:
+            pass
+
+        if response.status_code == 200:
+            return True
+
+        return False
+
+    def save_file(self, data):
+        """
+        Save file
+        """
+        # Get presigned URL
+        try:
+            response = requests.post(f"{self._url}/api/data",
+                                 headers=self._headers,
+                                 json=data,
+                                 timeout=DEFAULT_API_TIMEOUT)
+        except requests.exceptions.RequestException as err:
+            self._error(f"Got exception when uploading file {data['name']} to object storage: {str(err)}")
+            return False
+
+        if response.status_code == 409:
+            return True
+
+        if response.status_code != 200:
+            self._error(f"Got status code {response.status_code} when registering file {data['name']}")
+            return False
+
+        if 'url' in response.json():
+            try:
+                with open(data['originalPath'], 'rb') as fh:
+                    response = requests.put(response.json()['url'],
+                                            data=fh,
+                                            timeout=UPLOAD_TIMEOUT)
+                    if response.status_code != 200:
+                        self._error(f"Got status code {response.status_code} when uploading file {data['name']} to object storage")
+                        return None
+            except Exception as err:
+                self._error(f"Got exception when uploading file {data['name']} to object storage: {str(err)}")
+                return None
+
+        return True
+
+    def add_alert(self, data):
+        """
+        Add an alert
+        """
+        try:
+            response = requests.put(f"{self._url}/api/runs",
+                                    headers=self._headers,
+                                    json=data,
+                                    timeout=DEFAULT_API_TIMEOUT)
+        except requests.exceptions.RequestException:
+            pass
+
+        if response.status_code == 200:
+            return True
+
+        return False
+
+    def send_metrics(self, data):
+        """
+        Send metrics
+        """
+        try:
+            response = requests.post(f"{self._url}/api/metrics",
+                                     headers=self._headers_mp,
+                                     data=data,
+                                     timeout=DEFAULT_API_TIMEOUT)
+        except:
+            pass
+
+    def send_event(self, data):
+        """
+        Send events
+        """
+        try:
+            response = requests.post(f"{self._url}/api/events",
+                                     headers=self._headers_mp,
+                                     data=data,
+                                     timeout=DEFAULT_API_TIMEOUT)
+        except:
+            pass
+
+    def send_heartbeat(self):
+        """
+        Send heartbeat
+        """
+        try:
+            response = requests.put(f"{self._url}/api/runs/heartbeat",
+                                    headers=self._headers,
+                                    json={'name': self._name},
+                                    timeout=DEFAULT_API_TIMEOUT)
+        except:
+            pass