Merge pull request #23 from dwolfson/v.40

several fixes to coco config and to some of the widgets

Author: dwolfson

examples/Coco_config/config_cocoMDS1.py

@@ -40,16 +40,16 @@
 
     # Can also inherit event bus config from application properties
 
-    # event_bus_config = {
-    #     "producer": {
-    #         "bootstrap.servers": "localhost:9092"
-    #     },
-    #     "consumer": {
-    #         "bootstrap.servers": "localhost:9092"
-    #     }
-    # }
-
-    # o_client.set_event_bus(event_bus_config)
+    event_bus_config = {
+        "producer": {
+            "bootstrap.servers": "{{kafkaEndpoint}}"
+        },
+        "consumer": {
+            "bootstrap.servers": "{{kafkaEndpoint}}"
+        }
+    }
+
+    o_client.set_event_bus(event_bus_config)
 
     security_connection_body = {
         "class": "Connection",

examples/Coco_config/config_cocoMDS2.py

@@ -44,16 +44,16 @@
 
 #   Inherit event bus config
 
-    # event_bus_config = {
-    #     "producer": {
-    #         "bootstrap.servers": "localhost:9092"
-    #     },
-    #     "consumer": {
-    #         "bootstrap.servers": "localhost:9092"
-    #     }
-    # }
-    #
-    # o_client.set_event_bus(event_bus_config)
+    event_bus_config = {
+        "producer": {
+            "bootstrap.servers": "{{kafkaEndpoint}}"
+        },
+        "consumer": {
+            "bootstrap.servers": "{{kafkaEndpoint}}"
+        }
+    }
+
+    o_client.set_event_bus(event_bus_config)
 
     security_connection_body = {
         "class": "Connection",

examples/Coco_config/start_all_coco.py

@@ -0,0 +1,9 @@
+import subprocess
+
+startup_scripts = ["config_cocoMDS1.py", "config_cocoMDS2.py",
+                   "config_cocoMDS3.py", "config_cocoMDS4.py",
+                   "config_cocoMDS5.py", "config_cocoMDSx.py",
+                   "config_cocoView1.py", "config_exchangeDL01.py",
+                   "config_monitorDev01.py", "config_monitorGov01.py"]
+for script in startup_scripts:
+    subprocess.call(["python",script])

examples/widgets/catalog_user/view_asset_graph.py

@@ -48,9 +48,10 @@ def build_classifications(classification: dict) -> Markdown:
             if c_type == "Anchors":
                 continue
             class_md += f"* Classification: {c_type}\n"
-            class_props = c["classificationProperties"]
-            for prop in class_props.keys():
-                class_md += f"\t* {prop}: {class_props[prop]}\n"
+            class_props = c.get("classificationProperties","---")
+            if type(class_props) is list:
+                for prop in class_props.keys():
+                    class_md += f"\t* {prop}: {class_props[prop]}\n"
         if class_md == "":
             output = None
         else:

examples/widgets/operational/view_asset_changes.py

@@ -10,8 +10,8 @@
 import time
 import argparse
 from confluent_kafka import Consumer, KafkaException
-
-
+from datetime import datetime
+from rich.prompt import Prompt
 from rich.table import Table
 from rich.live import Live
 from rich.console import Console
@@ -21,15 +21,18 @@
 
 disable_ssl_warnings = True
 console = Console(width=200)
-
+now = datetime.now()
+current_time = now.strftime("%H:%M:%S")
 
 from confluent_kafka import Consumer, KafkaException
 
+earliest_latest = Prompt.ask("Msgs from earliest or latest:", default="earliest")
+
 # Define the Kafka consumer configuration.
 config = {
     'bootstrap.servers': 'localhost:9092',  # replace with your Kafka broker(s)
-    'group.id': "w4",  # replace with your consumer group
-    'auto.offset.reset': 'earliest'  # can be set to 'earliest' or 'latest'
+    'group.id': f"view_asset_events:{current_time}",  # replace with your consumer group
+    'auto.offset.reset': earliest_latest  # can be set to 'earliest' or 'latest'
 }
 
 # Initialize a Kafka consumer.
@@ -54,14 +57,6 @@
 
             type_name = event["elementHeader"]["type"]["typeName"]
             origin = event["elementHeader"]["origin"]["sourceServer"]
-            # classifications = event['elementHeader']["classifications"]
-            # classification_md = ""
-            # for c in classifications:
-            #     cp = c.get("classificationProperties", None)
-            #     if cp is not None:
-            #         cl_name = c["classificationProperties"].get("name", None)
-            #         cl = cl_name if not None else c["classificationName"]
-            #         classification_md += f"* classifications: {cl}\n"
 
             element_properties = event["elementProperties"]
             element_properties_keys = element_properties.keys()
@@ -72,10 +67,10 @@
             console.rule(style= "[bold red]")
             console.rule(f"\tMessage TimeStamp: {event_time}\t eventType: {event_type}\t typeName: {type_name}\t guid: {guid}")
             msg = (
-                      # f"classifications: \n{classification_md}\n"
+
                       f"properties: \n{props}\n\n")
             msg = Markdown(msg)
-            # msg = json.dumps(event, indent=4)
+
             console.print(msg)
 finally:
     # Close down consumer to commit final offsets.

examples/widgets/operational/view_asset_events.py

@@ -17,7 +17,7 @@
 from rich.console import Console
 from rich.pretty import pprint
 
-from pyegeria import AutomatedCuration
+from pyegeria import AutomatedCuration, PostgreSQL_Server_Integration_Connector_GUID
 from pyegeria._exceptions import (InvalidParameterException, PropertyServerException, UserNotAuthorizedException,
                                   print_exception_response, )
 
@@ -834,11 +834,12 @@ def test_add_catalog_target(self):
             a_client = AutomatedCuration(self.good_view_server_1, self.good_platform1_url, user_id=self.good_user_2,
                                          user_pwd="secret")
             token = a_client.create_egeria_bearer_token()
-            element_guid = "061a4fb3-7d41-4e52-9080-65e9c61084d2"
-            catalog_target_name = "deltalake experiments"
-            file_folder_integ_con = "cd6479e1-2fe7-4426-b358-8a0cf70be117"
+            element_guid = "c155848f-60db-4265-aeb7-75cd7124806f"
+            catalog_target_name = "laz postgres server"
+
             start_time = time.perf_counter()
-            guid = a_client.add_catalog_target(file_folder_integ_con, element_guid, catalog_target_name)
+            guid = a_client.add_catalog_target(PostgreSQL_Server_Integration_Connector_GUID,
+                                               element_guid, catalog_target_name)
             duration = time.perf_counter() - start_time
             print(f"guid returned is: {guid}")
             print(f"\n\tDuration was {duration} seconds")

tests/test_automated_curation_omvs.py

@@ -179,7 +179,7 @@ def test_restart_integration_connector(self, server:str = good_server_2):
             print_exception_response(e)
             assert e.related_http_code != "200", "Invalid parameters"
 
-    def test_refresh_integration_connectors(self, server:str = good_server_2):
+    def test_refresh_integration_connectors(self, server:str = good_server_3):
         try:
             server_name = "integration-daemon"
             connector = "FilesMonitor"