Add database name and schema to cloud scanStart command (sodadata#584)

core/sodasql/scan/dialect.py

@@ -115,6 +115,16 @@ def default_connection_properties(self, params: dict):
         # to be overridden by subclass
         pass
 
+    def get_warehouse_name_and_schema(self) -> dict:
+        """
+        This is a workaround to send the identifiers to soda cloud for integration with external
+        systems (e.g. metaphor), this will change in the future. The implementation should return a dict with
+        two keys 'database_name' and 'database_schema'
+
+        :return: dict
+        """
+        pass
+
     def safe_connection_data(self):
         """Return non-critically sensitive connection details.
 

core/sodasql/soda_server_client/soda_server_client.py

@@ -69,10 +69,13 @@ def scan_start(self, warehouse, scan_yml: ScanYml, scan_time,
                     }
                 soda_column_cfgs[column_name] = soda_column_cfg
 
+        database_and_schema = warehouse.dialect.get_warehouse_name_and_schema()
         return self.execute_command({
             'type': 'sodaSqlScanStart',
             'warehouseName': warehouse.name,
             'warehouseType': warehouse.dialect.type,
+            'warehouseDatabaseName': database_and_schema.get('database_name'),
+            'warehouseDatabaseSchema': database_and_schema.get('database_schema'),
             'tableName': scan_yml.table_name,
             'scanTime': scan_time,
             'columns': soda_column_cfgs,
@@ -172,7 +175,7 @@ def scan_monitor_measurements(self, scan_reference: dict, monitor_measurement_js
             'monitorMeasurement': monitor_measurement_json
         })
 
-    def historic_metrics(self, warehouse, table_name,  metrics):
+    def historic_metrics(self, warehouse, table_name, metrics):
         return self.execute_query({
             'type': 'sodaSqlHistoricMeasurements',
             'warehouseName': warehouse.name,

packages/athena/sodasql/dialects/athena_dialect.py

@@ -56,6 +56,12 @@ def default_connection_properties(self, params: dict):
             'catalog': 'AwsDataCatalog'
         }
 
+    def get_warehouse_name_and_schema(self) -> dict:
+        return {
+            'database_name': self.database,
+            'database_schema': self.catalog
+        }
+
     def safe_connection_data(self):
         return [
             self.type,

packages/bigquery/sodasql/dialects/bigquery_dialect.py

@@ -50,6 +50,12 @@ def default_connection_properties(self, params: dict):
             'dataset': params.get('database', 'Eg your_bigquery_dataset')
         }
 
+    def get_warehouse_name_and_schema(self) -> dict:
+        return {
+            'database_name': self.account_info_dict['project_id'],
+            'database_schema': self.dataset_name
+        }
+
     def safe_connection_data(self):
         return [
             self.account_info_dict['project_id']

packages/hive/sodasql/dialects/hive_dialect.py

@@ -47,6 +47,13 @@ def default_connection_properties(self, params: dict):
             'database': params.get('database', 'your_database')
         }
 
+    def get_warehouse_name_and_schema(self) -> dict:
+        # In Hive database and schema are interchangeable
+        return {
+            'database_name': self.database,
+            'database_schema': self.database
+        }
+
     def safe_connection_data(self):
         return [
             self.type,

packages/mysql/sodasql/dialects/mysql_dialect.py

@@ -41,6 +41,13 @@ def default_connection_properties(self, params: dict):
             'database': params.get('database', 'your_database')
         }
 
+    def get_warehouse_name_and_schema(self) -> dict:
+        # In MySQL schema is synonymous with database
+        return {
+            'database_name': self.database,
+            'database_schema': self.database
+        }
+
     def safe_connection_data(self):
         return [
             self.type,

packages/postgresql/sodasql/dialects/postgres_dialect.py

@@ -45,6 +45,12 @@ def default_connection_properties(self, params: dict):
             'schema': 'public'
         }
 
+    def get_warehouse_name_and_schema(self) -> dict:
+        return {
+            'database_name': self.database,
+            'database_schema': self.schema
+        }
+
     def safe_connection_data(self):
         return [
             self.type,

packages/snowflake/sodasql/dialects/snowflake_dialect.py

@@ -56,6 +56,12 @@ def default_connection_properties(self, params: dict):
             'schema': 'PUBLIC'
         }
 
+    def get_warehouse_name_and_schema(self) -> dict:
+        return {
+            'database_name': self.database,
+            'database_schema': self.schema
+        }
+
     def safe_connection_data(self):
         return [
             self.type,

packages/spark/sodasql/dialects/spark_dialect.py

@@ -169,6 +169,12 @@ def default_connection_properties(self, params: dict):
             'database': params.get('database', 'your_database')
         }
 
+    def get_warehouse_name_and_schema(self) -> dict:
+        return {
+            'database_name': self.database,
+            'database_schema': self.database
+        }
+
     def safe_connection_data(self):
         return [
             self.type,

packages/sqlserver/sodasql/dialects/sqlserver_dialect.py

@@ -66,6 +66,12 @@ def default_connection_properties(self, params: dict):
             'schema': 'public'
         }
 
+    def get_warehouse_name_and_schema(self) -> dict:
+        return {
+            'database_name': self.database,
+            'database_schema': self.schema
+        }
+
     def safe_connection_data(self):
         return [
             self.type,

pytest.ini

@@ -4,5 +4,5 @@ python_files = *_suite.py test_*.py
 python_classes = Suite Test
 log_level = DEBUG
 addopts = -v
-norecursedirs = examples/*
+norecursedirs = examples/* build/*