diff --git a/core/setup.py b/core/setup.py index a13f71f20..38277067a 100644 --- a/core/setup.py +++ b/core/setup.py @@ -22,7 +22,8 @@ 'click>=7.1.2, <8.0', 'cryptography==3.3.2', 'pyyaml>=5.4.1, <6.0', - 'requests>=2.23.0, <3.0' + 'requests>=2.23.0, <3.0', + 'Deprecated>=1.2.13, <1.3' ] # TODO Fix the params # TODO Add a warning that installing core doesn't give any warehouse functionality diff --git a/core/sodasql/dataset_analyzer.py b/core/sodasql/dataset_analyzer.py index 47a1c637d..9d3fa8dc6 100644 --- a/core/sodasql/dataset_analyzer.py +++ b/core/sodasql/dataset_analyzer.py @@ -13,6 +13,7 @@ from sodasql.scan.validity import Validity from sodasql.scan.warehouse import Warehouse +from deprecated import deprecated @dataclass @@ -24,13 +25,17 @@ class ColumnAnalysisResult: values_count: int = None valid_count: int = None - def to_json(self): + def to_dict(self): return { 'columnName': self.column_name, 'sourceType': self.source_type, 'validityFormat': self.validity_format } + @deprecated(version='2.1.0b19', reason='This function is deprecated, please use to_dict') + def to_json(self): + return self.to_dict() + class DatasetAnalyzer: @@ -72,8 +77,8 @@ def analyze(self, warehouse: Warehouse, table_name: str): row = warehouse.sql_fetchone( f'SELECT \n ' + (',\n '.join(validity_format_count_fields)) + ',\n' - f' COUNT({qualified_column_name}) \n' - f'FROM ({select_with_limit_query}) T' + f' COUNT({qualified_column_name}) \n' + f'FROM ({select_with_limit_query}) T' ) values_count = row[len(validity_counts)] diff --git a/core/sodasql/scan/column_metadata.py b/core/sodasql/scan/column_metadata.py index 4df5d4778..c02ac4ebd 100644 --- a/core/sodasql/scan/column_metadata.py +++ b/core/sodasql/scan/column_metadata.py @@ -8,6 +8,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from deprecated import deprecated + class ColumnMetadata: @@ -20,7 +22,7 @@ def __init__(self, name: str, data_type: str = None, logical_type: str = None, n def __str__(self): return self.name + (' ' + self.type if self.type else '') - def to_json(self): + def to_dict(self): return { 'name': self.name, # TODO kept backward compatibility, remove after https://github.com/sodadata/soda/issues/2385 is fixed @@ -31,3 +33,7 @@ def to_json(self): # deprecated, use logicalType 'semanticType': self.logical_type, } + + @deprecated(version='2.1.0b19', reason='This function is deprecated, please use to_dict') + def to_json(self): + return self.to_dict() diff --git a/core/sodasql/scan/group_value.py b/core/sodasql/scan/group_value.py index a42bcdca8..2be733a39 100644 --- a/core/sodasql/scan/group_value.py +++ b/core/sodasql/scan/group_value.py @@ -13,6 +13,8 @@ from sodasql.common.json_helper import JsonHelper +from deprecated import deprecated + @dataclass class GroupValue: @@ -41,8 +43,12 @@ def from_json_list(cls, json_list: List): group_values.append(group_value) return group_values - def to_json(self): + def to_dict(self): return { 'group': JsonHelper.to_jsonnable(self.group), 'value': JsonHelper.to_jsonnable(self.value) } + + @deprecated(version='2.1.0b19', reason='This function is deprecated, please use to_dict') + def to_json(self): + return self.to_dict() diff --git a/core/sodasql/scan/measurement.py b/core/sodasql/scan/measurement.py index 251ec5fc7..066e50980 100644 --- a/core/sodasql/scan/measurement.py +++ b/core/sodasql/scan/measurement.py @@ -10,7 +10,7 @@ # limitations under the License. from dataclasses import dataclass from typing import Optional, List - +from deprecated import deprecated from sodasql.common.json_helper import JsonHelper from sodasql.scan.group_value import GroupValue @@ -35,17 +35,21 @@ def __str__(self): else: return f'{self.metric}{column_str} = {self.value}' - def to_json(self): - json = { + def to_dict(self) -> dict: + dictionary = { 'metric': self.metric, } if self.group_values is None: - json['value'] = JsonHelper.to_jsonnable(self.value) + dictionary['value'] = JsonHelper.to_jsonnable(self.value) else: - json['groupValues'] = [group_value.to_json() for group_value in self.group_values] + dictionary['groupValues'] = [group_value.to_dict() for group_value in self.group_values] if self.column_name is not None: - json['columnName'] = self.column_name + dictionary['columnName'] = self.column_name - return json + return dictionary + + @deprecated(version='2.1.0b19', reason='This function is deprecated, please use to_dict') + def to_json(self): + return self.to_dict() diff --git a/core/sodasql/scan/scan.py b/core/sodasql/scan/scan.py index 0c826d99c..52e0e571f 100644 --- a/core/sodasql/scan/scan.py +++ b/core/sodasql/scan/scan.py @@ -137,7 +137,7 @@ def _process_cloud_custom_metrics(self): monitor_metric = monitor_metric_parser.monitor_metric monitor_measurement = monitor_metric.execute() self.scan_result.measurements.append(monitor_measurement) - monitor_measurement_json = monitor_measurement.to_json() + monitor_measurement_json = monitor_measurement.to_dict() self.soda_server_client.scan_monitor_measurements(self.scan_reference, monitor_measurement_json) if self.disable_sample_collection: @@ -207,7 +207,7 @@ def _query_columns_metadata(self): logger.warning(f'Unknown or unsupported columns found in YML: {", ".join(invalid_column_names)} \n' f'Scan will continue for known/supported columns.') - schema_measurement_value = [column_metadata.to_json() for column_metadata in self.column_metadatas] + schema_measurement_value = [column_metadata.to_dict() for column_metadata in self.column_metadatas] schema_measurement = Measurement(Metric.SCHEMA, value=schema_measurement_value) self._log_measurement(schema_measurement) self._flush_measurements([schema_measurement]) @@ -861,7 +861,7 @@ def _flush_measurements(self, measurements: List[Measurement]): """ self.scan_result.measurements.extend(measurements) if self.soda_server_client and measurements: - measurement_jsons = [measurement.to_json() for measurement in measurements] + measurement_jsons = [measurement.to_dict() for measurement in measurements] try: self.soda_server_client.scan_measurements(self.scan_reference, measurement_jsons) except Exception as e: @@ -876,7 +876,7 @@ def _flush_test_results(self, test_results: List[TestResult]): self.scan_result.add_test_results(test_results) if self.soda_server_client: - test_result_jsons = [test_result.to_json() for test_result in test_results] + test_result_jsons = [test_result.to_dict() for test_result in test_results] try: self.soda_server_client.scan_test_results(self.scan_reference, test_result_jsons) except Exception as e: diff --git a/core/sodasql/scan/scan_error.py b/core/sodasql/scan/scan_error.py index cf3d1cea1..5d80abb4e 100644 --- a/core/sodasql/scan/scan_error.py +++ b/core/sodasql/scan/scan_error.py @@ -10,7 +10,7 @@ # limitations under the License. from dataclasses import dataclass - +from deprecated import deprecated from sodasql.scan.test import Test @@ -22,7 +22,7 @@ class ScanError: def __str__(self) -> str: return f'[{self.get_type()}] {self.get_message()}' - def to_json(self) -> dict: + def to_dict(self) -> dict: json = { 'type': self.get_type(), 'message': self.get_message() @@ -34,6 +34,10 @@ def to_json(self) -> dict: json['errorCode'] = self.exception.error_code return json + @deprecated(version='2.1.0b19', reason='This function is deprecated, please use to_dict') + def to_json(self): + return self.to_dict() + def get_type(self) -> str: return 'error' diff --git a/core/sodasql/scan/scan_result.py b/core/sodasql/scan/scan_result.py index 9f2137754..e184e45c7 100644 --- a/core/sodasql/scan/scan_result.py +++ b/core/sodasql/scan/scan_result.py @@ -9,6 +9,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from typing import List +from deprecated import deprecated from sodasql.scan.measurement import Measurement from sodasql.scan.scan_error import ScanError @@ -61,13 +62,17 @@ def get_errors_count(self) -> int: def is_passed(self) -> bool: return not (self.has_test_failures() or self.has_errors()) - def to_json(self) -> dict: + def to_dict(self) -> dict: return { - 'measurements': [measurement.to_json() for measurement in self.measurements], - 'testResults': [test_result.to_json() for test_result in self.test_results], - 'errors': [scan_error.to_json() for scan_error in self.errors] + 'measurements': [measurement.to_dict() for measurement in self.measurements], + 'testResults': [test_result.to_dict() for test_result in self.test_results], + 'errors': [scan_error.to_dict() for scan_error in self.errors] } + @deprecated(version='2.1.0b19', reason='This function is deprecated, please use to_dict') + def to_json(self): + return self.to_dict() + def find_measurement(self, metric_type: str, column_name: str = None) -> Measurement: """ Returns the measurement or None if the measurement does not exist diff --git a/core/sodasql/scan/test_result.py b/core/sodasql/scan/test_result.py index a8587e9d8..8b3c97f08 100644 --- a/core/sodasql/scan/test_result.py +++ b/core/sodasql/scan/test_result.py @@ -10,6 +10,7 @@ # limitations under the License. import json from dataclasses import dataclass +from deprecated import deprecated from typing import Optional from sodasql.common.json_helper import JsonHelper @@ -36,7 +37,7 @@ def __str__(self): (f" with group values {self.group_values}" if self.group_values else '') + f' with measurements {json.dumps(JsonHelper.to_jsonnable(self.values))}') - def to_json(self): + def to_dict(self) -> dict: if not self.test or not self.test.expression: return { 'error': 'Invalid test result' @@ -63,3 +64,7 @@ def to_json(self): test_result_json['groupValues'] = JsonHelper.to_jsonnable(self.group_values) return test_result_json + + @deprecated(version='2.1.0b19', reason='This function is deprecated, please use to_dict') + def to_json(self): + return self.to_dict() diff --git a/core/sodasql/soda_server_client/monitor_measurement.py b/core/sodasql/soda_server_client/monitor_measurement.py index f624dbf7e..d3c9a0a7b 100644 --- a/core/sodasql/soda_server_client/monitor_measurement.py +++ b/core/sodasql/soda_server_client/monitor_measurement.py @@ -10,7 +10,7 @@ # limitations under the License. from dataclasses import dataclass - +from deprecated import deprecated from sodasql.scan.group_value import GroupValue from sodasql.scan.measurement import Measurement @@ -22,20 +22,29 @@ class MonitorMeasurement(Measurement): query_milliseconds: int = None @classmethod - def from_json(cls, json: dict): - assert isinstance(json, dict) + def from_dict(cls, dictionary: dict) -> 'MonitorMeasurement': + assert isinstance(dictionary, dict) return MonitorMeasurement( - metric_id=json.get('metricId'), - metric=json.get('metricType'), - sql=json.get('sql'), - column_name=json.get('columnName'), - value=json.get('value'), - group_values=GroupValue.from_json_list(json.get('groupValues')), - query_milliseconds=json.get('queryMilliseconds')) + metric_id=dictionary.get('metricId'), + metric=dictionary.get('metricType'), + sql=dictionary.get('sql'), + column_name=dictionary.get('columnName'), + value=dictionary.get('value'), + group_values=GroupValue.from_json_list(dictionary.get('groupValues')), + query_milliseconds=dictionary.get('queryMilliseconds')) + + @classmethod + @deprecated(version='2.1.0b19', reason='This function is deprecated, please use to_dict') + def from_json(cls, dictionary: dict): + cls.from_dict(cls, dictionary) + + def to_dict(self) -> dict: + dictionary = super().to_dict() + dictionary['metricId'] = self.metric_id + dictionary['sql'] = self.sql + dictionary['queryMilliseconds'] = self.query_milliseconds + return dictionary + @deprecated(version='2.1.0b19', reason='This function is deprecated, please use to_dict') def to_json(self): - json = super().to_json() - json['metricId'] = self.metric_id - json['sql'] = self.sql - json['queryMilliseconds'] = self.query_milliseconds - return json + return self.to_dict() diff --git a/core/sodasql/soda_server_client/soda_server_client.py b/core/sodasql/soda_server_client/soda_server_client.py index 3fdfcae3e..b7333e00f 100644 --- a/core/sodasql/soda_server_client/soda_server_client.py +++ b/core/sodasql/soda_server_client/soda_server_client.py @@ -86,7 +86,7 @@ def scan_ended(self, scan_reference, errors: List[ScanError] = None): } if errors: logger.debug(f'Soda Cloud scan end with errors') - scan_end_command['errors'] = [error.to_json() for error in errors] + scan_end_command['errors'] = [error.to_dict() for error in errors] else: logger.debug(f'Soda Cloud scan end ok') self.execute_command(scan_end_command) diff --git a/examples/aws-lambda/lambda-zip/soda_lambda.py b/examples/aws-lambda/lambda-zip/soda_lambda.py index 91957b50e..301c16f1f 100644 --- a/examples/aws-lambda/lambda-zip/soda_lambda.py +++ b/examples/aws-lambda/lambda-zip/soda_lambda.py @@ -28,4 +28,4 @@ def lambda_handler(event, context): scan_result = scan.execute() print("Finished: Soda Scan") - print(scan_result.to_json()) + print(scan_result.to_dict()) diff --git a/tests/local/independent/test_measurement_str.py b/tests/local/independent/test_measurement_str.py index d938308e6..7f898d551 100644 --- a/tests/local/independent/test_measurement_str.py +++ b/tests/local/independent/test_measurement_str.py @@ -22,43 +22,43 @@ class TestMeasurementStr(TestCase): def test_row_count(self): measurement = Measurement(metric=Metric.ROW_COUNT, value=5) self.assertEqual('row_count = 5', str(measurement)) - JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_json())) + JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_dict())) def test_column_measurement(self): measurement = Measurement(metric=Metric.MIN, column_name='AGE', value=3.4) self.assertEqual('min(AGE) = 3.4', str(measurement)) - JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_json())) + JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_dict())) def test_column_measurement_list_value(self): measurement = Measurement(metric=Metric.MINS, column_name='chars', value=['a', 'b']) self.assertEqual("mins(chars) = ['a', 'b']", str(measurement)) - JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_json())) + JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_dict())) def test_column_group_measurement(self): measurement = Measurement(metric=Metric.MIN, column_name='AGE', group_values=[GroupValue(group={'country': 'US'}, value=3.4)]) self.assertEqual("min(AGE): \n group{'country': 'US'} = 3.4", str(measurement)) - JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_json())) + JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_dict())) def test_column_group_measurement_decimal(self): measurement = Measurement(metric=Metric.MIN, column_name='AGE', group_values=[GroupValue(group={'country': 'US'}, value=decimal.Decimal(4.5))]) self.assertEqual("min(AGE): \n group{'country': 'US'} = 4.5", str(measurement)) - JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_json())) + JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_dict())) def test_column_group_measurement_decimal_value(self): measurement = Measurement(metric=Metric.MIN, column_name='AGE', group_values=[GroupValue(group={'country': 'US'}, value=decimal.Decimal(3))]) self.assertEqual("min(AGE): \n group{'country': 'US'} = 3", str(measurement)) - JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_json())) + JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_dict())) def test_column_group_measurement_empty_list(self): measurement = Measurement(metric=Metric.MIN, column_name='AGE', group_values=[]) self.assertEqual('min(AGE): no groups', str(measurement)) - JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_json())) + JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_dict())) diff --git a/tests/local/warehouse/scans/test_scan_result.py b/tests/local/warehouse/scans/test_scan_result.py index ed7d45aab..c1ef5aa91 100644 --- a/tests/local/warehouse/scans/test_scan_result.py +++ b/tests/local/warehouse/scans/test_scan_result.py @@ -37,9 +37,9 @@ def test_scan_result_with_test_error(self): scan_result = self.scan(scan_yml_dict) self.assertTrue(scan_result.has_test_failures()) self.assertEqual(len(scan_result.errors), 1) - error_json = scan_result.errors[0].to_json() - self.assertEqual(error_json['type'], 'test_execution_error') - self.assertIsNotNone(error_json['message']) + errors = scan_result.errors[0].to_dict() + self.assertEqual(errors['type'], 'test_execution_error') + self.assertIsNotNone(errors['message']) def test_scan_result_with_test_errors(self): scan_yml_dict = { @@ -55,10 +55,10 @@ def test_scan_result_with_test_errors(self): self.assertTrue(scan_result.has_test_failures()) self.assertEqual(len(scan_result.errors), 2) - error_json = scan_result.errors[0].to_json() - self.assertEqual(error_json['type'], 'test_execution_error') - self.assertIn('10 < error < 20', error_json['message']) + errors = scan_result.errors[0].to_dict() + self.assertEqual(errors['type'], 'test_execution_error') + self.assertIn('10 < error < 20', errors['message']) - error_json = scan_result.errors[1].to_json() - self.assertEqual(error_json['type'], 'test_execution_error') - self.assertIn('lskdfj(lkj)', error_json['message']) + errors = scan_result.errors[1].to_dict() + self.assertEqual(errors['type'], 'test_execution_error') + self.assertIn('lskdfj(lkj)', errors['message'])