Merge branch 'main' into main

.all-contributorsrc

@@ -195,6 +195,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "sintezcs",
+      "name": "Alexey Minakov",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2989480?v=4",
+      "profile": "https://dataart.com",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

README.md

@@ -108,6 +108,9 @@ Thanks goes to these wonderful people! ([emoji key](https://allcontributors.org/
     <td align="center"><a href="https://github.com/sebastianvillarroel"><img src="https://avatars.githubusercontent.com/u/16538437?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Sebastián Villarroel</b></sub></a><br /><a href="https://github.com/sodadata/soda-sql/commits?author=sebastianvillarroel" title="Code">💻</a></td>
     <td align="center"><a href="https://github.com/IIBenII"><img src="https://avatars.githubusercontent.com/u/40873921?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Benjamin Berriot</b></sub></a><br /><a href="https://github.com/sodadata/soda-sql/commits?author=IIBenII" title="Code">💻</a></td>
   </tr>
+  <tr>
+    <td align="center"><a href="https://dataart.com"><img src="https://avatars.githubusercontent.com/u/2989480?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Alexey Minakov</b></sub></a><br /><a href="https://github.com/sodadata/soda-sql/commits?author=sintezcs" title="Code">💻</a></td>
+  </tr>
 </table>
 
 <!-- markdownlint-restore -->

core/sodasql/cli/ingest.py

@@ -165,7 +165,7 @@ def flush_test_results(
             warehouse_database_schema=table.schema,
             table_name=table.name,
             scan_yml_columns=None,
-            scan_time=dt.datetime.now().isoformat(),
+            scan_time=dt.datetime.utcnow().isoformat(),
             origin=os.environ.get("SODA_SCAN_ORIGIN", "external"),
         )
         soda_server_client.scan_test_results(

core/sodasql/scan/file_system.py

@@ -67,12 +67,9 @@ def file_read_as_str(self, path: str) -> str:
     def file_write_from_str(self, path: str, file_content_str):
         expanded_path = os.path.expanduser(path)
         path_path: Path = Path(expanded_path)
-        is_new = not path_path.exists()
         try:
             with open(path_path, 'w+', encoding='utf-8') as f:
                 f.write(file_content_str)
-            if is_new:
-                os.chmod(path, 0o666)
         except Exception as e:
             logger.debug(f"Couldn't write {str(path)}: {str(e)}")
 

core/sodasql/telemetry/soda_exporter.py

@@ -0,0 +1,56 @@
+import logging
+from typing import Dict, Sequence, Optional
+
+from opentelemetry.sdk.trace.export import (
+    ConsoleSpanExporter,
+    SpanExportResult,
+)
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.exporter.otlp.proto.http import Compression
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_soda_spans(spans: Sequence[ReadableSpan]) -> Sequence[ReadableSpan]:
+    result = []
+    for span in spans:
+        if span.name.startswith("soda"):
+            result.append(span)
+        else:
+            logger.debug(f"Open Telemetry: Skipping non-soda span '{span.name}'.")
+
+    return result
+
+
+class SodaConsoleSpanExporter(ConsoleSpanExporter):
+    """Soda version of console exporter.
+
+    Does not export any non-soda spans for security and privacy reasons."""
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        return super().export(get_soda_spans(spans))
+
+
+class SodaOTLPSpanExporter(OTLPSpanExporter):
+    """Soda version of OTLP exporter.
+
+    Does not export any non-soda spans for security and privacy reasons."""
+    def __init__(
+        self,
+        endpoint: Optional[str] = None,
+        certificate_file: Optional[str] = None,
+        headers: Optional[Dict[str, str]] = None,
+        timeout: Optional[int] = None,
+        compression: Optional[Compression] = None,
+    ):
+        super().__init__(
+            endpoint,
+            certificate_file,
+            headers,
+            timeout,
+            compression,
+        )
+
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        return super().export(get_soda_spans(spans))

core/sodasql/telemetry/soda_telemetry.py

@@ -8,15 +8,14 @@
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import (
     BatchSpanProcessor,
-    ConsoleSpanExporter,
     SimpleSpanProcessor,
 )
-from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
 
 from sodasql.__version__ import SODA_SQL_VERSION
 from sodasql.common.config_helper import ConfigHelper
 from sodasql.scan.dialect import Dialect
 from sodasql.telemetry.memory_span_exporter import MemorySpanExporter
+from sodasql.telemetry.soda_exporter import SodaConsoleSpanExporter, SodaOTLPSpanExporter
 
 logger = logging.getLogger(__name__)
 
@@ -91,10 +90,10 @@ def __setup(self):
         local_debug_mode = self.soda_config.get_value('tracing_local_debug_mode')
 
         if local_debug_mode or logger.getEffectiveLevel() == logging.DEBUG:
-            self.__provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
+            self.__provider.add_span_processor(BatchSpanProcessor(SodaConsoleSpanExporter()))
 
         if not local_debug_mode:
-            otlp_exporter = OTLPSpanExporter(endpoint=self.ENDPOINT)
+            otlp_exporter = SodaOTLPSpanExporter(endpoint=self.ENDPOINT)
             otlp_processor = BatchSpanProcessor(otlp_exporter)
             self.__provider.add_span_processor(otlp_processor)
 

packages/bigquery/sodasql/dialects/bigquery_dialect.py

@@ -137,7 +137,7 @@ def sql_expr_regexp_like(self, expr: str, pattern: str):
         return f"REGEXP_CONTAINS({expr}, r'{self.qualify_regex(pattern)}')"
 
     @staticmethod
-    def __parse_json_credential(credential_name, parser):
+    def __parse_json_credential(credential_name, parser: Parser):
         account_info_path = parser.get_str_optional('account_info_json_path')
         try:
             if account_info_path:

packages/sqlserver/sodasql/dialects/sqlserver_dialect.py

@@ -52,6 +52,8 @@ def __init__(self, parser: Parser = None, type: str = SQLSERVER):
             self.database = parser.get_str_required_env('database')
             self.schema = parser.get_str_required_env('schema')
             self.trusted_connection = parser.get_bool_optional('trusted_connection', False)
+            self.encrypt = parser.get_bool_optional('encrypt', False)
+            self.trust_server_certificate = parser.get_bool_optional('trust_server_certificate', False)
 
     def default_connection_properties(self, params: dict):
         return {
@@ -99,6 +101,8 @@ def create_connection(self):
         try:
             conn = pyodbc.connect(
                 ('Trusted_Connection=YES;' if self.trusted_connection else '') +
+                ('TrustServerCertificate=YES;' if self.trust_server_certificate else '') +
+                ('Encrypt=YES;' if self.encrypt else '') +
                 'DRIVER={' + self.driver +
                 '};SERVER=' + self.host +
                 ';PORT=' + self.port +

tests/local/independent/test_telemetry.py

@@ -5,10 +5,12 @@
 from sodasql.telemetry.soda_telemetry import SodaTelemetry
 from sodasql.telemetry.soda_tracer import soda_trace
 from sodasql.telemetry.memory_span_exporter import MemorySpanExporter
+from sodasql.telemetry.soda_exporter import get_soda_spans
+from tests.common.telemetry_helper import telemetry_ensure_no_secrets
 
 from sodasql.__version__ import SODA_SQL_VERSION
 
-from tests.common.telemetry_helper import telemetry_ensure_no_secrets
+from opentelemetry.sdk.trace import ReadableSpan
 
 soda_telemetry = SodaTelemetry.get_instance()
 telemetry_exporter = MemorySpanExporter.get_instance()
@@ -114,3 +116,14 @@ def mock_fn():
 
     assert "Forbidden telemetry" in error_msg
     assert key in error_msg or value in error_msg
+
+def test_non_soda_span_filtering():
+    spans_input = [
+        ReadableSpan(name="test_1"),
+        ReadableSpan(name="soda_test_span"),
+        ReadableSpan(name="test_2"),
+    ]
+
+    spans_output = get_soda_spans(spans_input)
+    assert len(spans_output) == 1
+    assert spans_output[0].name == "soda_test_span"