Open Telemetry: Do not export non-soda spans in console and OTLP expo…

…rters. Fixes #627 (sodadata#632)

core/sodasql/telemetry/soda_exporter.py

@@ -0,0 +1,43 @@
+import logging
+from typing import Dict, List, Sequence
+
+from opentelemetry.sdk.trace.export import (
+    ConsoleSpanExporter,
+    SpanExportResult,
+    SpanExporter,
+)
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_soda_spans(spans: Sequence[ReadableSpan]) -> Sequence[ReadableSpan]:
+    result = []
+    for span in spans:
+        if span.name.startswith("soda"):
+            result.append(span)
+        else:
+            logger.debug(f"Open Telemetry: Skipping non-soda span '{span.name}'.")
+
+    return result
+
+
+class SodaConsoleSpanExporter(ConsoleSpanExporter):
+    """Soda version of console exporter.
+
+    Does not export any non-soda spans for security and privacy reasons."""
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        return super().export(get_soda_spans(spans))
+
+
+class SodaOTLPSpanExporter(OTLPSpanExporter):
+    """Soda version of OTLP exporter.
+
+    Does not export any non-soda spans for security and privacy reasons."""
+    def __init__(*args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        return super().export(get_soda_spans(spans))

core/sodasql/telemetry/soda_telemetry.py

@@ -8,15 +8,14 @@
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import (
     BatchSpanProcessor,
-    ConsoleSpanExporter,
     SimpleSpanProcessor,
 )
-from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
 
 from sodasql.__version__ import SODA_SQL_VERSION
 from sodasql.common.config_helper import ConfigHelper
 from sodasql.scan.dialect import Dialect
 from sodasql.telemetry.memory_span_exporter import MemorySpanExporter
+from sodasql.telemetry.soda_exporter import SodaConsoleSpanExporter, SodaOTLPSpanExporter
 
 logger = logging.getLogger(__name__)
 
@@ -91,10 +90,10 @@ def __setup(self):
         local_debug_mode = self.soda_config.get_value('tracing_local_debug_mode')
 
         if local_debug_mode or logger.getEffectiveLevel() == logging.DEBUG:
-            self.__provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
+            self.__provider.add_span_processor(BatchSpanProcessor(SodaConsoleSpanExporter()))
 
         if not local_debug_mode:
-            otlp_exporter = OTLPSpanExporter(endpoint=self.ENDPOINT)
+            otlp_exporter = SodaOTLPSpanExporter(endpoint=self.ENDPOINT)
             otlp_processor = BatchSpanProcessor(otlp_exporter)
             self.__provider.add_span_processor(otlp_processor)
 

packages/bigquery/sodasql/dialects/bigquery_dialect.py

@@ -137,7 +137,7 @@ def sql_expr_regexp_like(self, expr: str, pattern: str):
         return f"REGEXP_CONTAINS({expr}, r'{self.qualify_regex(pattern)}')"
 
     @staticmethod
-    def __parse_json_credential(credential_name, parser):
+    def __parse_json_credential(credential_name, parser: Parser):
         account_info_path = parser.get_str_optional('account_info_json_path')
         try:
             if account_info_path:

tests/local/independent/test_telemetry.py

@@ -5,10 +5,12 @@
 from sodasql.telemetry.soda_telemetry import SodaTelemetry
 from sodasql.telemetry.soda_tracer import soda_trace
 from sodasql.telemetry.memory_span_exporter import MemorySpanExporter
+from sodasql.telemetry.soda_exporter import get_soda_spans
+from tests.common.telemetry_helper import telemetry_ensure_no_secrets
 
 from sodasql.__version__ import SODA_SQL_VERSION
 
-from tests.common.telemetry_helper import telemetry_ensure_no_secrets
+from opentelemetry.sdk.trace import ReadableSpan
 
 soda_telemetry = SodaTelemetry.get_instance()
 telemetry_exporter = MemorySpanExporter.get_instance()
@@ -114,3 +116,14 @@ def mock_fn():
 
     assert "Forbidden telemetry" in error_msg
     assert key in error_msg or value in error_msg
+
+def test_non_soda_span_filtering():
+    spans_input = [
+        ReadableSpan(name="test_1"),
+        ReadableSpan(name="soda_test_span"),
+        ReadableSpan(name="test_2"),
+    ]
+
+    spans_output = get_soda_spans(spans_input)
+    assert len(spans_output) == 1
+    assert spans_output[0].name == "soda_test_span"