chore: add log levels and e2e testing

bizon/alerting/alerts.py

@@ -1,30 +1,23 @@
 from abc import ABC, abstractmethod
-from enum import Enum
 from typing import Dict, List
 
 from loguru import logger
 
-from bizon.alerting.models import AlertingConfig, AlertMethod
-
-
-class LogLevel(str, Enum):
-    DEBUG = "DEBUG"
-    INFO = "INFO"
-    WARNING = "WARNING"
-    ERROR = "ERROR"
-    CRITICAL = "CRITICAL"
+from bizon.alerting.models import AlertingConfig, AlertMethod, LogLevel
 
 
 class AbstractAlert(ABC):
 
-    def __init__(self, type: AlertMethod, config: AlertingConfig):
+    def __init__(self, type: AlertMethod, config: AlertingConfig, log_levels: List[LogLevel] = [LogLevel.ERROR]):
         self.type = type
         self.config = config
+        self.log_levels = log_levels
 
     @abstractmethod
     def handler(self, message: Dict) -> None:
         pass
 
-    def add_handlers(self, levels: List[LogLevel] = [LogLevel.ERROR]) -> None:
+    def add_handlers(self) -> None:
+        levels = [level.value for level in self.log_levels]
         for level in levels:
             logger.add(self.handler, level=level, format="{message}")

bizon/alerting/models.py

@@ -1,11 +1,19 @@
 from enum import Enum
-from typing import Union
+from typing import List, Optional, Union
 
 from pydantic import BaseModel
 
 from bizon.alerting.slack.config import SlackConfig
 
 
+class LogLevel(str, Enum):
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+
 class AlertMethod(str, Enum):
     """Alerting methods"""
 
@@ -16,4 +24,5 @@ class AlertingConfig(BaseModel):
     """Alerting configuration model"""
 
     type: AlertMethod
+    log_levels: Optional[List[LogLevel]] = [LogLevel.ERROR]
     config: Union[SlackConfig]

bizon/alerting/slack/handler.py

@@ -1,16 +1,17 @@
 import os
-from typing import Dict
+from typing import Dict, List
 
 import requests
 from loguru import logger
 
 from bizon.alerting.alerts import AbstractAlert, AlertMethod
+from bizon.alerting.models import LogLevel
 from bizon.alerting.slack.config import SlackConfig
 
 
 class SlackHandler(AbstractAlert):
-    def __init__(self, config: SlackConfig):
-        super().__init__(type=AlertMethod.SLACK, config=config)
+    def __init__(self, config: SlackConfig, log_levels: List[LogLevel] = [LogLevel.ERROR]):
+        super().__init__(type=AlertMethod.SLACK, config=config, log_levels=log_levels)
         self.webhook_url = config.webhook_url
 
     def handler(self, message: Dict) -> None:

bizon/engine/runner/runner.py

@@ -8,7 +8,6 @@
 
 from loguru import logger
 
-from bizon.alerting.alerts import LogLevel
 from bizon.alerting.models import AlertMethod
 from bizon.cli.utils import parse_from_yaml
 from bizon.common.models import BizonConfig, SyncMetadata
@@ -49,8 +48,11 @@ def __init__(self, config: dict):
             if self.bizon_config.alerting.type == AlertMethod.SLACK:
                 from bizon.alerting.slack.handler import SlackHandler
 
-                alert = SlackHandler(webhook_url=self.bizon_config.alerting.config.webhook_url)
-            alert.add_handlers(levels=[LogLevel.ERROR, LogLevel.WARNING])
+                alert = SlackHandler(
+                    config=self.bizon_config.alerting.config,
+                    log_levels=self.bizon_config.alerting.log_levels,
+                )
+            alert.add_handlers()
 
     @property
     def is_running(self) -> bool:

tests/alerts/test_slack_alert.py

@@ -1,12 +1,17 @@
+import json
+import os
+import tempfile
 import threading
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from queue import Queue
 
 import pytest
+import yaml
 from loguru import logger
 
 from bizon.alerting.slack.config import SlackConfig
 from bizon.alerting.slack.handler import SlackHandler
+from bizon.engine.engine import RunnerFactory
 
 
 class DummyWebhookHandler(BaseHTTPRequestHandler):
@@ -49,8 +54,13 @@ def dummy_webhook_server():
     DummyWebhookHandler.payload_queue.queue.clear()
 
 
-def test_slack_log_handler(dummy_webhook_server):
-    slack_handler = SlackHandler(SlackConfig(webhook_url="http://localhost:8123"))
+@pytest.fixture
+def webhook_url():
+    return "http://localhost:8123"
+
+
+def test_slack_log_handler(dummy_webhook_server, webhook_url):
+    slack_handler = SlackHandler(SlackConfig(webhook_url=webhook_url))
 
     slack_handler.add_handlers(levels=["ERROR", "WARNING"])
 
@@ -67,3 +77,60 @@ def test_slack_log_handler(dummy_webhook_server):
     logger.warning(WARNING_MESSAGE)
     warning_payload = DummyWebhookHandler.payload_queue.get(timeout=1)
     assert WARNING_MESSAGE in warning_payload
+
+
+def test_e2e_logger_to_file(dummy_webhook_server, webhook_url):
+
+    with tempfile.NamedTemporaryFile(delete=False) as temp:
+
+        BIZON_CONFIG_DUMMY_TO_FILE = f"""
+        name: test_job_3
+
+        source:
+          name: dummy
+          stream: creatures
+          authentication:
+            type: api_key
+            params:
+              token: dummy_key
+
+        destination:
+          name: file
+          config:
+            filepath: {temp.name}
+
+        transforms:
+          - label: transform_data
+            python: |
+              if 'name' in data:
+                data['name'] = fake_variable # this is purposely wrong to trigger an error
+
+        engine:
+          backend:
+            type: postgres
+            config:
+              database: bizon_test
+              schema: public
+              syncCursorInDBEvery: 2
+              host: {os.environ.get("POSTGRES_HOST", "localhost")}
+              port: 5432
+              username: postgres
+              password: bizon
+
+        alerting:
+            type: slack
+
+            config:
+              webhook_url: {webhook_url}
+
+            log_levels:
+                - ERROR
+        """
+
+        runner = RunnerFactory.create_from_config_dict(yaml.safe_load(BIZON_CONFIG_DUMMY_TO_FILE))
+
+        runner.run()
+
+        error_payload = DummyWebhookHandler.payload_queue.get(timeout=1)
+        assert "Error applying transformation" in error_payload
+        assert "fake_variable" in error_payload