fix(ml): resolve Finding and dict type mismatch in deduplication #259

backend/app/main.py

@@ -1532,8 +1532,13 @@ async def _run_repo_scan_task(
             except ValueError:
                 epsilon = 0.15
 
+            # CORRECT WAY:
             if not disable_dedup and SENTENCE_TRANSFORMERS_AVAILABLE:
-                findings = deduplicate(findings, epsilon)
+    # The assertion must be safely inside the block body
+              assert all(isinstance(f, Finding) for f in findings), \
+              f"Expected Finding objects, got {set(type(f).__name__ for f in findings)}"
+
+            findings = deduplicate(findings, epsilon)
 
             await _apply_fp_predictor(findings)
 

backend/app/ml/deduplicator.py

@@ -1,7 +1,7 @@
 from collections import defaultdict
-
+from typing import Any, Dict, List, Union
 from sklearn.cluster import DBSCAN
-
+from app.models import Finding
 
 def get_model():
     return None
@@ -17,7 +17,7 @@ def get_model():
 SENTENCE_TRANSFORMERS_AVAILABLE = _embed_findings is not None
 
 
-def embed_findings(findings):
+def embed_findings(findings: List[Union[Finding, Dict[str, Any]]]):
     """Wrapper so tests can patch deduplicator.embed_findings if needed."""
     if _embed_findings is None:
         raise RuntimeError(
@@ -28,12 +28,13 @@ def embed_findings(findings):
 
 
 def deduplicate(
-    findings: list[dict],
+    findings: List[Union[Finding, Dict[str, Any]]],
     epsilon: float = 0.15,
-) -> list[dict]:
+) -> List[Union[Finding, Dict[str, Any]]]:
     """
     Group similar findings using DBSCAN and return
     representative findings with duplicate metadata.
+    Handles both Pydantic Finding objects and raw dict inputs.
     """
     if not findings:
         return []
@@ -60,16 +61,10 @@ def deduplicate(
     for label, cluster_findings in clusters.items():
         if label == -1:
             for finding in cluster_findings:
-                # finding["duplicate_count"] = 0
-                # finding["related_files"] = []
                 results.append(finding)
             continue
 
         representative = cluster_findings[0]
-
-        # representative["duplicate_count"] = len(cluster_findings)
-        # representative["related_files"] = related_files
-
         results.append(representative)
 
     return results

backend/app/ml/embedder.py

@@ -1,32 +1,54 @@
+import logging
+from typing import Any, Dict, List, Union
 import numpy as np
 
+from app.models import Finding
+
+logger = logging.getLogger(__name__)
+
+MODEL_NAME = "all-MiniLM-L6-v2"
+
 try:
     from sentence_transformers import SentenceTransformer
 
-    MODEL = SentenceTransformer("all-MiniLM-L6-v2")
-except ImportError:
+    MODEL = SentenceTransformer(MODEL_NAME)
+except Exception:
     MODEL = None
+    logger.warning(
+        "Failed to load sentence-transformers model: %s",
+        MODEL_NAME,
+        exc_info=True,
+    )
+
+
+def _extract_text(finding: Union[Finding, Dict[str, Any]]) -> str:
+    """Safely extracts title and description from either a Pydantic Finding object or a raw dict."""
+    if isinstance(finding, dict):
+        title = finding.get("title", "")
+        description = finding.get("description", "")
+        return f"{title} {description}".strip()
+
+    title = getattr(finding, "title", "")
+    description = getattr(finding, "description", "")
+    return f"{title} {description}".strip()
 
 
-def embed_findings(findings: list[dict]) -> np.ndarray:
+def embed_findings(findings: List[Union[Finding, Dict[str, Any]]]) -> np.ndarray:
     """
     Convert findings into embeddings.
 
     Each finding is converted to:
-    "{rule_id} {message} {file_path}"
+    "{title} {description}"
 
     Returns:
         np.ndarray of shape (n, 384)
     """
     if MODEL is None:
         raise RuntimeError(
-            "sentence-transformers is not installed. "
+            "sentence-transformers is not installed or failed to initialize. "
             "Install it using: pip install sentence-transformers"
         )
 
-    texts = [
-        f"{getattr(finding, 'title', '')} {getattr(finding, 'description', '')}"
-        for finding in findings
-    ]
+    texts = [_extract_text(finding) for finding in findings]
 
     return MODEL.encode(texts, convert_to_numpy=True)