From 5b58cccd0c439deb05dfea92cbbacbdaa78f0f43 Mon Sep 17 00:00:00 2001 From: prasiddhi-105 Date: Mon, 6 Jul 2026 09:45:26 +0530 Subject: [PATCH 1/3] fix: resolve type hint mismatch in deduplicator and broaden embedder exceptions (#259) --- backend/app/main.py | 7 ++++++- backend/app/ml/deduplicator.py | 10 +++++----- backend/app/ml/embedder.py | 27 ++++++++++++++++++--------- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/backend/app/main.py b/backend/app/main.py index 66168de..63ff51d 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1402,7 +1402,12 @@ async def _run_repo_scan_task( epsilon = 0.15 if not disable_dedup and SENTENCE_TRANSFORMERS_AVAILABLE: - findings = deduplicate(findings, epsilon) + # Runtime safety verification assertion guard + assert all(isinstance(f, Finding) for f in findings), \ + + f"Expected Finding objects, got {set(type(f).__name__ for f in findings)}" + + findings = deduplicate(findings, epsilon) await _apply_fp_predictor(findings) diff --git a/backend/app/ml/deduplicator.py b/backend/app/ml/deduplicator.py index fae1036..eed1d63 100644 --- a/backend/app/ml/deduplicator.py +++ b/backend/app/ml/deduplicator.py @@ -1,7 +1,7 @@ from collections import defaultdict - +from typing import Union, List, Dict, Any from sklearn.cluster import DBSCAN - +from app.models import Finding def get_model(): return None @@ -28,9 +28,9 @@ def embed_findings(findings): def deduplicate( - findings: list[dict], + findings: List[Union[Finding, Dict[str, Any]]], epsilon: float = 0.15, -) -> list[dict]: +) -> List[Union[Finding, Dict[str, Any]]]: """ Group similar findings using DBSCAN and return representative findings with duplicate metadata. @@ -72,4 +72,4 @@ def deduplicate( results.append(representative) - return results + return results \ No newline at end of file diff --git a/backend/app/ml/embedder.py b/backend/app/ml/embedder.py index f05ad71..70a071e 100644 --- a/backend/app/ml/embedder.py +++ b/backend/app/ml/embedder.py @@ -1,32 +1,41 @@ +import logging import numpy as np try: from sentence_transformers import SentenceTransformer MODEL = SentenceTransformer("all-MiniLM-L6-v2") -except ImportError: +except Exception: MODEL = None + logging.getLogger(__name__).warning( + "Failed to load sentence-transformers model: all-MiniLM-L6-v2", + exc_info=True + ) -def embed_findings(findings: list[dict]) -> np.ndarray: +def _extract_text(finding) -> str: + """Safely extracts title and description from either a Pydantic Finding object or a raw dict.""" + if isinstance(finding, dict): + return f"{finding.get('title', '')} {finding.get('description', '')}".strip() + return f"{getattr(finding, 'title', '')} {getattr(finding, 'description', '')}".strip() + + +def embed_findings(findings: list) -> np.ndarray: """ Convert findings into embeddings. Each finding is converted to: - "{rule_id} {message} {file_path}" + "{title} {description}" Returns: - np.ndarray of shape (n, 384) + np.ndarray of shape (n, 384) """ if MODEL is None: raise RuntimeError( - "sentence-transformers is not installed. " + "sentence-transformers is not installed or failed to initialize. " "Install it using: pip install sentence-transformers" ) - texts = [ - f"{getattr(finding, 'title', '')} {getattr(finding, 'description', '')}" - for finding in findings - ] + texts = [_extract_text(finding) for finding in findings] return MODEL.encode(texts, convert_to_numpy=True) From aa3466abf514a600612ce5776bc13902f769bc7e Mon Sep 17 00:00:00 2001 From: prasiddhi-105 Date: Mon, 6 Jul 2026 09:50:53 +0530 Subject: [PATCH 2/3] fix: resolve type hint mismatch in deduplicator and broaden embedder exceptions (#259) --- backend/app/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/app/main.py b/backend/app/main.py index 63ff51d..166b06f 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1401,13 +1401,13 @@ async def _run_repo_scan_task( except ValueError: epsilon = 0.15 + # CORRECT WAY: if not disable_dedup and SENTENCE_TRANSFORMERS_AVAILABLE: - # Runtime safety verification assertion guard - assert all(isinstance(f, Finding) for f in findings), \ - - f"Expected Finding objects, got {set(type(f).__name__ for f in findings)}" + # The assertion must be safely inside the block body + assert all(isinstance(f, Finding) for f in findings), \ + f"Expected Finding objects, got {set(type(f).__name__ for f in findings)}" - findings = deduplicate(findings, epsilon) + findings = deduplicate(findings, epsilon) await _apply_fp_predictor(findings) From 90d7f6833ce22278faf25620c98f019aabc31048 Mon Sep 17 00:00:00 2001 From: prasiddhi-105 Date: Fri, 31 Jul 2026 18:56:49 +0530 Subject: [PATCH 3/3] fix(ml): resolve Finding/dict type mismatch in deduplication (#259) --- backend/app/ml/deduplicator.py | 13 ++++--------- backend/app/ml/embedder.py | 31 ++++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/backend/app/ml/deduplicator.py b/backend/app/ml/deduplicator.py index eed1d63..aa05636 100644 --- a/backend/app/ml/deduplicator.py +++ b/backend/app/ml/deduplicator.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import Union, List, Dict, Any +from typing import Any, Dict, List, Union from sklearn.cluster import DBSCAN from app.models import Finding @@ -17,7 +17,7 @@ def get_model(): SENTENCE_TRANSFORMERS_AVAILABLE = _embed_findings is not None -def embed_findings(findings): +def embed_findings(findings: List[Union[Finding, Dict[str, Any]]]): """Wrapper so tests can patch deduplicator.embed_findings if needed.""" if _embed_findings is None: raise RuntimeError( @@ -34,6 +34,7 @@ def deduplicate( """ Group similar findings using DBSCAN and return representative findings with duplicate metadata. + Handles both Pydantic Finding objects and raw dict inputs. """ if not findings: return [] @@ -60,16 +61,10 @@ def deduplicate( for label, cluster_findings in clusters.items(): if label == -1: for finding in cluster_findings: - # finding["duplicate_count"] = 0 - # finding["related_files"] = [] results.append(finding) continue representative = cluster_findings[0] - - # representative["duplicate_count"] = len(cluster_findings) - # representative["related_files"] = related_files - results.append(representative) - return results \ No newline at end of file + return results diff --git a/backend/app/ml/embedder.py b/backend/app/ml/embedder.py index 70a071e..7969f5e 100644 --- a/backend/app/ml/embedder.py +++ b/backend/app/ml/embedder.py @@ -1,26 +1,39 @@ import logging +from typing import Any, Dict, List, Union import numpy as np +from app.models import Finding + +logger = logging.getLogger(__name__) + +MODEL_NAME = "all-MiniLM-L6-v2" + try: from sentence_transformers import SentenceTransformer - MODEL = SentenceTransformer("all-MiniLM-L6-v2") + MODEL = SentenceTransformer(MODEL_NAME) except Exception: MODEL = None - logging.getLogger(__name__).warning( - "Failed to load sentence-transformers model: all-MiniLM-L6-v2", - exc_info=True + logger.warning( + "Failed to load sentence-transformers model: %s", + MODEL_NAME, + exc_info=True, ) -def _extract_text(finding) -> str: +def _extract_text(finding: Union[Finding, Dict[str, Any]]) -> str: """Safely extracts title and description from either a Pydantic Finding object or a raw dict.""" if isinstance(finding, dict): - return f"{finding.get('title', '')} {finding.get('description', '')}".strip() - return f"{getattr(finding, 'title', '')} {getattr(finding, 'description', '')}".strip() + title = finding.get("title", "") + description = finding.get("description", "") + return f"{title} {description}".strip() + + title = getattr(finding, "title", "") + description = getattr(finding, "description", "") + return f"{title} {description}".strip() -def embed_findings(findings: list) -> np.ndarray: +def embed_findings(findings: List[Union[Finding, Dict[str, Any]]]) -> np.ndarray: """ Convert findings into embeddings. @@ -28,7 +41,7 @@ def embed_findings(findings: list) -> np.ndarray: "{title} {description}" Returns: - np.ndarray of shape (n, 384) + np.ndarray of shape (n, 384) """ if MODEL is None: raise RuntimeError(