Merge branch 'main' into fix/tls-verification-disabled

Author: namann5

.github/workflows/ci.yml

@@ -41,9 +41,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Check formatting hygiene on changed files
-        run: |
-          git fetch origin "${{ github.base_ref }}" --depth=1
-          git diff --check "origin/${{ github.base_ref }}"...HEAD
+        run: git diff --check "${{ github.event.pull_request.base.sha }}"...HEAD
 
   issue-template-label-validation:
     runs-on: ubuntu-latest

backend/secuscan/config.py

@@ -33,6 +33,7 @@ class Settings(BaseSettings):
     raw_output_dir: str = str(PROJECT_ROOT / "data" / "raw")
     reports_dir: str = str(PROJECT_ROOT / "data" / "reports")
     plugins_dir: str = str(PROJECT_ROOT.parent / "plugins")
+    disabled_plugins: List[str] = []
     wordlists_dir: str = str(PROJECT_ROOT / "wordlists")
     knowledgebase_dir: str = str(PROJECT_ROOT / "data" / "knowledgebase")
 

backend/secuscan/database.py

@@ -23,6 +23,7 @@ class Database:
     def __init__(self, db_path: str):
         self.db_path = db_path
         self._connection = None
+        self._in_transaction: bool = False
 
     @property
     def connection(self) -> aiosqlite.Connection:
@@ -380,7 +381,7 @@ async def _create_schema(self):
             CREATE INDEX IF NOT EXISTS idx_findings_owner ON findings(owner_id);
             CREATE INDEX IF NOT EXISTS idx_findings_cpe ON findings(cpe);
             CREATE INDEX IF NOT EXISTS idx_findings_validated ON findings(validated);
-            CREATE INDEX IF NOT EXISTS idx_findings_group_id ON findings(owner_id, finding_group_id);
+            CREATE UNIQUE INDEX IF NOT EXISTS idx_findings_group_id ON findings(owner_id, finding_group_id);
             CREATE INDEX IF NOT EXISTS idx_findings_asset_id ON findings(owner_id, asset_id);
 
             -- Reports indexes (new)
@@ -769,19 +770,27 @@ async def transaction(self) -> AsyncIterator["Database"]:
 
         If any statement raises, the entire transaction is rolled back.
         On success the transaction is committed automatically.
+
+        Nested calls are safe: when a transaction is already active the
+        inner context manager becomes a no-op so the outer transaction
+        controls the commit/rollback.
         """
-        await self.begin()
-        try:
+        if self._in_transaction:
             yield self
-            await self.commit()
-        except Exception:
-            await self.rollback()
-            raise
+        else:
+            await self.begin()
+            try:
+                yield self
+                await self.commit()
+            except Exception:
+                await self.rollback()
+                raise
 
     async def execute(self, query: str, params: tuple = ()):
         """Execute a write query and return the cursor (so callers can inspect rowcount)."""
         cursor = await self.connection.execute(query, params)
-        await self.connection.commit()
+        if not self._in_transaction:
+            await self.connection.commit()
         return cursor
 
     async def execute_no_commit(self, query: str, params: tuple = ()):
@@ -790,16 +799,25 @@ async def execute_no_commit(self, query: str, params: tuple = ()):
         return cursor
 
     async def begin(self):
-        """Begin a transaction."""
+        """Begin a transaction. No-op if already in a transaction."""
+        if self._in_transaction:
+            return
         await self.connection.execute("BEGIN")
+        self._in_transaction = True
 
     async def commit(self):
-        """Commit the current transaction."""
+        """Commit the current transaction. No-op if not in a transaction."""
+        if not self._in_transaction:
+            return
         await self.connection.commit()
+        self._in_transaction = False
 
     async def rollback(self):
-        """Roll back the current transaction."""
+        """Roll back the current transaction. No-op if not in a transaction."""
+        if not self._in_transaction:
+            return
         await self.connection.rollback()
+        self._in_transaction = False
 
     async def fetchone(self, query: str, params: tuple = ()) -> Optional[Dict]:
         """Fetch one row."""

backend/secuscan/executor.py

@@ -1349,6 +1349,7 @@ async def _persist_finding(
     ) -> Dict[str, Any]:
         u_id = str(uuid.uuid4()).replace("-", "")
         finding_id = f"finding:{task_id}:{u_id[:8]}"
+        finding_group_id = finding.get("finding_group_id")
 
         _validate_risk_fields(finding)
         exploitability = finding.get("exploitability")
@@ -1395,6 +1396,41 @@ async def _persist_finding(
                 asset_exposure, risk_score, risk_factors_json
             ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
                       ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT (owner_id, finding_group_id) DO UPDATE SET
+                task_id = EXCLUDED.task_id,
+                plugin_id = EXCLUDED.plugin_id,
+                title = EXCLUDED.title,
+                category = EXCLUDED.category,
+                severity = EXCLUDED.severity,
+                target = EXCLUDED.target,
+                description = EXCLUDED.description,
+                remediation = EXCLUDED.remediation,
+                proof = EXCLUDED.proof,
+                cvss = EXCLUDED.cvss,
+                cve = EXCLUDED.cve,
+                metadata_json = EXCLUDED.metadata_json,
+                discovered_at = EXCLUDED.discovered_at,
+                exploitability = EXCLUDED.exploitability,
+                confidence = EXCLUDED.confidence,
+                validated = EXCLUDED.validated,
+                validation_method = EXCLUDED.validation_method,
+                confidence_reason = EXCLUDED.confidence_reason,
+                finding_kind = EXCLUDED.finding_kind,
+                asset_id = EXCLUDED.asset_id,
+                last_seen_at = EXCLUDED.last_seen_at,
+                occurrence_count = COALESCE(findings.occurrence_count, 0) + EXCLUDED.occurrence_count,
+                corroborating_sources_json = EXCLUDED.corroborating_sources_json,
+                evidence_count = EXCLUDED.evidence_count,
+                analyst_status = EXCLUDED.analyst_status,
+                retest_status = EXCLUDED.retest_status,
+                evidence_json = EXCLUDED.evidence_json,
+                asset_refs_json = EXCLUDED.asset_refs_json,
+                service_fingerprint = EXCLUDED.service_fingerprint,
+                cpe = EXCLUDED.cpe,
+                references_json = EXCLUDED.references_json,
+                asset_exposure = EXCLUDED.asset_exposure,
+                risk_score = EXCLUDED.risk_score,
+                risk_factors_json = EXCLUDED.risk_factors_json
             """,
             (
                 finding_id,
@@ -1418,7 +1454,7 @@ async def _persist_finding(
                 finding.get("validation_method"),
                 finding.get("confidence_reason"),
                 str(finding.get("finding_kind") or "observation"),
-                finding.get("finding_group_id"),
+                finding_group_id,
                 finding.get("asset_id"),
                 first_seen_at,
                 last_seen_at,
@@ -1437,6 +1473,13 @@ async def _persist_finding(
                 json.dumps(risk_factors),
             ),
         )
+
+        row = await db.fetchone(
+            "SELECT id, occurrence_count FROM findings WHERE owner_id = ? AND finding_group_id = ?",
+            (owner_id, finding_group_id),
+        )
+        finding_id = row["id"] if row else finding_id
+        occurrence_count = int(row["occurrence_count"]) if row else occurrence_count
         return {
             **finding,
             "id": finding_id,
@@ -1468,25 +1511,25 @@ async def _upsert_findings_and_report(self, db, task_id: str, owner_id: str, plu
             result=parsed,
         )
         findings_data: List[Dict[str, Any]] = []
-        for finding in structured_result.get("findings", []):
-            findings_data.append(
-                await self._persist_finding(
-                    db,
-                    owner_id=owner_id,
-                    task_id=task_id,
-                    plugin_id=plugin_id,
-                    target=target,
-                    finding=finding,
+        async with db.transaction():
+            for finding in structured_result.get("findings", []):
+                findings_data.append(
+                    await self._persist_finding(
+                        db,
+                        owner_id=owner_id,
+                        task_id=task_id,
+                        plugin_id=plugin_id,
+                        target=target,
+                        finding=finding,
+                    )
                 )
-            )
 
-        structured_result["findings"] = findings_data
-        structured_result["severity_counts"] = self._build_severity_counts(findings_data)
-        structured_result["finding_groups"] = build_finding_groups(findings_data)
-        structured_result["asset_summary"] = build_asset_summary(findings_data, asset_services)
-        structured_result["scan_diff"] = build_scan_diff(findings_data, previous_findings)
+            structured_result["findings"] = findings_data
+            structured_result["severity_counts"] = self._build_severity_counts(findings_data)
+            structured_result["finding_groups"] = build_finding_groups(findings_data)
+            structured_result["asset_summary"] = build_asset_summary(findings_data, asset_services)
+            structured_result["scan_diff"] = build_scan_diff(findings_data, previous_findings)
 
-        async with db.transaction():
             await db.execute(
                 "UPDATE tasks SET structured_json = ? WHERE id = ?",
                 (json.dumps(structured_result), task_id)
@@ -1534,25 +1577,25 @@ async def _upsert_findings_and_report_from_scanner(self, db, task_id: str, owner
             result=result,
         )
         findings_data: List[Dict[str, Any]] = []
-        for finding in structured_result.get("findings", []):
-            findings_data.append(
-                await self._persist_finding(
-                    db,
-                    owner_id=owner_id,
-                    task_id=task_id,
-                    plugin_id=plugin_id,
-                    target=target,
-                    finding=finding,
+        async with db.transaction():
+            for finding in structured_result.get("findings", []):
+                findings_data.append(
+                    await self._persist_finding(
+                        db,
+                        owner_id=owner_id,
+                        task_id=task_id,
+                        plugin_id=plugin_id,
+                        target=target,
+                        finding=finding,
+                    )
                 )
-            )
 
-        structured_result["findings"] = findings_data
-        structured_result["severity_counts"] = self._build_severity_counts(findings_data)
-        structured_result["finding_groups"] = build_finding_groups(findings_data)
-        structured_result["asset_summary"] = build_asset_summary(findings_data, asset_services)
-        structured_result["scan_diff"] = build_scan_diff(findings_data, previous_findings)
+            structured_result["findings"] = findings_data
+            structured_result["severity_counts"] = self._build_severity_counts(findings_data)
+            structured_result["finding_groups"] = build_finding_groups(findings_data)
+            structured_result["asset_summary"] = build_asset_summary(findings_data, asset_services)
+            structured_result["scan_diff"] = build_scan_diff(findings_data, previous_findings)
 
-        async with db.transaction():
             await db.execute(
                 "UPDATE tasks SET structured_json = ? WHERE id = ?",
                 (json.dumps(structured_result), task_id)

backend/secuscan/finding_intelligence.py

@@ -42,6 +42,19 @@ def _now_iso() -> str:
     return datetime.now(timezone.utc).isoformat()
 
 
+def generate_finding_key(finding: Dict[str, Any], plugin_id: str, target: str, owner_id: str) -> str:
+    """
+    Generate a stable deduplication key for a finding that is consistent
+    across different scan tasks targeting the same asset. Unlike the per-task
+    finding ID, this key intentionally excludes any task identifier so that
+    the same vulnerability discovered by separate tasks produces the same key.
+    """
+    asset_ref = _guess_asset_ref(finding, target)
+    asset_id = _stable_id("asset", target, asset_ref)
+    signature = _issue_signature(finding)
+    return _stable_id("group", plugin_id, asset_id, signature, owner_id)
+
+
 def _parse_timestamp(raw: Any) -> str:
     if isinstance(raw, datetime):
         return raw.astimezone(timezone.utc).isoformat()

backend/secuscan/migrations/008_add_finding_group_unique.sql

@@ -0,0 +1,18 @@
+-- Migration: 008_add_finding_group_unique
+-- Replaces the non-unique index idx_findings_group_id with a UNIQUE
+-- index on (owner_id, finding_group_id) so that the same vulnerability
+-- found by different tasks does not produce duplicate rows.
+
+BEGIN TRANSACTION;
+
+-- Remove duplicates keeping the most recently discovered row per group
+DELETE FROM findings WHERE rowid NOT IN (
+    SELECT MIN(rowid) FROM findings
+    WHERE owner_id IS NOT NULL AND finding_group_id IS NOT NULL
+    GROUP BY owner_id, finding_group_id
+);
+
+DROP INDEX IF EXISTS idx_findings_group_id;
+CREATE UNIQUE INDEX IF NOT EXISTS idx_findings_group_id ON findings(owner_id, finding_group_id);
+
+COMMIT;

backend/secuscan/parser_sandbox.py

@@ -79,6 +79,7 @@ def __init__(self, plugin_id: str, reason: str, stderr: str = "") -> None:
         self.reason = reason
         # Keep stderr private; callers must not surface this to API consumers.
         self._stderr_diagnostic: str = stderr
+        self.stderr_excerpt = stderr[:2000] if stderr else ""
         # User-facing message: reason only — no stderr content.
         super().__init__(f"Parser sandbox failed for '{plugin_id}' ({reason})")
 
@@ -262,7 +263,7 @@ def _read_stderr() -> None:
             plugin_id,
             _sanitize_stderr(stderr_text),
         )
-        raise ParserSandboxError(plugin_id, f"timed out after {timeout_seconds}s")
+        raise ParserSandboxError(plugin_id, f"timed out after {timeout_seconds}s", stderr_text)
 
     if proc.returncode != 0:
         logger.error(
@@ -280,6 +281,7 @@ def _read_stderr() -> None:
         raise ParserSandboxError(
             plugin_id,
             f"subprocess exited with code {proc.returncode}",
+            stderr_text,
         )
 
     stdout_bytes = b"".join(stdout_chunks)
@@ -302,6 +304,7 @@ def _read_stderr() -> None:
         raise ParserSandboxError(
             plugin_id,
             f"parser returned non-JSON output: {exc}",
+            stderr_text,
         )
 
     if not isinstance(parsed, (dict, list)):