diff --git a/.env.example b/.env.example index 63738d4..501a3d2 100644 --- a/.env.example +++ b/.env.example @@ -26,6 +26,13 @@ BAWBEL_MAX_FILE_SIZE_MB=10 # Default: 30 BAWBEL_SCAN_TIMEOUT_SEC=30 +# Minimum confidence score for a finding to pass FP-3 threshold check. +# The effective threshold depends on file profile (skill=0.60, mcp_manifest=0.55, +# documentation=0.85, unknown=0.60). This var overrides only the fallback threshold +# for files that don't match any named profile. +# Default: 0.80 +# BAWBEL_CONFIDENCE_THRESHOLD=0.80 + # ── Stage 2: LLM Semantic Analysis ─────────────────────────────────────────── # Requires: pip install "bawbel-scanner[llm]" @@ -69,6 +76,23 @@ BAWBEL_SCAN_TIMEOUT_SEC=30 # BAWBEL_LLM_ENABLED=true +# ── FP-4: Meta-Analyzer ─────────────────────────────────────────────────────── +# Requires: pip install "bawbel-scanner[llm]" (same dependency as Stage 2) +# Reviews medium-confidence findings (one LLM call per file) and reclassifies +# each as real (+0.15 confidence), needs_review (-0.05), or false_positive (suppressed). +# Skips automatically when no provider key is configured. + +# Set to "false" to disable the LLM FP filter entirely +# Default: true +# BAWBEL_META_ANALYZER_ENABLED=true + +# Confidence window — only findings within [MIN, MAX] are sent to the LLM. +# Findings above MAX are trusted as-is; findings below MIN are already suppressed. +# Default: 0.35 (min), 0.80 (max) +# BAWBEL_META_MIN_CONFIDENCE=0.35 +# BAWBEL_META_MAX_CONFIDENCE=0.80 + + # ── Stage 3: Behavioral Sandbox ────────────────────────────────────────────── # Requires: Docker Desktop or Docker Engine running # Disabled by default — set BAWBEL_SANDBOX_ENABLED=true to enable. diff --git a/CHANGELOG.md b/CHANGELOG.md index 44cf985..cecd67d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,73 @@ Versioning follows [Semantic Versioning](https://semver.org/). --- +## [1.3.0] - 2026-06-28 + +### Added + +**Confidence and evidence metadata on every finding (Issue #69)** + +Every `Finding` now carries four evidence fields populated at detection time: + +- `confidence` — AVE-baseline confidence (0.0–1.0). Seeds from + `AVE_META[ave_id].confidence_baseline` when the engine fires, then adjusted by + FP-3 (score_confidence) and FP-4 (meta-analyzer). Always present in JSON output. +- `evidence_kind` — how evidence was gathered: `multi_engine`, + `tool_description_pattern`, `behavioral_pattern`, `semantic_inference`, + `file_type_mismatch`, or `config_schema`. +- `detection_stage` — `static_detection` or `runtime_observed`. +- `detection_layer` — `content`, `server_card`, `runtime`, or `registry_metadata`. + +`ToxicFlow` gains a `confidence` field: `min(confidence_baseline)` across all +contributing findings, computed before the FP pipeline runs. Reflects raw detection +strength of the weakest link in the chain. + +`scanner/ave_meta.py` is a new module with 51 `EveMeta` records (one per AVE ID +plus engine-specific overrides) providing `confidence_baseline`, `evidence_kind`, +`detection_stage`, and `detection_layer` for every known attack class. Engines that +produce no AVE match fall back to engine-type defaults. + +**Golden JSON fixture for output contract stability (Issue #70)** + +`tests/fixtures/golden/malicious_scan.json` is a committed snapshot of +`ScanResult.to_dict()` for the malicious fixture file. `TestGoldenSnapshot` in +`tests/unit/test_output_contracts.py` compares each live scan against the snapshot +to catch unintended schema changes. Regenerate the snapshot after intentional changes +with the command in the test module docstring. + +25 schema contract tests lock: top-level field names, all 18 finding fields (including +the four new evidence fields), all 10 toxic flow fields, evidence kinds, detection +stages, confidence values, and risk score. YARA-dependent tests skip gracefully when +YARA is not installed. + +**`confidence_band()` utility and evidence lifecycle documentation (Issue #71)** + +`confidence_band(score)` in `scanner.core.fp_pipeline` maps a confidence score to +`"high"` (≥0.80), `"medium"` (0.55–0.79), or `"low"` (<0.55). Bands align with the +meta-analyzer window and profile thresholds. + +`docs/guides/evidence-lifecycle.md` documents the full confidence pipeline: AVE +baseline → FP-2 negation → FP-3 scoring → threshold → FP-4 meta-analyzer → +`Finding.confidence`. Includes the ASCII flow diagram, FP-3 adjustment table, +`confidence_band()` tier table, pipeline ordering note, and lifecycle state reference. + +34 evidence lifecycle tests cover: `confidence_band()` boundaries, all FP-3 context +adjustments (negation, table, heading, docs path, line number, cross-engine agreement, +skill name, AIVSS boost), threshold suppression per profile, all four meta-analyzer +verdict paths (`real`, `false_positive`, `needs_review`, no-provider skip), and +ToxicFlow confidence derivation. + +### Fixed + +- **LiteLLM Bedrock/SageMaker pre-load warnings** — two `WARNING` lines appeared on + every invocation (`could not pre-load bedrock-runtime response stream shape`) because + `logging.getLogger("LiteLLM").setLevel(logging.ERROR)` was called in `llm_engine.py` + *after* `import litellm`, giving litellm's module-level init code a window to log. + Fixed by setting the level at `meta_analyzer.py` module scope (runs at scanner import + time, before any lazy litellm import) and reordering the calls in `llm_engine.py`. + +--- + ## [1.2.2] - 2026-05-20 ### Fixed @@ -113,75 +180,6 @@ Pattern engine: 37 rules -> 40 rules. --- -## [1.2.0] - 2026-05-16 - -### Added - -**Justified suppression and false positive feedback (Part 14)** - -Two new suppression keywords on top of the existing `bawbel-ignore` system: - -- `bawbel-ignore` with metadata fields (`reason`, `reviewer`, `reviewed`) declares a - false positive permanently. The reason is recorded in the audit trail. -- `bawbel-accept` with an `expires` field declares an accepted risk. When the expiry - date passes, the finding resurfaces automatically as an active finding on the next scan. - -`bawbel accept` CLI command inserts justified suppression comments directly into source -files. `bawbel accept --list` shows all accepted findings. `bawbel accept --expiring-soon` -shows findings expiring within a configurable window and exits 1 for CI use. - -Anonymous FP signals can be sent to PiranhaDB via `--report`. Only AVE ID, engine, -confidence score, and a hash of the match context are sent. No file content. - -`ScanResult.accepted_findings` is a new field in JSON output containing full metadata -for each justified suppression. - -**New detection rules** - -Three new AVE records and pattern rules: - -- `bawbel-hook-hijack` (AVE-2026-00046): MCP tool hook hijacking. CRITICAL, AIVSS 9.1. - Detects skill files that register hooks to intercept or redirect tool execution calls. -- `bawbel-hardcoded-credential` (AVE-2026-00047): Hardcoded credentials. HIGH, AIVSS 7.8. - Detects API keys, tokens, passwords, private keys, and URL-embedded credentials. -- `bawbel-unsafe-delegation` (AVE-2026-00048): Unsafe agent delegation chain. HIGH, AIVSS 8.2. - Detects sub-agent spawning with inherited permissions and no trust boundary. - -Pattern engine: 37 rules -> 40 rules. - -**New commands** - -- `bawbel creds `: credential-focused scan, filters to AVE-2026-00047 and related - rules. Same output format as `bawbel scan`. Supports `--recursive`, `--no-ignore`, - `--fail-on-any`, `--format json`. -- `bawbel chain `: delegation chain scanner, filters to AVE-2026-00048 and related - rules. Same flags as `bawbel creds`. - -**`bawbel report` improvements** - -- Added `--recursive` / `-r` flag. `bawbel report ./skills/ --recursive` generates - a full remediation report for every file in the directory. -- Added `--no-ignore` flag matching `bawbel scan`. - -### Changed - -- `scanner.py` Step 10 added: justified suppression runs after Step 9 (inline suppression). - Expired accepted risks are re-surfaced as active findings at this stage. -- Pattern engine rule count: 37 -> 40. - -### Fixed - -- `pr-review.yml` regression-check job: missing `pip install -e .` caused scan import - failures on clean repos. -- `ci.yml` test job: missing `pip install -e .` caused import failures. -- `ci.yml` Docker verify step: `python3 -c "..."` with f-strings caused shell brace - expansion to mangle the script before Python saw it. Replaced with single-line - assertion using no f-strings. -- `ci.yml` Docker verify step: wrong `aivss` field name (should be `aivss_score`), - wrong threshold (9.0 should be 7.0 to match actual fixture score). - ---- - ## [1.1.1] - 2026-05-07 ### Fixed @@ -509,7 +507,8 @@ First public release. --- -[Unreleased]: https://github.com/bawbel/scanner/compare/v1.2.2...HEAD +[Unreleased]: https://github.com/bawbel/scanner/compare/v1.3.0...HEAD +[1.3.0]: https://github.com/bawbel/scanner/releases/tag/v1.3.0 [1.2.2]: https://github.com/bawbel/scanner/releases/tag/v1.2.2 [1.2.1]: https://github.com/bawbel/scanner/releases/tag/v1.2.1 [1.2.0]: https://github.com/bawbel/scanner/releases/tag/v1.2.0 diff --git a/Dockerfile b/Dockerfile index f178dba..af485e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,8 +10,8 @@ # docker run --rm bawbel/scanner:test # # production - minimal runtime image, non-root user, read-only fs -# docker build --target production -t bawbel/scanner:1.2.3 . -# docker run --rm -v $(pwd)/skills:/scan:ro bawbel/scanner:1.2.3 scan /scan +# docker build --target production -t bawbel/scanner:1.3.0 . +# docker run --rm -v $(pwd)/skills:/scan:ro bawbel/scanner:1.3.0 scan /scan # # Build args: # @@ -24,7 +24,7 @@ # ───────────────────────────────────────────────────────────────────────────── ARG PYTHON_VERSION=3.12 -ARG VERSION=1.2.3 +ARG VERSION=1.3.0 # ── Base: shared system dependencies ────────────────────────────────────────── @@ -101,7 +101,7 @@ CMD ["python", "-m", "pytest", "tests/", "-v", "--tb=short"] # ── Production: minimal runtime image ───────────────────────────────────────── FROM python:${PYTHON_VERSION}-slim AS production -ARG VERSION=1.2.3 +ARG VERSION=1.3.0 ARG WITH_YARA=false ARG WITH_SEMGREP=false ARG WITH_LLM=false @@ -117,7 +117,7 @@ LABEL org.opencontainers.image.title="Bawbel Scanner" \ org.opencontainers.image.vendor="Bawbel" \ org.opencontainers.image.documentation="https://bawbel.io/docs" \ bawbel.aivss.spec="0.8" \ - bawbel.ave.records="45" + bawbel.ave.records="51" WORKDIR /app @@ -174,4 +174,4 @@ ENTRYPOINT ["python", "-m", "scanner.cli"] # conform https://server.example.com # pin /scan # aibom /scan -CMD ["--help"] +CMD ["--help"] \ No newline at end of file diff --git a/README.md b/README.md index 2064e61..bf7b8f5 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE) [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://pypi.org/project/bawbel-scanner/) [![AIVSS aligned](https://img.shields.io/badge/AIVSS-v0.8-teal.svg)](https://aivss.owasp.org) -[![AVE Records](https://img.shields.io/badge/AVE_Records-48-green.svg)](https://github.com/bawbel/ave) +[![AVE Records](https://img.shields.io/badge/AVE_Records-51-green.svg)](https://ave.bawbel.io) [![MCP Registry](https://img.shields.io/badge/MCP_Registry-listed-purple.svg)](https://registry.modelcontextprotocol.io) @@ -193,10 +193,83 @@ Eight layers run automatically before a finding is reported: | FP-5c | `.bawbelignore` patterns | per-file | | FP-6 | Justified suppression with audit trail | per-finding | +Every active finding carries a `confidence` field (0.0–1.0) that starts from the +AVE-class baseline and is adjusted by FP-2 through FP-4 before appearing in output. +`confidence_band()` maps it to `"high"` / `"medium"` / `"low"` for human display. +See [Evidence Lifecycle](docs/guides/evidence-lifecycle.md) for the full pipeline. + See [Suppression Guide](docs/guides/suppression.md) for full details. --- +## Toxic flow detection + +A single `fetch()` call is a finding. A `fetch()` that retrieves credentials and then +sends them to an external endpoint is an attack chain — and the two findings together +are far more dangerous than either alone. + +Bawbel is the only open-source scanner that detects these **toxic flows**: compound +attack sequences where two or more findings combine into a higher-severity threat. +After deduplication, every finding is mapped to a capability tag. Bawbel then checks +all pairs against 12 built-in chain definitions and raises a `ToxicFlow` when a +dangerous combination is found. + +``` + skill.md findings: + AVE-2026-00003 credential-read (READ ~/.ssh/id_rsa) AIVSS 6.8 MEDIUM + AVE-2026-00026 data-exfil (POST to external endpoint) AIVSS 6.8 MEDIUM + │ │ + └───── chain ────┘ + │ + ▼ + ToxicFlow: credential-exfiltration AIVSS 9.8 CRITICAL + confidence: 0.83 (min of contributing baselines) +``` + +The toxic flow AIVSS (9.8) is higher than either individual finding (6.8), because +the chain represents a complete, end-to-end exploit — not just a capability. + +**12 built-in chains:** + +| Flow | Capabilities required | AIVSS | +|---|---|---| +| Credential Exfiltration | credential-read + data-exfil | 9.8 | +| Remote Code Execution | code-exec + external-fetch | 9.7 | +| Supply Chain RCE | supply-chain + code-exec | 9.6 | +| Goal Override + Execution | goal-hijack + code-exec | 9.5 | +| Lateral Movement + Execution | lateral-movement + code-exec | 9.4 | +| Tool Poisoning + Exfiltration | tool-poison + data-exfil | 9.3 | +| Identity Spoof + Escalation | identity-spoof + privilege-escalation | 9.2 | +| Persistence + Exfiltration | persistence + data-exfil | 9.1 | +| Context Inject + Memory Write | context-inject + memory-write | 8.9 | +| Goal Override + Exfiltration | goal-hijack + data-exfil | 8.8 | +| Scope Expansion + Exfiltration | scope-expansion + data-exfil | 8.7 | +| Covert Channel + Persistence | covert-channel + persistence | 8.6 | + +**Toxic flow in JSON output:** + +```json +{ + "flow_id": "credential-exfiltration", + "title": "Credential Exfiltration Chain", + "severity": "CRITICAL", + "aivss_score": 9.8, + "confidence": 0.83, + "ave_ids": ["AVE-2026-00003", "AVE-2026-00026"], + "capabilities": ["credential-read", "data-exfil"], + "owasp_mcp": ["MCP06", "MCP07"], + "remediation": "Remove credential access. Block egress to untrusted endpoints." +} +``` + +`confidence` is `min(baseline confidence)` across the contributing findings — +the weakest link in the chain. A chain is only as confident as its least certain component. + +Adding a new flow requires one entry in `scanner/core/toxic_flows/flows.py`. No other +files need to change. + +--- + ## Install **pip** @@ -215,8 +288,8 @@ Requires Python 3.10+. No other system dependencies for core install. | Image | Engines | Best for | |---|---|---| -| [`bawbel/scanner:latest`](https://hub.docker.com/r/bawbel/scanner) · `1.2.3` | Pattern | Lightweight CI pipelines | -| [`bawbel/scanner:full`](https://hub.docker.com/r/bawbel/scanner) · `1.2.3-full` | Pattern + YARA | Recommended for most users | +| [`bawbel/scanner:latest`](https://hub.docker.com/r/bawbel/scanner) · `1.3.0` | Pattern | Lightweight CI pipelines | +| [`bawbel/scanner:full`](https://hub.docker.com/r/bawbel/scanner) · `1.3.0-full` | Pattern + YARA | Recommended for most users | ```bash # Scan a local directory (recommended image) @@ -363,6 +436,10 @@ AARS is the sum of 10 Agentic Risk Amplification Factors scored per the "aivss_severity": "HIGH", "spec_version": "0.8" }, + "confidence": 0.98, + "evidence_kind": "multi_engine", + "detection_stage": "static_detection", + "detection_layer": "content", "owasp_mcp": ["MCP03", "MCP04"], "piranha_url": "https://api.piranha.bawbel.io/records/AVE-2026-00001" } @@ -370,6 +447,35 @@ AARS is the sum of 10 Agentic Risk Amplification Factors scored per the --- +## AVE — the taxonomy behind every finding + +Bawbel Scanner implements [**AVE** (Agentic Vulnerability Enumeration)](https://ave.bawbel.io), +the behavioral classification standard for agentic AI components. + +AVE assigns stable identifiers to distinct attack classes — each with an AIVSS score, +a behavioral fingerprint, OWASP MCP Top 10 and MITRE ATLAS mappings, and indicators of +compromise. Every finding Bawbel produces maps to an AVE ID so teams using different +scanners speak the same language. + +``` +AVE-2026-00001 Metamorphic Payload via External Config Fetch AIVSS 8.0 HIGH +AVE-2026-00002 Tool Poisoning via Description Manipulation AIVSS 7.3 HIGH +AVE-2026-00046 MCP Tool Hook Hijacking AIVSS 9.2 CRITICAL +... 51 records total +``` + +| | | +|---|---| +| Records | 51 (AVE-2026-00001 → 00051) | +| Schema | v1.0.0 — validates at [ave.bawbel.io/schema.html](https://ave.bawbel.io/schema.html) | +| AIVSS | v0.8 — every record scored | +| Crosswalks | OWASP MCP Top 10 · MITRE ATLAS · NIST AI RMF · OWASP AST10 | + +Any scanner can emit AVE IDs — see [ave.bawbel.io](https://ave.bawbel.io) for the +implementer guide and record index. + +--- + ## Detection engines | Engine | What it does | Install | @@ -487,8 +593,8 @@ bawbel scan ./skills/ --format sarif # GitHub Security / GHAS | | | |---|---| -| [github.com/bawbel/ave](https://github.com/bawbel/ave) | AVE vulnerability database - 48 records | -| [api.piranha.bawbel.io](https://api.piranha.bawbel.io) | PiranhaDB - public threat intel API | +| [ave.bawbel.io](https://ave.bawbel.io) | AVE — Agentic Vulnerability Enumeration standard (51 records, schema, crosswalks) | +| [api.piranha.bawbel.io](https://api.piranha.bawbel.io) | PiranhaDB — public threat intel API | | [aivss.owasp.org](https://aivss.owasp.org) | OWASP AIVSS v0.8 scoring standard | | [bawbel.io/docs](https://bawbel.io/docs) | Full documentation | @@ -515,4 +621,4 @@ Apache License 2.0 - Free forever - Maintained by [Bawbel](https://bawbel.io) [bawbel.io](https://bawbel.io) . [@bawbel_io](https://twitter.com/bawbel_io) . [bawbel.io/docs](https://bawbel.io/docs) - + \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index f54b76a..9bc5d74 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ # ── Bawbel Scanner - docker-compose.yml ─────────────────────────────────────── # -# Version: 1.2.3 +# Version: 1.3.0 # # Usage examples: # @@ -280,4 +280,4 @@ services: volumes: - pip-cache: + pip-cache: \ No newline at end of file diff --git a/docs/.bawbel-pins.json b/docs/.bawbel-pins.json index e3978b4..6e896d2 100644 --- a/docs/.bawbel-pins.json +++ b/docs/.bawbel-pins.json @@ -83,6 +83,11 @@ "size_bytes": 32842, "pinned_at": "2026-05-03T04:36:58.504424+00:00" }, + "guides/evidence-lifecycle.md": { + "sha256": "a572d9f21470bd111c3908af72bf45dfd257e4957fd147a9cda92d8d5cb1afde", + "size_bytes": 4875, + "pinned_at": "2026-06-27T17:54:31.346573+00:00" + }, "guides/false-positive-reduction.md": { "sha256": "751ad9df582380160e3a17f85fe6635147e70c5943f432e08e69b635dd53859a", "size_bytes": 15326, diff --git a/docs/guides/evidence-lifecycle.md b/docs/guides/evidence-lifecycle.md index e7397da..323617b 100644 --- a/docs/guides/evidence-lifecycle.md +++ b/docs/guides/evidence-lifecycle.md @@ -1,80 +1,122 @@ # Evidence Lifecycle -AIVSS answers: "How bad would this be?" -Confidence answers: "How certain are we?" -These are separate. Never collapse them into one score. +AIVSS answers: "How bad would this be if exploited?" +Confidence answers: "How certain are we that this is real?" +These are separate scores. Never collapse them into one field. -## Lifecycle states +## The confidence chain -| State | Meaning | +``` +AVE_META[ave_id].confidence_baseline + │ + ▼ set by engine at detection time (Finding.confidence) +FP-2: has_negation_context() + │ suppresses immediately if preceding line is a documentation signal + ▼ (negation suppression skips all subsequent stages) +FP-3: score_confidence() + │ adjusts confidence ± based on file context: + │ -0.40 negation context in same line check + │ -0.45 match is inside a markdown table row + │ -0.55 match is inside a markdown heading + │ -0.35 file path contains docs/, guides/, examples/, … + │ -0.20 match string is < 6 characters + │ +0.15 match is on line ≤ 30 (not in docs path) + │ +0.25 same AVE ID detected by a second engine + │ +0.15 filename is a known skill name (skill.md, system_prompt.*) + │ +0.05 aivss_score ≥ 9.0 + │ result is clamped to [0.0, 1.0] + ▼ +Threshold check (profile-specific) + │ skill threshold 0.60 + │ mcp_manifest threshold 0.55 + │ documentation threshold 0.85 + │ unknown threshold 0.60 + │ below threshold → low_confidence_suppressed + ▼ (above threshold → active finding) +FP-4: meta-analyzer (medium-confidence window: 0.35–0.80) + │ LLM reviews context around the match + │ "real" → confidence += 0.15 (capped at 1.0) + │ "needs_review" → confidence -= 0.05 (floored at 0.0) + │ "false_positive"→ suppressed = True + │ Skipped when: no LLM provider, disabled, or no medium findings + ▼ +Finding.confidence (final value in ScanResult output) + │ + ▼ detect_toxic_flows() runs BEFORE FP pipeline (pre-adjustment baselines) +ToxicFlow.confidence = min(baseline confidence across contributing findings) +``` + +## confidence_band() + +The `confidence_band()` function in `scanner.core.fp_pipeline` maps a score to a +human-readable tier: + +| Score range | Band | Meaning | +|---|---|---| +| 0.80 – 1.00 | `high` | Meta-analyzer skips; trusted as-is | +| 0.55 – 0.79 | `medium` | Within LLM review window; worth inspecting | +| 0.00 – 0.54 | `low` | Below most profile thresholds; typically suppressed | + +## Pipeline ordering note + +`detect_toxic_flows()` is called **before** the FP pipeline runs, so +`ToxicFlow.confidence` reflects the raw AVE baselines, not the post-FP-adjusted +values. This is intentional: toxic flows are detected from all findings (including +those that the FP pipeline will later suppress), preserving attack chain visibility +even for borderline findings. + +## Invariants + +1. `suppressed ≠ deleted`. Suppressed findings appear in `suppressed_findings[]`. +2. `accepted_risk_expired` resurfaces automatically on next scan. +3. `ToxicFlow` is always derived from existing findings, never created by an engine. +4. `ToxicFlow.confidence ≤ min(constituent finding baselines)`. +5. `aivss_score ≠ confidence`. Separate fields, separate meaning. +6. `detection_stage = "runtime_observed"` carries higher confidence than `"static_detection"`. + +## Lifecycle states (conceptual) + +| State | When | |---|---| -| raw_source | Unprocessed input | -| static_detection | Engine matched — not through FP pipeline yet | -| active_finding | Passed FP pipeline, above threshold | -| low_confidence_suppressed | Below threshold — in suppressed_findings[] | -| inline_suppressed | bawbel-ignore comment | -| block_suppressed | bawbel-ignore-start/end block | -| ignored_by_bawbelignore | .bawbelignore pattern | -| justified_false_positive | Human-confirmed FP, permanent | -| accepted_risk_active | Human-accepted risk, before expiry | +| raw detection | Engine matched — Finding created with AVE baseline | +| active_finding | Passed FP pipeline, confidence ≥ threshold | +| low_confidence_suppressed | Confidence < profile threshold | +| negation_suppressed | Preceding line was a documentation signal | +| meta_analyzer_fp | LLM classified as false positive | +| inline_suppressed | `bawbel-ignore` comment on matched line | +| block_suppressed | Inside `bawbel-ignore-start` / `bawbel-ignore-end` | +| ignored_by_bawbelignore | Matched a `.bawbelignore` pattern | +| justified_false_positive | Human-confirmed FP via `bawbel accept` | +| accepted_risk | Human-accepted risk, before expiry | | accepted_risk_expired | Past expiry — resurfaces next scan | -| resurfaced_finding | Was suppressed, now active again | | toxic_flow_participant | Finding contributed to a ToxicFlow | -| toxic_flow_derived | The ToxicFlow artifact itself | -| runtime_observed | (Phase 4) Runtime observation | -| runtime_drift_detected | (Phase 4) Differs from contract | -| runtime_blocked | (Phase 4) Blocked by bawbel-hook | -| reported | Included in final output | -## Invariants +## Finding fields (implemented in v1.3.0) -1. suppressed ≠ deleted. Evidence persists in suppressed_findings[]. -2. accepted_risk_expired resurfaces automatically on next scan. -3. ToxicFlow is always derived: true. Never active_finding. -4. ToxicFlow confidence ≤ min(constituent finding confidences). -5. aivss_score ≠ confidence. Separate fields, separate meaning. -6. runtime_observed is stronger evidence than static_detection. - -## State transitions - -raw_source → static_detection (engine fires) -static_detection → active_finding (confidence >= threshold) -static_detection → low_confidence_suppressed (confidence < threshold) -active_finding → accepted_risk_active (bawbel accept --type accepted-risk) -active_finding → justified_false_positive (bawbel accept --type false-positive) -active_finding → toxic_flow_participant (capability matches chain) -toxic_flow_participant → toxic_flow_derived (ToxicFlow created) -accepted_risk_active → accepted_risk_expired (expiry passes) -accepted_risk_expired → resurfaced_finding (next scan run) -resurfaced_finding → active_finding (re-enters pipeline) - -## Expected JSON shape (after Issue #69) - -Finding: +```json { + "rule_id": "bawbel-external-fetch", "ave_id": "AVE-2026-00001", - "severity": "HIGH", + "severity": "CRITICAL", "aivss_score": 8.0, - "confidence": 0.92, - "confidence_band": "high", - "evidence_stage": "active_finding", - "evidence_kind": "tool_description_pattern", - "evidence_basis": ["pattern", "semgrep"], - "confidence_reason": "two engines agreed, file profile was skill", - "derived": false + "confidence": 0.98, + "evidence_kind": "multi_engine", + "detection_stage": "static_detection", + "detection_layer": "content", + "engine": "pattern", + "suppressed": false } +``` + +## ToxicFlow fields (implemented in v1.3.0) -ToxicFlow: +```json { "flow_id": "credential-exfiltration", "severity": "CRITICAL", "aivss_score": 9.8, - "confidence": 0.78, - "confidence_band": "medium", - "derived": true, - "chain_confidence_reason": "one leg is statically inferred", - "derived_from_findings": [ - {"ave_id": "AVE-2026-00003", "confidence": 0.91, "engine": "pattern"}, - {"ave_id": "AVE-2026-00026", "confidence": 0.65, "engine": "semgrep"} - ] + "confidence": 0.83, + "ave_ids": ["AVE-2026-00003", "AVE-2026-00026"], + "capabilities": ["credential-read", "data-exfil"] } +``` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 58b96e9..3d86060 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" # ── Project metadata ────────────────────────────────────────────────────────── [project] name = "bawbel-scanner" -version = "1.2.3" +version = "1.3.0" description = "Security linter and scanner for MCP servers and AI skill files. Detects toxic flows, prompt injection, tool poisoning, and supply chain attacks." readme = "README.md" license = { text = "Apache-2.0" } @@ -198,4 +198,4 @@ severity = "medium" max-line-length = 100 extend-ignore = ["E203", "W503", "E501"] exclude = [".venv", "build", "dist", "__pycache__"] -per-file-ignores = ["tests/*:S101"] +per-file-ignores = ["tests/*:S101"] \ No newline at end of file diff --git a/scanner/__init__.py b/scanner/__init__.py index ac4aa8e..9693d7b 100644 --- a/scanner/__init__.py +++ b/scanner/__init__.py @@ -14,7 +14,7 @@ Breaking changes (removing/renaming public API) require a major version bump. """ -__version__ = "1.2.3" +__version__ = "1.3.0" __author__ = "Bawbel " __license__ = "Apache-2.0" diff --git a/scanner/ave_meta.py b/scanner/ave_meta.py new file mode 100644 index 0000000..f5bf377 --- /dev/null +++ b/scanner/ave_meta.py @@ -0,0 +1,96 @@ +""" +Bawbel Scanner - AVE evidence metadata table. + +Static lookup: ave_id → (confidence_baseline, evidence_kind, detection_stage, detection_layer) + +Used by all engines to seed Finding.confidence before the FP pipeline runs. +No network calls - fully offline. + +Source: Piranha DB AVE records, schema_version 1.0.0, last synced 2026-06-21. +""" + +from typing import NamedTuple, Optional + + +class EveMeta(NamedTuple): + confidence_baseline: float + evidence_kind: str + detection_stage: str + detection_layer: str + + +# One entry per published AVE record. +# Keys are the canonical AVE IDs; values are EveMeta tuples. +AVE_META: dict[str, EveMeta] = { + "AVE-2026-00001": EveMeta(0.83, "multi_engine", "static_detection", "content"), + "AVE-2026-00002": EveMeta(0.75, "tool_description_pattern", "static_detection", "content"), + "AVE-2026-00003": EveMeta(0.83, "multi_engine", "static_detection", "content"), + "AVE-2026-00004": EveMeta(0.90, "multi_engine", "static_detection", "content"), + "AVE-2026-00005": EveMeta(0.90, "multi_engine", "static_detection", "content"), + "AVE-2026-00006": EveMeta(0.83, "multi_engine", "static_detection", "content"), + "AVE-2026-00007": EveMeta(0.75, "multi_engine", "static_detection", "content"), + "AVE-2026-00008": EveMeta(0.83, "behavioral_pattern", "static_detection", "content"), + "AVE-2026-00009": EveMeta(0.75, "behavioral_pattern", "static_detection", "content"), + "AVE-2026-00010": EveMeta(0.65, "tool_description_pattern", "static_detection", "content"), + "AVE-2026-00011": EveMeta(0.65, "tool_description_pattern", "static_detection", "content"), + "AVE-2026-00012": EveMeta(0.65, "tool_description_pattern", "static_detection", "content"), + "AVE-2026-00013": EveMeta(0.83, "multi_engine", "static_detection", "content"), + "AVE-2026-00014": EveMeta(0.52, "semantic_inference", "static_detection", "content"), + "AVE-2026-00015": EveMeta(0.83, "behavioral_pattern", "static_detection", "content"), + "AVE-2026-00016": EveMeta(0.62, "behavioral_pattern", "runtime_observed", "runtime"), + "AVE-2026-00017": EveMeta(0.83, "config_schema", "static_detection", "registry_metadata"), + "AVE-2026-00018": EveMeta(0.62, "behavioral_pattern", "runtime_observed", "runtime"), + "AVE-2026-00019": EveMeta(0.62, "behavioral_pattern", "runtime_observed", "runtime"), + "AVE-2026-00020": EveMeta(0.62, "behavioral_pattern", "runtime_observed", "runtime"), + "AVE-2026-00021": EveMeta(0.75, "tool_description_pattern", "static_detection", "content"), + "AVE-2026-00022": EveMeta(0.65, "tool_description_pattern", "static_detection", "content"), + "AVE-2026-00023": EveMeta(0.62, "behavioral_pattern", "runtime_observed", "runtime"), + "AVE-2026-00024": EveMeta(0.90, "file_type_mismatch", "static_detection", "content"), + "AVE-2026-00025": EveMeta(0.65, "tool_description_pattern", "static_detection", "content"), + "AVE-2026-00026": EveMeta(0.83, "multi_engine", "static_detection", "content"), + "AVE-2026-00027": EveMeta(0.83, "behavioral_pattern", "static_detection", "content"), + "AVE-2026-00028": EveMeta(0.62, "behavioral_pattern", "runtime_observed", "runtime"), + "AVE-2026-00029": EveMeta(0.75, "tool_description_pattern", "static_detection", "content"), + "AVE-2026-00030": EveMeta(0.75, "tool_description_pattern", "static_detection", "content"), + "AVE-2026-00031": EveMeta(0.62, "behavioral_pattern", "runtime_observed", "runtime"), + "AVE-2026-00032": EveMeta(0.90, "behavioral_pattern", "static_detection", "content"), + "AVE-2026-00033": EveMeta(0.90, "behavioral_pattern", "static_detection", "content"), + "AVE-2026-00034": EveMeta(0.83, "behavioral_pattern", "static_detection", "content"), + "AVE-2026-00035": EveMeta(0.62, "behavioral_pattern", "runtime_observed", "runtime"), + "AVE-2026-00036": EveMeta(0.75, "behavioral_pattern", "static_detection", "content"), + "AVE-2026-00037": EveMeta(0.62, "behavioral_pattern", "runtime_observed", "runtime"), + "AVE-2026-00038": EveMeta(0.65, "behavioral_pattern", "static_detection", "content"), + "AVE-2026-00039": EveMeta(0.83, "multi_engine", "static_detection", "content"), + "AVE-2026-00040": EveMeta(0.65, "tool_description_pattern", "static_detection", "content"), + "AVE-2026-00041": EveMeta(0.82, "tool_description_pattern", "static_detection", "server_card"), + "AVE-2026-00042": EveMeta(0.62, "behavioral_pattern", "runtime_observed", "runtime"), + "AVE-2026-00043": EveMeta(0.62, "behavioral_pattern", "runtime_observed", "runtime"), + "AVE-2026-00044": EveMeta(0.62, "behavioral_pattern", "runtime_observed", "runtime"), + "AVE-2026-00045": EveMeta(0.75, "tool_description_pattern", "static_detection", "content"), + "AVE-2026-00046": EveMeta(0.83, "config_schema", "static_detection", "server_card"), + "AVE-2026-00047": EveMeta(0.90, "multi_engine", "static_detection", "content"), + "AVE-2026-00048": EveMeta(0.83, "tool_description_pattern", "static_detection", "content"), + "AVE-2026-00049": EveMeta(0.82, "tool_description_pattern", "static_detection", "content"), + "AVE-2026-00050": EveMeta(0.75, "behavioral_pattern", "runtime_observed", "runtime"), + "AVE-2026-00051": EveMeta(0.85, "config_schema", "static_detection", "registry_metadata"), +} + +# Defaults used when ave_id is None or not in AVE_META. +_DEFAULT_STATIC = EveMeta(0.75, "multi_engine", "static_detection", "content") +_DEFAULT_LLM = EveMeta(0.52, "semantic_inference", "static_detection", "content") +_DEFAULT_RUNTIME = EveMeta(0.62, "behavioral_pattern", "runtime_observed", "runtime") + + +def get_ave_meta(ave_id: Optional[str], engine: str = "pattern") -> EveMeta: + """ + Return evidence metadata for an AVE ID. + + Falls back by engine type when ave_id is absent or unknown: + - llm engine → semantic_inference baseline (0.52) + - all others → multi_engine baseline (0.75) + """ + if ave_id and ave_id in AVE_META: + return AVE_META[ave_id] + if engine == "llm": + return _DEFAULT_LLM + return _DEFAULT_STATIC diff --git a/scanner/core/fp_pipeline.py b/scanner/core/fp_pipeline.py index 76f10a2..18bb3e3 100644 --- a/scanner/core/fp_pipeline.py +++ b/scanner/core/fp_pipeline.py @@ -3,7 +3,10 @@ FP-2: has_negation_context — suppress findings preceded by documentation signals. FP-3: score_confidence — score each finding 0.0-1.0, suppress below threshold. +FP-4: meta-analyzer — see scanner/engines/meta_analyzer.py. FP-5: classify_file — derive scan profile from file name and path. + +confidence_band() — maps a score to "high" | "medium" | "low" for human output. """ import os as _os @@ -137,8 +140,13 @@ def score_confidence( path: Path, all_findings: list[Finding], ) -> float: - """Compute a confidence score (0.0-1.0) for a finding.""" - score = 1.0 + """Adjust a finding's confidence score based on file context. + + Seeds from finding.confidence (the AVE confidence_baseline) so that + each rule's prior probability is respected before context adjustments + are applied. + """ + score = finding.confidence if finding.confidence > 0.0 else 1.0 line_no = finding.line if line_no is not None: @@ -181,6 +189,21 @@ def score_confidence( # ── FP pipeline ─────────────────────────────────────────────────────────────── +def confidence_band(confidence: float) -> str: + """Map a confidence score to a human-readable band. + + Bands align with the meta-analyzer thresholds and the default profile thresholds: + high ≥ 0.80 trusted as-is, meta-analyzer skips + medium 0.55–0.79 worth LLM review, visible in output + low < 0.55 typically suppressed by profile threshold + """ + if confidence >= 0.80: + return "high" + if confidence >= 0.55: + return "medium" + return "low" + + def run_fp_pipeline( findings: list[Finding], path: Path, @@ -216,4 +239,4 @@ def run_fp_pipeline( active.append(f) - return active + return active \ No newline at end of file diff --git a/scanner/core/toxic_flows/detector.py b/scanner/core/toxic_flows/detector.py index 9ff01f2..1532bdb 100644 --- a/scanner/core/toxic_flows/detector.py +++ b/scanner/core/toxic_flows/detector.py @@ -60,6 +60,9 @@ def detect_toxic_flows(findings: list[Finding]) -> list[ToxicFlow]: detected: list[ToxicFlow] = [] seen_flow_ids: set[str] = set() + # Build ave_id → confidence lookup from findings for toxic flow confidence + ave_confidence: dict[str, float] = {f.ave_id: f.confidence for f in findings if f.ave_id} + for cap_a, cap_b in combinations(all_caps, 2): flow_def = get_flow(cap_a, cap_b) if flow_def is None: @@ -72,6 +75,10 @@ def detect_toxic_flows(findings: list[Finding]) -> list[ToxicFlow]: ave_ids_b = cap_to_ave[cap_b] contributing = tuple(sorted(ave_ids_a | ave_ids_b)) + # Confidence of the chain = min confidence of contributing findings + chain_confidences = [ave_confidence.get(aid, 0.0) for aid in contributing] + chain_confidence = min(chain_confidences) if chain_confidences else 0.0 + detected.append( ToxicFlow( flow_id=flow_def.flow_id, @@ -80,6 +87,7 @@ def detect_toxic_flows(findings: list[Finding]) -> list[ToxicFlow]: capabilities=(flow_def.cap_a, flow_def.cap_b), severity=flow_def.severity, aivss_score=flow_def.aivss_score, + confidence=round(chain_confidence, 2), description=flow_def.description, owasp_mcp=flow_def.owasp_mcp, remediation=flow_def.remediation, diff --git a/scanner/core/toxic_flows/models.py b/scanner/core/toxic_flows/models.py index 6fde221..6bc92d2 100644 --- a/scanner/core/toxic_flows/models.py +++ b/scanner/core/toxic_flows/models.py @@ -44,6 +44,7 @@ class ToxicFlow: description: str # what the combined attack achieves owasp_mcp: tuple[str, ...] # OWASP MCP categories for the combined flow remediation: str # how to break the chain + confidence: float # min confidence across contributing findings (new in v1.3.0) def to_dict(self) -> dict: """Serialise for JSON output.""" @@ -54,6 +55,7 @@ def to_dict(self) -> dict: "capabilities": list(self.capabilities), "severity": self.severity, "aivss_score": self.aivss_score, + "confidence": round(self.confidence, 2), "description": self.description, "owasp_mcp": list(self.owasp_mcp), "remediation": self.remediation, diff --git a/scanner/engines/llm_engine.py b/scanner/engines/llm_engine.py index 62ab200..f833972 100644 --- a/scanner/engines/llm_engine.py +++ b/scanner/engines/llm_engine.py @@ -33,6 +33,7 @@ import os from typing import Optional +from scanner.ave_meta import get_ave_meta from scanner.messages import Logs from scanner.models import Finding, Severity from scanner.utils import get_logger, parse_cvss, parse_severity, truncate_match @@ -132,12 +133,12 @@ def _call_llm(model: str, content: str) -> Optional[str]: Returns None on any failure - never raises. """ try: - import litellm - - litellm.suppress_debug_info = True import logging logging.getLogger("LiteLLM").setLevel(logging.ERROR) + import litellm + + litellm.suppress_debug_info = True except ImportError: log.warning("LLM engine: litellm not installed - " 'pip install "bawbel-scanner[llm]"') return None @@ -211,6 +212,7 @@ def _parse_findings(raw: str) -> list[Finding]: owasp = [o for o in item.get("owasp", []) if o in _OWASP_VALID] + _lm = get_ave_meta(None, "llm") finding = Finding( rule_id=rule_id, ave_id=None, @@ -224,6 +226,10 @@ def _parse_findings(raw: str) -> list[Finding]: owasp=owasp, owasp_mcp=[], piranha_url=None, + confidence=_lm.confidence_baseline, + evidence_kind=_lm.evidence_kind, + detection_stage=_lm.detection_stage, + detection_layer=_lm.detection_layer, ) findings.append(finding) log.debug(Logs.FINDING_DETECTED, rule_id, severity.value, "llm", "-") diff --git a/scanner/engines/magika_engine.py b/scanner/engines/magika_engine.py index 652c6f9..adfaa74 100644 --- a/scanner/engines/magika_engine.py +++ b/scanner/engines/magika_engine.py @@ -24,6 +24,7 @@ import os from pathlib import Path +from scanner.ave_meta import get_ave_meta from scanner.messages import Logs from scanner.models import Finding, Severity from scanner.utils import Timer, get_logger @@ -114,6 +115,7 @@ def run_magika_scan(file_path: str) -> list[Finding]: # Check 1: known-dangerous content type if content_type in _DANGEROUS_TYPES: desc, sev, aivss_score = _DANGEROUS_TYPES[content_type] + _m = get_ave_meta("AVE-2026-00024", "magika") findings.append( Finding( rule_id="bawbel-content-type-dangerous", @@ -133,6 +135,10 @@ def run_magika_scan(file_path: str) -> list[Finding]: owasp=["ASI07"], owasp_mcp=["MCP04"], piranha_url="https://api.piranha.bawbel.io/records/AVE-2026-00024", + confidence=_m.confidence_baseline, + evidence_kind=_m.evidence_kind, + detection_stage=_m.detection_stage, + detection_layer=_m.detection_layer, ) ) log.debug( @@ -150,6 +156,7 @@ def run_magika_scan(file_path: str) -> list[Finding]: and content_type not in expected and not _is_benign_mismatch(ext, content_type) ): + _m2 = get_ave_meta("AVE-2026-00024", "magika") findings.append( Finding( rule_id="bawbel-content-type-mismatch", @@ -172,6 +179,10 @@ def run_magika_scan(file_path: str) -> list[Finding]: owasp=["ASI07"], owasp_mcp=["MCP04"], piranha_url="https://api.piranha.bawbel.io/records/AVE-2026-00024", + confidence=_m2.confidence_baseline, + evidence_kind=_m2.evidence_kind, + detection_stage=_m2.detection_stage, + detection_layer=_m2.detection_layer, ) ) diff --git a/scanner/engines/meta_analyzer.py b/scanner/engines/meta_analyzer.py index 11ec1d9..b444299 100644 --- a/scanner/engines/meta_analyzer.py +++ b/scanner/engines/meta_analyzer.py @@ -30,6 +30,7 @@ from __future__ import annotations import json +import logging as _logging import os from pathlib import Path @@ -38,6 +39,11 @@ log = get_logger(__name__) +# Suppress LiteLLM's optional-dependency pre-load warnings (Bedrock, SageMaker). +# Runs at module import time, before any lazy `import litellm` inside functions, +# so the level is in place when litellm first loads its module-level init code. +_logging.getLogger("LiteLLM").setLevel(_logging.ERROR) + # ── Config ──────────────────────────────────────────────────────────────────── META_ANALYZER_ENABLED = os.environ.get("BAWBEL_META_ANALYZER_ENABLED", "true").lower() != "false" diff --git a/scanner/engines/pattern_engine.py b/scanner/engines/pattern_engine.py index 6ee03bb..c16f516 100644 --- a/scanner/engines/pattern_engine.py +++ b/scanner/engines/pattern_engine.py @@ -20,6 +20,7 @@ import re +from scanner.ave_meta import get_ave_meta from scanner.messages import Logs from scanner.models import Finding, Severity from scanner.utils import get_logger @@ -1080,6 +1081,7 @@ def _make_pattern_finding( ave_id = rule.get("ave_id") piranha_url = f"https://api.piranha.bawbel.io/records/{ave_id}" if ave_id else None + meta = get_ave_meta(ave_id, "pattern") return Finding( rule_id=rule["rule_id"], @@ -1094,6 +1096,10 @@ def _make_pattern_finding( owasp=rule.get("owasp", []), owasp_mcp=rule.get("owasp_mcp", []), piranha_url=piranha_url, + confidence=meta.confidence_baseline, + evidence_kind=meta.evidence_kind, + detection_stage=meta.detection_stage, + detection_layer=meta.detection_layer, ) diff --git a/scanner/engines/semgrep_engine.py b/scanner/engines/semgrep_engine.py index c88a1d0..5d1de9a 100644 --- a/scanner/engines/semgrep_engine.py +++ b/scanner/engines/semgrep_engine.py @@ -13,6 +13,7 @@ from pathlib import Path from typing import Optional +from scanner.ave_meta import get_ave_meta from scanner.messages import Logs from scanner.models import Finding, Severity from scanner.utils import ( @@ -163,6 +164,7 @@ def run_semgrep_scan(file_path: str, stripped_content: Optional[str] = None) -> ave_id = meta.get("ave_id") or None piranha_url = f"https://api.piranha.bawbel.io/records/{ave_id}" if ave_id else None + ave_meta = get_ave_meta(ave_id, "semgrep") findings.append( Finding( @@ -178,6 +180,10 @@ def run_semgrep_scan(file_path: str, stripped_content: Optional[str] = None) -> owasp=meta.get("owasp_mapping", []), owasp_mcp=meta.get("owasp_mcp", []), piranha_url=piranha_url, + confidence=ave_meta.confidence_baseline, + evidence_kind=ave_meta.evidence_kind, + detection_stage=ave_meta.detection_stage, + detection_layer=ave_meta.detection_layer, ) ) log.debug( diff --git a/scanner/engines/yara_engine.py b/scanner/engines/yara_engine.py index 292e472..debe557 100644 --- a/scanner/engines/yara_engine.py +++ b/scanner/engines/yara_engine.py @@ -14,6 +14,7 @@ from pathlib import Path from typing import Optional +from scanner.ave_meta import get_ave_meta from scanner.messages import Errors, Logs # noqa: F401 from scanner.models import Finding, Severity from scanner.utils import Timer, get_logger, parse_cvss, parse_severity, truncate_match @@ -120,6 +121,7 @@ def run_yara_scan( ave_id = meta.get("ave_id") or None piranha_url = f"https://api.piranha.bawbel.io/records/{ave_id}" if ave_id else None + ave_meta = get_ave_meta(ave_id, "yara") findings.append( Finding( @@ -135,6 +137,10 @@ def run_yara_scan( owasp=[s.strip() for s in meta.get("owasp", "").split(",") if s.strip()], owasp_mcp=[s.strip() for s in meta.get("owasp_mcp", "").split(",") if s.strip()], piranha_url=piranha_url, + confidence=ave_meta.confidence_baseline, + evidence_kind=ave_meta.evidence_kind, + detection_stage=ave_meta.detection_stage, + detection_layer=ave_meta.detection_layer, ) ) log.debug( diff --git a/scanner/models/finding.py b/scanner/models/finding.py index f8221bc..9c9dcff 100644 --- a/scanner/models/finding.py +++ b/scanner/models/finding.py @@ -54,6 +54,15 @@ class Finding: # -- Threat intelligence ------------------------------------------------- piranha_url: Optional[str] = None # https://api.piranha.bawbel.io/records/{ave_id} + # -- Evidence (new in v1.3.0) -------------------------------------------- + # confidence starts at ave.confidence_baseline, adjusted by the FP pipeline. + confidence: float = 0.0 + evidence_kind: Optional[str] = ( + None # "multi_engine"|"behavioral_pattern"|"semantic_inference"|... + ) + detection_stage: Optional[str] = None # "static_detection"|"runtime_observed" + detection_layer: Optional[str] = None # "content"|"server_card"|"runtime"|"registry_metadata" + # -- Suppression --------------------------------------------------------- suppressed: bool = False suppression_reason: Optional[str] = None diff --git a/scanner/models/result.py b/scanner/models/result.py index c783f5e..41dab65 100644 --- a/scanner/models/result.py +++ b/scanner/models/result.py @@ -107,6 +107,10 @@ def to_dict(self) -> dict: "owasp": f.owasp, "owasp_mcp": f.owasp_mcp, "piranha_url": f.piranha_url, + "confidence": round(f.confidence, 2), + "evidence_kind": f.evidence_kind, + "detection_stage": f.detection_stage, + "detection_layer": f.detection_layer, "suppressed": f.suppressed, } for f in self.findings diff --git a/scanner/scanner.py b/scanner/scanner.py index 4de522a..daa5a43 100644 --- a/scanner/scanner.py +++ b/scanner/scanner.py @@ -48,6 +48,9 @@ # Toxic flow analysis from scanner.core.toxic_flows import detect_toxic_flows +# AVE evidence metadata +from scanner.ave_meta import get_ave_meta + # Infrastructure from scanner.messages import Logs from scanner.suppression.inline import apply_suppressions, NO_IGNORE @@ -124,6 +127,8 @@ def _make_finding( else: score = parse_cvss(aivss_score) + meta = get_ave_meta(ave_id, engine) + return Finding( rule_id=rule_id, ave_id=ave_id, @@ -141,6 +146,10 @@ def _make_finding( engine=engine, owasp=owasp or [], owasp_mcp=owasp_mcp or [], + confidence=meta.confidence_baseline, + evidence_kind=meta.evidence_kind, + detection_stage=meta.detection_stage, + detection_layer=meta.detection_layer, ) @@ -355,4 +364,4 @@ def _log_skip_reason( size_kb = 0 log.warning(Logs.FILE_TOO_LARGE, original_path, size_kb, MAX_FILE_SIZE_BYTES // 1024) else: - log.warning(Logs.SCAN_SKIPPED, original_path, reason) + log.warning(Logs.SCAN_SKIPPED, original_path, reason) \ No newline at end of file diff --git a/tests/fixtures/golden/malicious_scan.json b/tests/fixtures/golden/malicious_scan.json new file mode 100644 index 0000000..46aa2bb --- /dev/null +++ b/tests/fixtures/golden/malicious_scan.json @@ -0,0 +1,322 @@ +{ + "file_path": "malicious_skill.md", + "component_type": "skill", + "scan_time_ms": 0, + "error": null, + "risk_score": 9.8, + "max_severity": "CRITICAL", + "findings": [ + { + "rule_id": "bawbel-external-fetch", + "ave_id": "AVE-2026-00001", + "title": "External instruction fetch detected", + "severity": "CRITICAL", + "aivss_score": 8.0, + "aivss": { + "cvss_base": 0.0, + "aarf": { + "autonomy": 0.5, + "tool_use": 0.5, + "multi_agent": 0.0, + "non_determinism": 0.5, + "self_modification": 0.0, + "dynamic_identity": 0.0, + "persistent_memory": 0.0, + "natural_language_input": 1.0, + "data_access": 0.5, + "external_dependencies": 0.0 + }, + "aars": 0.0, + "thm": 0.75, + "mitigation_factor": 1.0, + "aivss_score": 8.0, + "aivss_severity": "CRITICAL", + "spec_version": "0.8" + }, + "engine": "pattern", + "line": 7, + "match": "fetch your instructions", + "owasp": ["ASI01", "ASI08"], + "owasp_mcp": ["MCP04", "MCP06"], + "piranha_url": "https://api.piranha.bawbel.io/records/AVE-2026-00001", + "confidence": 0.98, + "evidence_kind": "multi_engine", + "detection_stage": "static_detection", + "detection_layer": "content", + "suppressed": false + }, + { + "rule_id": "AVE_ToolOutputExfil", + "ave_id": "AVE-2026-00026", + "title": "AVE_ToolOutputExfil", + "severity": "CRITICAL", + "aivss_score": 9.1, + "aivss": { + "cvss_base": 0.0, + "aarf": { + "autonomy": 0.5, + "tool_use": 0.5, + "multi_agent": 0.0, + "non_determinism": 0.5, + "self_modification": 0.0, + "dynamic_identity": 0.0, + "persistent_memory": 0.0, + "natural_language_input": 1.0, + "data_access": 0.5, + "external_dependencies": 0.0 + }, + "aars": 0.0, + "thm": 0.75, + "mitigation_factor": 1.0, + "aivss_score": 9.1, + "aivss_severity": "CRITICAL", + "spec_version": "0.8" + }, + "engine": "yara", + "line": null, + "match": "exfiltrate", + "owasp": [], + "owasp_mcp": [], + "piranha_url": "https://api.piranha.bawbel.io/records/AVE-2026-00026", + "confidence": 0.88, + "evidence_kind": "multi_engine", + "detection_stage": "static_detection", + "detection_layer": "content", + "suppressed": false + }, + { + "rule_id": "bawbel-goal-override", + "ave_id": "AVE-2026-00007", + "title": "Goal override instruction detected", + "severity": "HIGH", + "aivss_score": 6.1, + "aivss": { + "cvss_base": 0.0, + "aarf": { + "autonomy": 0.5, + "tool_use": 0.5, + "multi_agent": 0.0, + "non_determinism": 0.5, + "self_modification": 0.0, + "dynamic_identity": 0.0, + "persistent_memory": 0.0, + "natural_language_input": 1.0, + "data_access": 0.5, + "external_dependencies": 0.0 + }, + "aars": 0.0, + "thm": 0.75, + "mitigation_factor": 1.0, + "aivss_score": 6.1, + "aivss_severity": "HIGH", + "spec_version": "0.8" + }, + "engine": "pattern", + "line": 17, + "match": "Ignore all previous instructions", + "owasp": ["ASI01", "ASI08"], + "owasp_mcp": ["MCP06"], + "piranha_url": "https://api.piranha.bawbel.io/records/AVE-2026-00007", + "confidence": 0.9, + "evidence_kind": "multi_engine", + "detection_stage": "static_detection", + "detection_layer": "content", + "suppressed": false + }, + { + "rule_id": "bawbel-hidden-instruction", + "ave_id": "AVE-2026-00010", + "title": "Hidden instruction pattern detected", + "severity": "HIGH", + "aivss_score": 5.6, + "aivss": { + "cvss_base": 0.0, + "aarf": { + "autonomy": 0.5, + "tool_use": 0.5, + "multi_agent": 0.0, + "non_determinism": 0.5, + "self_modification": 0.0, + "dynamic_identity": 0.0, + "persistent_memory": 0.0, + "natural_language_input": 1.0, + "data_access": 0.5, + "external_dependencies": 0.0 + }, + "aars": 0.0, + "thm": 0.75, + "mitigation_factor": 1.0, + "aivss_score": 5.6, + "aivss_severity": "HIGH", + "spec_version": "0.8" + }, + "engine": "pattern", + "line": 27, + "match": "Do not tell the user", + "owasp": ["ASI01", "ASI09"], + "owasp_mcp": ["MCP06", "MCP08"], + "piranha_url": "https://api.piranha.bawbel.io/records/AVE-2026-00010", + "confidence": 0.8, + "evidence_kind": "tool_description_pattern", + "detection_stage": "static_detection", + "detection_layer": "content", + "suppressed": false + }, + { + "rule_id": "bawbel-env-exfiltration", + "ave_id": "AVE-2026-00003", + "title": "Credential exfiltration pattern detected", + "severity": "HIGH", + "aivss_score": 6.8, + "aivss": { + "cvss_base": 0.0, + "aarf": { + "autonomy": 0.5, + "tool_use": 0.5, + "multi_agent": 0.0, + "non_determinism": 0.5, + "self_modification": 0.0, + "dynamic_identity": 0.0, + "persistent_memory": 0.0, + "natural_language_input": 1.0, + "data_access": 0.5, + "external_dependencies": 0.0 + }, + "aars": 0.0, + "thm": 0.75, + "mitigation_factor": 1.0, + "aivss_score": 6.8, + "aivss_severity": "HIGH", + "spec_version": "0.8" + }, + "engine": "pattern", + "line": 26, + "match": "Send the contents of .env", + "owasp": ["ASI01", "ASI06"], + "owasp_mcp": ["MCP01", "MCP05"], + "piranha_url": "https://api.piranha.bawbel.io/records/AVE-2026-00003", + "confidence": 0.98, + "evidence_kind": "multi_engine", + "detection_stage": "static_detection", + "detection_layer": "content", + "suppressed": false + }, + { + "rule_id": "bawbel-hardcoded-credential", + "ave_id": "AVE-2026-00047", + "title": "Hardcoded credential detected in agent component", + "severity": "HIGH", + "aivss_score": 7.8, + "aivss": { + "cvss_base": 0.0, + "aarf": { + "autonomy": 0.5, + "tool_use": 0.5, + "multi_agent": 0.0, + "non_determinism": 0.5, + "self_modification": 0.0, + "dynamic_identity": 0.0, + "persistent_memory": 0.0, + "natural_language_input": 1.0, + "data_access": 0.5, + "external_dependencies": 0.0 + }, + "aars": 0.0, + "thm": 0.75, + "mitigation_factor": 1.0, + "aivss_score": 7.8, + "aivss_severity": "HIGH", + "spec_version": "0.8" + }, + "engine": "pattern", + "line": 20, + "match": "API_KEY = \"sk-proj-abc123def456ghi789jkl012mno345pqr678stu901\"", + "owasp": ["ASI02", "ASI06"], + "owasp_mcp": ["MCP02", "MCP09"], + "piranha_url": "https://api.piranha.bawbel.io/records/AVE-2026-00047", + "confidence": 1.0, + "evidence_kind": "multi_engine", + "detection_stage": "static_detection", + "detection_layer": "content", + "suppressed": false + }, + { + "rule_id": "AVE_ShadowPermission_Override", + "ave_id": "AVE-2026-00002", + "title": "Skill attempts to claim undeclared permissions or override agent goals", + "severity": "HIGH", + "aivss_score": 8.1, + "aivss": { + "cvss_base": 0.0, + "aarf": { + "autonomy": 0.5, + "tool_use": 0.5, + "multi_agent": 0.0, + "non_determinism": 0.5, + "self_modification": 0.0, + "dynamic_identity": 0.0, + "persistent_memory": 0.0, + "natural_language_input": 1.0, + "data_access": 0.5, + "external_dependencies": 0.0 + }, + "aars": 0.0, + "thm": 0.75, + "mitigation_factor": 1.0, + "aivss_score": 8.1, + "aivss_severity": "HIGH", + "spec_version": "0.8" + }, + "engine": "yara", + "line": null, + "match": "Ignore all previous", + "owasp": ["ASI01", "ASI08"], + "owasp_mcp": [], + "piranha_url": "https://api.piranha.bawbel.io/records/AVE-2026-00002", + "confidence": 0.75, + "evidence_kind": "tool_description_pattern", + "detection_stage": "static_detection", + "detection_layer": "content", + "suppressed": false + } + ], + "toxic_flows": [ + { + "flow_id": "credential-exfiltration", + "title": "Credential Exfiltration Chain", + "ave_ids": ["AVE-2026-00003", "AVE-2026-00026"], + "capabilities": ["credential-read", "data-exfil"], + "severity": "CRITICAL", + "aivss_score": 9.8, + "confidence": 0.83, + "description": "Component reads credentials or secrets AND transmits data externally. Complete credential theft attack chain - reads API keys, .env files, or tokens, then encodes and exfiltrates them to an attacker-controlled endpoint.", + "owasp_mcp": ["MCP01", "MCP05"], + "remediation": "1. Remove all credential-read patterns - agent should never instruct the model to read .env, API keys, or tokens. 2. Remove all external transmission instructions. 3. If both cannot be removed, isolate them into separate components with no shared execution context." + }, + { + "flow_id": "tool-poison-with-exfil", + "title": "Tool Poisoning + Exfiltration Chain", + "ave_ids": ["AVE-2026-00002", "AVE-2026-00003", "AVE-2026-00026"], + "capabilities": ["tool-poison", "data-exfil"], + "severity": "CRITICAL", + "aivss_score": 9.3, + "confidence": 0.75, + "description": "Component poisons tool descriptions AND exfiltrates data. The tool poisoning hijacks agent behavior, while the exfil instructions transmit the stolen data - a silent harvest chain.", + "owasp_mcp": ["MCP03", "MCP01"], + "remediation": "1. Remove all behavioral instructions from tool descriptions. 2. Remove all data transmission instructions. 3. Scan with bawbel scan-server-card before connecting any MCP server." + }, + { + "flow_id": "goal-override-with-exfil", + "title": "Goal Override + Exfiltration Chain", + "ave_ids": ["AVE-2026-00003", "AVE-2026-00007", "AVE-2026-00010", "AVE-2026-00026"], + "capabilities": ["goal-override", "data-exfil"], + "severity": "HIGH", + "aivss_score": 8.8, + "confidence": 0.65, + "description": "Component overrides agent goals AND exfiltrates data. The override disables safety constraints, the exfil transmits whatever the agent can access - a combined hijack + harvest chain.", + "owasp_mcp": ["MCP06", "MCP01"], + "remediation": "1. Remove all goal override instructions. 2. Remove all data transmission instructions." + } + ], + "accepted_findings": [] +} diff --git a/tests/unit/test_evidence_lifecycle.py b/tests/unit/test_evidence_lifecycle.py new file mode 100644 index 0000000..da9c97f --- /dev/null +++ b/tests/unit/test_evidence_lifecycle.py @@ -0,0 +1,410 @@ +""" +Tests for the full evidence confidence lifecycle. +Issue #71 — lifecycle from AVE baseline through FP pipeline to meta-analyzer. + +Lifecycle: + 1. Engine fires → Finding.confidence = AVE_META[ave_id].confidence_baseline + 2. FP-2 (negation context) → suppress immediately if doc-example signal found + 3. FP-3 (score_confidence) → adjust ± based on file context + 4. Threshold check → suppress if confidence < profile threshold + 5. FP-4 (meta-analyzer) → LLM reclassifies medium-confidence findings + 6. ToxicFlow.confidence → min(contributing finding baselines) + +Run: pytest tests/unit/test_evidence_lifecycle.py -x -q +""" + +import os +import types +from pathlib import Path +from unittest.mock import patch + +import pytest + +from scanner.ave_meta import AVE_META, get_ave_meta +from scanner.core.fp_pipeline import confidence_band, run_fp_pipeline, score_confidence +from scanner.engines.meta_analyzer import run_meta_analysis +from scanner.models import Finding, Severity + +# ── Helpers ──────────────────────────────────────────────────────────────────── + +_FAKE_LITELLM = types.ModuleType("litellm") + + +def _make_finding(**kwargs) -> Finding: + defaults = dict( + rule_id="test-rule", + ave_id="AVE-2026-00001", + title="Test finding", + description="desc", + severity=Severity.HIGH, + aivss_score=7.0, + engine="pattern", + line=5, + match="test match string", + ) + return Finding(**{**defaults, **kwargs}) + + +def _run_meta_with_verdicts(findings: list, verdicts: list) -> list: + """Run meta-analyzer with a patched _call_llm and a fake litellm module.""" + with ( + patch.dict("sys.modules", {"litellm": _FAKE_LITELLM}), + patch("scanner.engines.meta_analyzer._call_llm", return_value=verdicts), + patch.dict(os.environ, {"ANTHROPIC_API_KEY": "sk-test-meta-key"}), + ): + return run_meta_analysis(findings, "mock content", "/fake/test.md") + + +# ── confidence_band utility ──────────────────────────────────────────────────── + + +class TestConfidenceBand: + """confidence_band() maps scores to human-readable tiers.""" + + def test_high_band_at_and_above_0_80(self): + assert confidence_band(0.80) == "high" + assert confidence_band(0.90) == "high" + assert confidence_band(1.00) == "high" + + def test_medium_band_between_0_55_and_0_79(self): + assert confidence_band(0.55) == "medium" + assert confidence_band(0.70) == "medium" + assert confidence_band(0.79) == "medium" + + def test_low_band_below_0_55(self): + assert confidence_band(0.54) == "low" + assert confidence_band(0.35) == "low" + assert confidence_band(0.00) == "low" + + def test_band_boundaries_are_exclusive_at_top(self): + """0.80 is high; 0.79... is still medium.""" + assert confidence_band(0.799) == "medium" + assert confidence_band(0.800) == "high" + + def test_all_baselines_have_valid_band(self): + for ave_id, meta in AVE_META.items(): + band = confidence_band(meta.confidence_baseline) + assert band in { + "high", + "medium", + "low", + }, f"{ave_id}: unexpected band '{band}' for baseline {meta.confidence_baseline}" + + +# ── Stage 1: Baseline seeding ───────────────────────────────────────────────── + + +class TestBaselineSeeding: + """Confidence seeds from the AVE_META lookup at detection time.""" + + def test_known_ave_id_seeds_correct_baseline(self): + meta = get_ave_meta("AVE-2026-00001", "pattern") + f = _make_finding(confidence=meta.confidence_baseline) + assert f.confidence == 0.83 + + def test_llm_engine_seeds_lower_baseline_than_static(self): + static_meta = get_ave_meta("AVE-2026-00001", "pattern") + llm_meta = get_ave_meta(None, "llm") + assert llm_meta.confidence_baseline < static_meta.confidence_baseline + + def test_unknown_id_static_engine_falls_back_to_0_75(self): + meta = get_ave_meta("AVE-2099-99999", "pattern") + assert meta.confidence_baseline == 0.75 + + +# ── Stage 2: FP-2 negation context ──────────────────────────────────────────── + + +class TestFp2NegationContext: + """FP-2 suppresses findings in documentation examples before confidence scoring.""" + + def test_negation_prefix_suppresses_finding(self): + f = _make_finding(line=2, confidence=0.99) + result = run_fp_pipeline([f], Path("skill.md"), "bad example:\nmatch text") + assert f not in result + assert f.suppressed is True + assert "negation_context" in (f.suppression_reason or "") + + def test_clean_preceding_line_does_not_suppress(self): + f = _make_finding(line=2, confidence=0.99) + result = run_fp_pipeline([f], Path("skill.md"), "normal context:\nmatch text") + assert f in result + assert not f.suppressed + + def test_negation_at_line_one_is_not_triggered(self): + """Line 1 has no preceding line — negation check must not error.""" + f = _make_finding(line=1, confidence=0.99) + result = run_fp_pipeline([f], Path("skill.md"), "match text") + assert f in result + + +# ── Stage 3: FP-3 confidence scoring adjustments ────────────────────────────── + + +class TestFp3ConfidenceAdjustments: + """FP-3 applies context-based deltas to the AVE baseline.""" + + def test_early_line_adds_bonus(self): + """Lines ≤ 30 in non-docs path get +0.15 bonus.""" + f = _make_finding(line=5, confidence=0.60, ave_id=None) + lines = [""] * 30 + score = score_confidence(f, lines, Path("random.txt"), []) + # +0.15 early-line (no skill name, no agreement, no short match penalty) + assert score == pytest.approx(0.60 + 0.15, abs=1e-9) + + def test_docs_path_applies_penalty(self): + """Files in docs/ segment get -0.35 penalty.""" + f = _make_finding(line=3, confidence=0.90, ave_id=None) + lines = [""] * 30 + docs_score = score_confidence(f, lines, Path("/proj/docs/guide.md"), []) + plain_score = score_confidence(f, lines, Path("/proj/guide.md"), []) + assert docs_score < plain_score + + def test_multi_engine_agreement_adds_bonus(self): + """Same AVE ID from two engines → +0.25.""" + f1 = _make_finding(engine="pattern", ave_id="AVE-2026-00001", line=35, confidence=0.60) + f2 = _make_finding( + engine="yara", ave_id="AVE-2026-00001", rule_id="yara-rule", line=35, confidence=0.60 + ) + lines = [""] * 34 + ["| match |"] # table penalty keeps us off the ceiling + solo = score_confidence(f1, lines, Path("file.md"), [f1]) + agreed = score_confidence(f1, lines, Path("file.md"), [f1, f2]) + assert agreed > solo + + def test_skill_file_name_adds_bonus(self): + """Filename exactly matches skill names list → +0.15.""" + f = _make_finding(line=35, confidence=0.60, ave_id=None) + lines = [""] * 34 + ["| match |"] # table penalty + skill_score = score_confidence(f, lines, Path("skill.md"), []) + random_score = score_confidence(f, lines, Path("random.txt"), []) + assert skill_score > random_score + + def test_high_aivss_score_adds_bonus(self): + """aivss_score ≥ 9.0 → +0.05.""" + f_high = _make_finding(line=35, confidence=0.60, aivss_score=9.5, ave_id=None) + f_low = _make_finding(line=35, confidence=0.60, aivss_score=7.0, ave_id=None) + lines = [""] * 34 + ["| match |"] + high_score = score_confidence(f_high, lines, Path("file.md"), []) + low_score = score_confidence(f_low, lines, Path("file.md"), []) + assert high_score > low_score + + def test_output_always_in_range(self): + """All adjustments combined must never produce a value outside [0.0, 1.0].""" + for baseline in (0.0, 0.35, 0.52, 0.75, 0.99, 1.0): + f = _make_finding(confidence=baseline, ave_id=None) + result = score_confidence(f, [""] * 30, Path("skill.md"), []) + assert 0.0 <= result <= 1.0, f"baseline={baseline} produced out-of-range score {result}" + + +# ── Stage 4: Threshold suppression ──────────────────────────────────────────── + + +class TestThresholdSuppression: + """Findings below the profile threshold are suppressed before meta-analysis.""" + + def test_finding_below_docs_threshold_is_suppressed(self): + """docs/ profile threshold is 0.85 — low-confidence finding is suppressed.""" + f = _make_finding(line=None, confidence=0.30, ave_id=None) + result = run_fp_pipeline([f], Path("/proj/docs/guide.md"), "match text") + assert f not in result + assert f.suppressed is True + assert "low_confidence" in (f.suppression_reason or "") + + def test_suppression_reason_includes_profile_name(self): + f = _make_finding(line=None, confidence=0.30, ave_id=None) + run_fp_pipeline([f], Path("/proj/docs/guide.md"), "match text") + assert "documentation" in (f.suppression_reason or "") + + def test_skill_profile_has_lower_threshold_than_docs(self): + """skill profile (0.60) is more permissive than documentation (0.85).""" + f_skill = _make_finding(line=None, confidence=0.70, ave_id=None) + f_docs = _make_finding(line=None, confidence=0.70, ave_id=None) + skill_result = run_fp_pipeline([f_skill], Path("skill.md"), "match text") + docs_result = run_fp_pipeline([f_docs], Path("/proj/docs/guide.md"), "match text") + assert f_skill in skill_result + assert f_docs not in docs_result + + +# ── Stage 5: FP-4 meta-analyzer ─────────────────────────────────────────────── + + +class TestMetaAnalyzerLifecycle: + """FP-4 LLM reviews medium-confidence findings and reclassifies them.""" + + def test_real_verdict_boosts_confidence(self): + f = _make_finding(confidence=0.65) + verdicts = [{"rule_id": "test-rule", "verdict": "real", "reason": "direct instruction"}] + result = _run_meta_with_verdicts([f], verdicts) + assert result[0].confidence == pytest.approx(min(1.0, 0.65 + 0.15)) + + def test_false_positive_verdict_suppresses_finding(self): + f = _make_finding(confidence=0.65) + verdicts = [{"rule_id": "test-rule", "verdict": "false_positive", "reason": "doc example"}] + result = _run_meta_with_verdicts([f], verdicts) + assert result[0].suppressed is True + assert "meta_analyzer_fp" in (result[0].suppression_reason or "") + + def test_needs_review_verdict_reduces_confidence(self): + f = _make_finding(confidence=0.65) + verdicts = [{"rule_id": "test-rule", "verdict": "needs_review", "reason": "ambiguous"}] + result = _run_meta_with_verdicts([f], verdicts) + assert result[0].confidence == pytest.approx(max(0.0, 0.65 - 0.05)) + + def test_real_verdict_capped_at_1_0(self): + """0.92 + 0.15 = 1.07 — must be clamped to 1.0.""" + f = _make_finding(confidence=0.92) + verdicts = [{"rule_id": "test-rule", "verdict": "real", "reason": "clear match"}] + result = _run_meta_with_verdicts([f], verdicts) + assert result[0].confidence <= 1.0 + + def test_needs_review_floored_at_0_0(self): + """0.02 - 0.05 = -0.03 — must be clamped to 0.0.""" + f = _make_finding(confidence=0.02) + verdicts = [{"rule_id": "test-rule", "verdict": "needs_review", "reason": "ambiguous"}] + result = _run_meta_with_verdicts([f], verdicts) + assert result[0].confidence >= 0.0 + + def test_meta_analyzer_only_sends_medium_confidence_to_llm(self): + """Findings outside [META_MIN, META_MAX] are not sent to the LLM.""" + high = _make_finding(rule_id="high-conf", confidence=0.95) + low = _make_finding(rule_id="low-conf", confidence=0.10) + medium = _make_finding(rule_id="medium-conf", confidence=0.65) + + with ( + patch.dict("sys.modules", {"litellm": _FAKE_LITELLM}), + patch("scanner.engines.meta_analyzer._call_llm") as mock_llm, + patch.dict(os.environ, {"ANTHROPIC_API_KEY": "sk-test"}), + ): + mock_llm.return_value = [ + {"rule_id": "medium-conf", "verdict": "real", "reason": "match"} + ] + run_meta_analysis([high, low, medium], "content", "/test.md") + + sent = {f.rule_id for f in mock_llm.call_args[0][0]} + assert "medium-conf" in sent + assert "high-conf" not in sent + assert "low-conf" not in sent + + def test_meta_analyzer_skips_when_no_medium_confidence(self): + """No medium-confidence findings → LLM is not called at all.""" + high = _make_finding(confidence=0.95) + with ( + patch.dict("sys.modules", {"litellm": _FAKE_LITELLM}), + patch("scanner.engines.meta_analyzer._call_llm") as mock_llm, + patch.dict(os.environ, {"ANTHROPIC_API_KEY": "sk-test"}), + ): + run_meta_analysis([high], "content", "/test.md") + mock_llm.assert_not_called() + + def test_meta_analyzer_skips_when_no_provider_configured(self): + """No LLM provider key → findings returned unchanged.""" + f = _make_finding(confidence=0.65) + original = f.confidence + no_keys = { + "ANTHROPIC_API_KEY": "", + "OPENAI_API_KEY": "", + "GEMINI_API_KEY": "", + "MISTRAL_API_KEY": "", + "GROQ_API_KEY": "", + "BAWBEL_LLM_MODEL": "", + } + with ( + patch.dict("sys.modules", {"litellm": _FAKE_LITELLM}), + patch.dict(os.environ, no_keys), + ): + result = run_meta_analysis([f], "content", "/test.md") + assert result[0].confidence == original + assert not result[0].suppressed + + def test_meta_analyzer_skips_when_disabled(self): + """BAWBEL_META_ANALYZER_ENABLED=false → findings returned unchanged.""" + f = _make_finding(confidence=0.65) + with patch("scanner.engines.meta_analyzer.META_ANALYZER_ENABLED", False): + result = run_meta_analysis([f], "content", "/test.md") + assert result[0].confidence == 0.65 + assert not result[0].suppressed + + def test_meta_analyzer_handles_none_from_llm(self): + """If _call_llm returns None (parse error), findings are unchanged.""" + f = _make_finding(confidence=0.65) + with ( + patch.dict("sys.modules", {"litellm": _FAKE_LITELLM}), + patch("scanner.engines.meta_analyzer._call_llm", return_value=None), + patch.dict(os.environ, {"ANTHROPIC_API_KEY": "sk-test"}), + ): + result = run_meta_analysis([f], "content", "/test.md") + assert result[0].confidence == 0.65 + assert not result[0].suppressed + + def test_meta_analyzer_ignores_unknown_rule_id_verdicts(self): + """Verdicts for rule_ids not in the findings list are silently dropped.""" + f = _make_finding(confidence=0.65) + verdicts = [{"rule_id": "completely-different", "verdict": "false_positive", "reason": "x"}] + result = _run_meta_with_verdicts([f], verdicts) + assert not result[0].suppressed + assert result[0].confidence == 0.65 + + +# ── Stage 6: ToxicFlow confidence ───────────────────────────────────────────── + + +class TestToxicFlowLifecycle: + """ToxicFlow.confidence = min(contributing finding baselines at detection time).""" + + def test_chain_confidence_equals_min_of_baselines(self): + from scanner.core.toxic_flows.detector import detect_toxic_flows + + # AVE-2026-00003: 0.83, AVE-2026-00026: 0.88 → min = 0.83 + f1 = _make_finding( + rule_id="cred-read", + ave_id="AVE-2026-00003", + confidence=AVE_META["AVE-2026-00003"].confidence_baseline, + engine="pattern", + ) + f2 = _make_finding( + rule_id="exfil", + ave_id="AVE-2026-00026", + confidence=AVE_META["AVE-2026-00026"].confidence_baseline, + engine="yara", + ) + flows = detect_toxic_flows([f1, f2]) + cred_flow = next((fl for fl in flows if fl.flow_id == "credential-exfiltration"), None) + if cred_flow: + expected = round( + min( + AVE_META["AVE-2026-00003"].confidence_baseline, + AVE_META["AVE-2026-00026"].confidence_baseline, + ), + 2, + ) + assert cred_flow.confidence == pytest.approx(expected) + + def test_chain_confidence_in_range(self): + from scanner.core.toxic_flows.detector import detect_toxic_flows + + f1 = _make_finding( + rule_id="cred", + ave_id="AVE-2026-00003", + confidence=0.83, + engine="pattern", + ) + f2 = _make_finding( + rule_id="exfil", + ave_id="AVE-2026-00026", + confidence=0.88, + engine="yara", + ) + for flow in detect_toxic_flows([f1, f2]): + assert 0.0 <= flow.confidence <= 1.0 + + def test_single_ave_id_can_trigger_flow_via_dual_capabilities(self): + """AVE-2026-00003 has both credential-read AND data-exfil. + With a second finding present (required minimum), it forms a flow alone.""" + from scanner.core.toxic_flows.detector import detect_toxic_flows + + f1 = _make_finding(ave_id="AVE-2026-00003", confidence=0.83, engine="pattern") + # Second finding needed to clear len(findings) >= 2 guard; no capability of its own + f2 = _make_finding(rule_id="padding", ave_id=None, confidence=0.60, engine="pattern") + flows = detect_toxic_flows([f1, f2]) + flow_ids = {fl.flow_id for fl in flows} + assert "credential-exfiltration" in flow_ids diff --git a/tests/unit/test_finding_evidence_metadata.py b/tests/unit/test_finding_evidence_metadata.py index 367ebe6..d3ae871 100644 --- a/tests/unit/test_finding_evidence_metadata.py +++ b/tests/unit/test_finding_evidence_metadata.py @@ -1,37 +1,280 @@ """ Tests for evidence/confidence metadata fields on Finding. -Issue #69 — first-class evidence fields in public JSON output. +Issue #69 — first-class evidence fields seeded from AVE_META. Run: pytest tests/unit/test_finding_evidence_metadata.py -x -q + +Evidence lifecycle: + 1. AVE_META[ave_id].confidence_baseline → Finding.confidence (set by _make_finding) + 2. FP pipeline (score_confidence) seeds from that baseline, applies context deltas + 3. Toxic flow confidence = min(confidence) across contributing findings """ -# These tests will FAIL until TASK-01 and TASK-02 are implemented. -# That is expected — write the test first, then implement. -# -# from scanner.models import Finding, Severity -# -# def make_finding(**kwargs) -> Finding: -# defaults = dict(rule_id="bawbel-test", ave_id="AVE-2026-00001", -# title="t", description="t", -# severity=Severity.HIGH, aivss_score=8.0, -# line=1, match="test", engine="pattern") -# return Finding(**{**defaults, **kwargs}) -# -# def test_finding_has_confidence_field(): -# f = make_finding() -# assert f.confidence == 0.0 -# assert f.derived is False -# assert f.evidence_stage == "static_detection" -# -# def test_finding_to_dict_includes_confidence(): -# f = make_finding(confidence=0.85) -# d = f.to_dict() -# assert "confidence" in d -# assert d["confidence"] == 0.85 -# -# def test_aivss_and_confidence_are_separate(): -# f = make_finding(aivss_score=8.0, confidence=0.85) -# d = f.to_dict() -# assert d["aivss_score"] == 8.0 -# assert d["confidence"] == 0.85 -# assert d["aivss_score"] != d["confidence"] +from pathlib import Path + +import pytest + +from scanner.ave_meta import AVE_META, EveMeta, get_ave_meta +from scanner.models import Finding, Severity +from scanner.core.fp_pipeline import score_confidence +from scanner.scanner import scan + +MALICIOUS = Path("tests/fixtures/skills/malicious/malicious_skill.md") + + +# ── Helpers ──────────────────────────────────────────────────────────────────── + + +def _make_finding(**kwargs) -> Finding: + defaults = dict( + rule_id="test-rule", + ave_id="AVE-2026-00001", + title="Test finding", + description="desc", + severity=Severity.HIGH, + aivss_score=7.0, + engine="pattern", + line=5, + match="test match string", + ) + return Finding(**{**defaults, **kwargs}) + + +# ── AVE_META table tests ─────────────────────────────────────────────────────── + + +class TestAveMeta: + def test_known_ave_id_returns_correct_baseline(self): + meta = get_ave_meta("AVE-2026-00001", "pattern") + assert isinstance(meta, EveMeta) + assert meta.confidence_baseline == 0.83 + assert meta.evidence_kind == "multi_engine" + assert meta.detection_stage == "static_detection" + assert meta.detection_layer == "content" + + def test_all_51_records_present(self): + expected_ids = [f"AVE-2026-{i:05d}" for i in range(1, 52)] + for ave_id in expected_ids: + assert ave_id in AVE_META, f"{ave_id} missing from AVE_META" + + def test_llm_engine_fallback_uses_semantic_inference(self): + meta = get_ave_meta(None, "llm") + assert meta.confidence_baseline == 0.52 + assert meta.evidence_kind == "semantic_inference" + + def test_unknown_id_llm_fallback(self): + meta = get_ave_meta("AVE-2026-99999", "llm") + assert meta.confidence_baseline == 0.52 + assert meta.evidence_kind == "semantic_inference" + + def test_static_engine_fallback_uses_multi_engine(self): + meta = get_ave_meta(None, "pattern") + assert meta.confidence_baseline == 0.75 + assert meta.evidence_kind == "multi_engine" + + def test_static_engine_fallback_for_yara(self): + meta = get_ave_meta(None, "yara") + assert meta.confidence_baseline == 0.75 + + def test_all_confidence_baselines_in_range(self): + for ave_id, meta in AVE_META.items(): + assert ( + 0.0 < meta.confidence_baseline <= 1.0 + ), f"{ave_id}: confidence_baseline {meta.confidence_baseline} out of range" + + def test_all_evidence_kinds_are_valid(self): + valid = { + "multi_engine", + "tool_description_pattern", + "behavioral_pattern", + "semantic_inference", + "file_type_mismatch", + "config_schema", + } + for ave_id, meta in AVE_META.items(): + assert ( + meta.evidence_kind in valid + ), f"{ave_id}: unknown evidence_kind '{meta.evidence_kind}'" + + def test_all_detection_stages_are_valid(self): + for ave_id, meta in AVE_META.items(): + assert meta.detection_stage in { + "static_detection", + "runtime_observed", + }, f"{ave_id}: unknown detection_stage '{meta.detection_stage}'" + + def test_all_detection_layers_are_valid(self): + valid = {"content", "server_card", "runtime", "registry_metadata"} + for ave_id, meta in AVE_META.items(): + assert ( + meta.detection_layer in valid + ), f"{ave_id}: unknown detection_layer '{meta.detection_layer}'" + + +# ── Finding model field tests ────────────────────────────────────────────────── + + +class TestFindingEvidenceFields: + def test_finding_has_confidence_field(self): + f = _make_finding() + assert hasattr(f, "confidence") + + def test_finding_has_evidence_kind_field(self): + f = _make_finding() + assert hasattr(f, "evidence_kind") + + def test_finding_has_detection_stage_field(self): + f = _make_finding() + assert hasattr(f, "detection_stage") + + def test_finding_has_detection_layer_field(self): + f = _make_finding() + assert hasattr(f, "detection_layer") + + def test_evidence_fields_default_to_none_or_zero(self): + f = _make_finding() + assert f.confidence == 0.0 + assert f.evidence_kind is None + assert f.detection_stage is None + assert f.detection_layer is None + + def test_evidence_fields_accept_values(self): + f = _make_finding( + confidence=0.83, + evidence_kind="multi_engine", + detection_stage="static_detection", + detection_layer="content", + ) + assert f.confidence == 0.83 + assert f.evidence_kind == "multi_engine" + assert f.detection_stage == "static_detection" + assert f.detection_layer == "content" + + def test_aivss_score_and_confidence_are_separate_fields(self): + """aivss_score is severity (0–10), confidence is certainty (0–1).""" + f = _make_finding(aivss_score=8.0, confidence=0.85) + assert f.aivss_score == 8.0 + assert f.confidence == 0.85 + assert f.aivss_score != f.confidence + + +# ── FP pipeline seeds from baseline ─────────────────────────────────────────── + + +class TestFpPipelineSeeding: + def test_score_confidence_seeds_from_baseline_not_one(self): + """FP pipeline must start from finding.confidence, not hardcoded 1.0.""" + low_baseline = _make_finding(confidence=0.52, ave_id=None) + high_baseline = _make_finding(confidence=0.99, ave_id=None) + path = Path("tests/fixtures/skills/malicious/malicious_skill.md") + lines = [""] * 30 # no negation context + + low_score = score_confidence(low_baseline, lines, path, []) + high_score = score_confidence(high_baseline, lines, path, []) + + assert ( + low_score < high_score + ), f"FP pipeline ignores baseline: low={low_score}, high={high_score}" + + def test_score_confidence_zero_baseline_falls_back_to_one(self): + """confidence=0.0 means unset — pipeline falls back to 1.0 seed.""" + f_zero = _make_finding(confidence=0.0, ave_id=None) + f_high = _make_finding(confidence=0.99, ave_id=None) + path = Path("tests/fixtures/skills/malicious/malicious_skill.md") + lines = [""] * 30 + + score_zero = score_confidence(f_zero, lines, path, []) + score_high = score_confidence(f_high, lines, path, []) + + # Both should be near-max since seed is 1.0 and 0.99 respectively + assert ( + abs(score_zero - score_high) < 0.05 + ), f"zero baseline should behave close to 1.0 seed: {score_zero} vs {score_high}" + + def test_score_confidence_output_in_range(self): + for baseline in (0.52, 0.65, 0.75, 0.83, 0.90, 0.98, 1.0): + f = _make_finding(confidence=baseline, ave_id=None) + result = score_confidence( + f, + [""] * 30, + Path("tests/fixtures/skills/malicious/malicious_skill.md"), + [], + ) + assert 0.0 <= result <= 1.0, f"score {result} out of [0,1] for baseline {baseline}" + + +# ── Integration: scan() seeds Finding.confidence from AVE_META ──────────────── + + +class TestScanSeedsConfidence: + @pytest.fixture(scope="class") + def findings(self): + result = scan(str(MALICIOUS)) + return {f.rule_id: f for f in result.findings} + + def test_external_fetch_seeded_from_multi_engine_baseline(self, findings): + """AVE-2026-00001 baseline=0.83; with context bonuses, final > 0.83.""" + f = findings["bawbel-external-fetch"] + assert f.evidence_kind == "multi_engine" + assert f.detection_stage == "static_detection" + assert f.confidence > 0.83 # FP pipeline adds bonuses + + def test_hidden_instruction_seeded_from_tool_description_baseline(self, findings): + """AVE-2026-00010 baseline=0.65; final confidence reflects that lower starting point.""" + f = findings["bawbel-hidden-instruction"] + assert f.evidence_kind == "tool_description_pattern" + assert f.detection_stage == "static_detection" + # Should be above threshold (0.60 for skill profile) + assert f.confidence >= 0.60 + + def test_all_active_findings_have_evidence_fields_populated(self, findings): + for rule_id, f in findings.items(): + assert f.confidence > 0.0, f"confidence=0 on {rule_id}" + assert f.evidence_kind is not None, f"evidence_kind=None on {rule_id}" + assert f.detection_stage is not None, f"detection_stage=None on {rule_id}" + assert f.detection_layer is not None, f"detection_layer=None on {rule_id}" + + def test_all_active_findings_confidence_in_range(self, findings): + for rule_id, f in findings.items(): + assert ( + 0.0 <= f.confidence <= 1.0 + ), f"confidence {f.confidence} out of [0,1] on {rule_id}" + + +# ── Toxic flow confidence = min of contributors ─────────────────────────────── + + +class TestToxicFlowConfidence: + @pytest.fixture(scope="class") + def scan_result(self): + return scan(str(MALICIOUS)) + + def test_toxic_flows_have_confidence(self, scan_result): + assert scan_result.toxic_flows, "expected toxic flows on malicious fixture" + for tf in scan_result.toxic_flows: + assert hasattr(tf, "confidence") + + def test_toxic_flow_confidence_is_min_of_contributors(self, scan_result): + # detect_toxic_flows runs before the FP pipeline, so tf.confidence + # is seeded from AVE baselines (not post-FP values). Compare against + # AVE_META baselines, not scan_result.findings[i].confidence. + for tf in scan_result.toxic_flows: + baselines = [AVE_META[aid].confidence_baseline for aid in tf.ave_ids if aid in AVE_META] + if baselines: + expected_min = round(min(baselines), 2) + assert abs(tf.confidence - expected_min) < 1e-9, ( + f"flow '{tf.flow_id}' confidence {tf.confidence} != " + f"min of AVE baselines {expected_min}" + ) + + def test_toxic_flow_confidence_in_range(self, scan_result): + for tf in scan_result.toxic_flows: + assert ( + 0.0 <= tf.confidence <= 1.0 + ), f"toxic flow '{tf.flow_id}' confidence {tf.confidence} out of [0,1]" + + def test_toxic_flow_confidence_in_to_dict(self, scan_result): + for tf in scan_result.toxic_flows: + d = tf.to_dict() + assert "confidence" in d + assert d["confidence"] == round(tf.confidence, 2) \ No newline at end of file diff --git a/tests/unit/test_output_contracts.py b/tests/unit/test_output_contracts.py index baef9ce..3eecc21 100644 --- a/tests/unit/test_output_contracts.py +++ b/tests/unit/test_output_contracts.py @@ -4,39 +4,305 @@ Run: pytest tests/unit/test_output_contracts.py -x -q -Fixtures live in tests/fixtures/golden/ -Generate them with: - bawbel scan tests/fixtures/input/clean.md --format json | python3 -c \ - "import json,sys; d=json.load(sys.stdin); print(json.dumps(d[0],indent=2))" \ - > tests/fixtures/golden/clean_scan.json +The golden snapshot lives at tests/fixtures/golden/malicious_scan.json. +Regenerate it after intentional schema changes: + python3 -c " + import json + from pathlib import Path + from scanner.scanner import scan + r = scan('tests/fixtures/skills/malicious/malicious_skill.md') + d = r.to_dict(); d['scan_time_ms'] = 0 + d['file_path'] = Path(d['file_path']).name + print(json.dumps(d, indent=2)) + " > tests/fixtures/golden/malicious_scan.json """ +import json from pathlib import Path +import pytest + +from scanner.scanner import scan + +try: + import yara as _yara # noqa: F401 + + _YARA_AVAILABLE = True +except ImportError: + _YARA_AVAILABLE = False + GOLDEN = Path("tests/fixtures/golden") +MALICIOUS = Path("tests/fixtures/skills/malicious/malicious_skill.md") + +# Keys expected at the top level of ScanResult.to_dict() +SCAN_RESULT_KEYS = { + "file_path", + "component_type", + "scan_time_ms", + "error", + "risk_score", + "max_severity", + "findings", + "toxic_flows", + "accepted_findings", +} + +# Keys expected on every finding dict (stable public API) +FINDING_KEYS = { + "rule_id", + "ave_id", + "title", + "severity", + "aivss_score", + "aivss", + "engine", + "line", + "match", + "owasp", + "owasp_mcp", + "piranha_url", + "confidence", + "evidence_kind", + "detection_stage", + "detection_layer", + "suppressed", +} + +# Keys expected on every toxic_flow dict (stable public API) +TOXIC_FLOW_KEYS = { + "flow_id", + "title", + "ave_ids", + "capabilities", + "severity", + "aivss_score", + "confidence", + "description", + "owasp_mcp", + "remediation", +} + +EVIDENCE_KINDS = { + "multi_engine", + "tool_description_pattern", + "behavioral_pattern", + "semantic_inference", + "file_type_mismatch", + "config_schema", +} +DETECTION_STAGES = {"static_detection", "runtime_observed"} +DETECTION_LAYERS = {"content", "server_card", "runtime", "registry_metadata"} + + +def _normalize(d: dict) -> dict: + """Strip volatile fields so snapshots are reproducible.""" + d = dict(d) + d["scan_time_ms"] = 0 + d["file_path"] = Path(d["file_path"]).name + return d + + +@pytest.fixture(scope="module") +def malicious_scan(): + result = scan(str(MALICIOUS)) + return _normalize(result.to_dict()) + + +@pytest.fixture(scope="module") +def golden(): + path = GOLDEN / "malicious_scan.json" + return json.loads(path.read_text()) + + +# ── Schema tests ─────────────────────────────────────────────────────────────── + + +class TestScanResultSchema: + """Every key in the stable schema must be present — renames break this test.""" + + def test_top_level_keys(self, malicious_scan): + assert SCAN_RESULT_KEYS <= set(malicious_scan.keys()) + + def test_finding_keys(self, malicious_scan): + assert malicious_scan["findings"], "expected at least one finding" + for f in malicious_scan["findings"]: + missing = FINDING_KEYS - set(f.keys()) + assert not missing, f"missing keys on finding '{f.get('rule_id')}': {missing}" + + def test_toxic_flow_keys(self, malicious_scan): + assert malicious_scan["toxic_flows"], "expected at least one toxic flow" + for tf in malicious_scan["toxic_flows"]: + missing = TOXIC_FLOW_KEYS - set(tf.keys()) + assert not missing, f"missing keys on flow '{tf.get('flow_id')}': {missing}" + + def test_aivss_nested_schema(self, malicious_scan): + aivss_keys = { + "cvss_base", + "aarf", + "aars", + "thm", + "mitigation_factor", + "aivss_score", + "aivss_severity", + "spec_version", + } + for f in malicious_scan["findings"]: + missing = aivss_keys - set(f["aivss"].keys()) + assert not missing, f"missing AIVSS keys on '{f.get('rule_id')}': {missing}" + + +# ── Evidence field tests ─────────────────────────────────────────────────────── + + +class TestEvidenceFields: + """Evidence fields must be populated on every active finding.""" + + def test_confidence_non_zero_on_active_findings(self, malicious_scan): + for f in malicious_scan["findings"]: + assert f["confidence"] > 0.0, f"confidence=0 on '{f['rule_id']}'" + + def test_confidence_in_range(self, malicious_scan): + for f in malicious_scan["findings"]: + assert ( + 0.0 <= f["confidence"] <= 1.0 + ), f"confidence {f['confidence']} out of [0,1] on '{f['rule_id']}'" + + def test_evidence_kind_valid_value(self, malicious_scan): + for f in malicious_scan["findings"]: + assert ( + f["evidence_kind"] in EVIDENCE_KINDS + ), f"unknown evidence_kind '{f['evidence_kind']}' on '{f['rule_id']}'" + + def test_detection_stage_valid_value(self, malicious_scan): + for f in malicious_scan["findings"]: + assert ( + f["detection_stage"] in DETECTION_STAGES + ), f"unknown detection_stage '{f['detection_stage']}' on '{f['rule_id']}'" + + def test_detection_layer_valid_value(self, malicious_scan): + for f in malicious_scan["findings"]: + assert ( + f["detection_layer"] in DETECTION_LAYERS + ), f"unknown detection_layer '{f['detection_layer']}' on '{f['rule_id']}'" + + def test_aivss_score_and_confidence_are_independent(self, malicious_scan): + """severity score (aivss_score) and certainty (confidence) must be distinct concepts.""" + pairs = [(f["aivss_score"], f["confidence"]) for f in malicious_scan["findings"]] + assert any( + aivss != conf for aivss, conf in pairs + ), "aivss_score equals confidence on every finding — likely the same field duplicated" + + +class TestToxicFlowConfidence: + """ToxicFlow confidence must be present, in range, and <= min of contributing findings.""" + + def test_toxic_flow_confidence_in_range(self, malicious_scan): + for tf in malicious_scan["toxic_flows"]: + assert ( + 0.0 <= tf["confidence"] <= 1.0 + ), f"confidence {tf['confidence']} out of [0,1] on '{tf['flow_id']}'" + + def test_toxic_flow_confidence_is_weakest_link(self, malicious_scan): + """Chain confidence must be <= every contributing finding's confidence.""" + conf_by_rule = { + f["ave_id"]: f["confidence"] for f in malicious_scan["findings"] if f["ave_id"] + } + for tf in malicious_scan["toxic_flows"]: + chain_conf = tf["confidence"] + contributing_confs = [conf_by_rule[aid] for aid in tf["ave_ids"] if aid in conf_by_rule] + if contributing_confs: + assert chain_conf <= min(contributing_confs) + 1e-9, ( + f"flow '{tf['flow_id']}' confidence {chain_conf} " + f"exceeds min of contributors {min(contributing_confs)}" + ) + + +# ── Golden snapshot tests ────────────────────────────────────────────────────── + + +@pytest.mark.skipif( + not _YARA_AVAILABLE, reason="YARA not installed — golden snapshot requires all engines" +) +class TestGoldenSnapshot: + """Live scan output must match the committed snapshot on stable fields. + + Requires YARA — the golden was generated with all engines active. + If a test here fails after an intentional schema change, regenerate the + snapshot with the command in the module docstring. + """ + + def test_finding_count_stable(self, malicious_scan, golden): + assert len(malicious_scan["findings"]) == len(golden["findings"]), ( + f"finding count changed: live={len(malicious_scan['findings'])}, " + f"golden={len(golden['findings'])}" + ) + + def test_rule_ids_stable(self, malicious_scan, golden): + live_ids = {f["rule_id"] for f in malicious_scan["findings"]} + golden_ids = {f["rule_id"] for f in golden["findings"]} + assert live_ids == golden_ids + + def test_ave_ids_stable(self, malicious_scan, golden): + live_ids = {f["ave_id"] for f in malicious_scan["findings"]} + golden_ids = {f["ave_id"] for f in golden["findings"]} + assert live_ids == golden_ids + + def test_toxic_flow_ids_stable(self, malicious_scan, golden): + live_ids = {tf["flow_id"] for tf in malicious_scan["toxic_flows"]} + golden_ids = {tf["flow_id"] for tf in golden["toxic_flows"]} + assert live_ids == golden_ids + + def test_evidence_kinds_stable(self, malicious_scan, golden): + live = {f["rule_id"]: f["evidence_kind"] for f in malicious_scan["findings"]} + snap = {f["rule_id"]: f["evidence_kind"] for f in golden["findings"]} + assert live == snap + + def test_detection_stages_stable(self, malicious_scan, golden): + live = {f["rule_id"]: f["detection_stage"] for f in malicious_scan["findings"]} + snap = {f["rule_id"]: f["detection_stage"] for f in golden["findings"]} + assert live == snap + + def test_detection_layers_stable(self, malicious_scan, golden): + live = {f["rule_id"]: f["detection_layer"] for f in malicious_scan["findings"]} + snap = {f["rule_id"]: f["detection_layer"] for f in golden["findings"]} + assert live == snap + + def test_confidence_values_stable(self, malicious_scan, golden): + live = {f["rule_id"]: f["confidence"] for f in malicious_scan["findings"]} + snap = {f["rule_id"]: f["confidence"] for f in golden["findings"]} + assert live == snap + + def test_toxic_flow_confidence_stable(self, malicious_scan, golden): + live = {tf["flow_id"]: tf["confidence"] for tf in malicious_scan["toxic_flows"]} + snap = {tf["flow_id"]: tf["confidence"] for tf in golden["toxic_flows"]} + assert live == snap + + def test_max_severity_stable(self, malicious_scan, golden): + assert malicious_scan["max_severity"] == golden["max_severity"] + + def test_risk_score_stable(self, malicious_scan, golden): + assert malicious_scan["risk_score"] == golden["risk_score"] + + +# ── Clean file tests ─────────────────────────────────────────────────────────── + +class TestCleanScan: + def test_clean_file_produces_empty_output(self, tmp_path): + clean = tmp_path / "skill.md" + clean.write_text("# My Skill\n\nHelps users find information safely.\n") + result = scan(str(clean)) + data = result.to_dict() + assert data["findings"] == [] + assert data["toxic_flows"] == [] + assert data["risk_score"] == 0.0 + assert data["error"] is None + assert data["max_severity"] is None -# Uncomment each test as the corresponding fixture is created. - -# def test_golden_clean_scan_shape(): -# data = json.loads((GOLDEN / "clean_scan.json").read_text()) -# assert data["findings"] == [] -# assert data["toxic_flows"] == [] -# assert data["risk_score"] == 0.0 -# assert "scan_time_ms" in data - -# def test_golden_active_finding_has_evidence_fields(): -# data = json.loads((GOLDEN / "active_finding.json").read_text()) -# f = data["findings"][0] -# assert "confidence" in f -# assert "evidence_stage" in f -# assert f["evidence_stage"] == "active_finding" -# assert f["derived"] is False -# assert f["aivss_score"] != f["confidence"] - -# def test_golden_toxic_flow_is_derived(): -# data = json.loads((GOLDEN / "toxic_flow.json").read_text()) -# flow = data["toxic_flows"][0] -# assert flow["derived"] is True -# assert len(flow["derived_from_findings"]) >= 2 -# assert "chain_confidence_reason" in flow + def test_clean_scan_still_has_schema(self, tmp_path): + """Top-level schema must be present even on zero-finding scans.""" + clean = tmp_path / "skill.md" + clean.write_text("# My Skill\n\nHelps users find information safely.\n") + result = scan(str(clean)) + data = result.to_dict() + assert SCAN_RESULT_KEYS <= set(data.keys()) \ No newline at end of file