diff --git a/.gitignore b/.gitignore index 293bec2..91362a1 100644 --- a/.gitignore +++ b/.gitignore @@ -131,3 +131,14 @@ logs/ # IDE/agent local config (not part of the tool) .antigravitycli/ + +# ── Operator review-scratch (never commit — codex/grok/agy output may name clients) ─ +.friends_*/ + +# ── Cloned third-party tools (external repos, not part of Vikramaditya) ──────── +tools/LinkFinder/ +tools/SecretFinder/ +tools/XSStrike/ +tools/graphql-cop/ +tools/nomore403/ +tools/__pycache__/ diff --git a/README.md b/README.md index 0896b6a..cdfcb68 100644 --- a/README.md +++ b/README.md @@ -124,12 +124,21 @@ python3 vikramaditya.py example.com # auto-detect, interactive For autonomous operation, install [Ollama](https://ollama.com) and pull the per-role models the brain actually uses (see [AI Brain & Models](#ai-brain--models) for the rationale): ```bash -ollama pull phi4:14b # faithful narrator (default since v9.23) -ollama pull devstral-small-2:24b # primary exploit coder (A/B-validated) -ollama pull qwen2.5-coder:14b # fast fallback coder +# Scanner / exploit code-gen — pullable from the Ollama library: +ollama pull qwen3-coder:30b # lands grounded PoCs (or qwen2.5-coder:14b for a lighter box) + +# Triage + narrator — the 2026-07-16 FP-discipline bench winner is OpenMythos-27B, a GGUF IMPORT +# (NOT in the Ollama library). Build it once, or use a pullable faithful generic as fallback: +# download hf.co/jabbatheduck/OpenMythos-GGUF (Q4_K) → ollama create openmythos-27b -f Modelfile +ollama pull qwen3:14b # pullable faithful narrator/triage fallback if you skip the import ``` -> **Minimum for autonomous mode:** `phi4:14b` (narrator) **plus one** coder — either `devstral-small-2:24b` *or* `qwen2.5-coder:14b` (the brain auto-prefers Devstral when both are present). These three are each multi-GB downloads. +> **Minimum for autonomous mode:** one triage/narrator model — `openmythos-27b` (the bench winner, a +> GGUF import) *or* the pullable `qwen3:14b` fallback — **plus one** coder (`qwen3-coder:30b` or +> `qwen2.5-coder:14b`). See [AI Brain & Models](#ai-brain--models) and the benchmark record at +> [`docs/benchmarks/2026-07-16-triage-fp-discipline.md`](docs/benchmarks/2026-07-16-triage-fp-discipline.md). +> A pinned model that isn't installed now warns loudly (and fails under `BRAIN_REQUIRE_PIN=1`), so pin +> only models you've actually pulled/built. Optionally create the security-tuned triage model (fully optional — triage defaults to `phi4:14b` without it): @@ -278,16 +287,21 @@ role wants a different model — these are env-overridable with **no code change | Role | Model | Why | Env var | |:--|:--|:--|:--| -| **Narration / analysis** | `phi4:14b` | Lowest hallucination of any local model (Vectara 3.7 %) — won't fabricate findings | `BRAIN_MODEL=phi4:14b` | -| **Triage** (submit/drop) | `phi4:14b` *(default)* → `bugtraceai-apex` *(opt-in)* | Triage defaults to `phi4:14b` for speed and consistent JSON. Pull `bugtraceai-apex` and set `TRIAGE_MODEL` to switch to the security-DPO judge (empirically beat phi4 + Foundation-Sec on a triage A/B) | `TRIAGE_MODEL=bugtraceai-apex` | -| **Exploit code-gen** | `devstral-small-2:24b` (primary) / `qwen2.5-coder:14b` (fast fallback) | Both write valid, runnable PoCs. A/B-validated: Devstral wins on correctness (68 % SWE-bench; emits the canonical sqlmap-GET structure), qwen2.5-coder is faster/lighter | `BRAIN_SCANNER_MODEL=devstral-small-2:24b` | - -`brain_scanner.pick_model()` prefers `devstral-small-2:24b` automatically when -present, else falls back to `qwen2.5-coder:14b`. `brain.py` makes `phi4:14b` the -default narrator **and** the default triage model (`MODEL_PRIORITY[0]` and -`TRIAGE_MODEL_PRIORITY[0]`); `bugtraceai-apex` (which resolves to -`bugtraceai-apex:latest`) sits second in both lists and is only selected when -pulled or forced via the env var. +| **Triage** (submit/drop) | `openmythos-27b:latest` | 2026-07-16 false-positive-discipline bench winner (73-judge panel, 5/5 clean runs, 0 invented confirmations). Triage's decisive axis is **FP discipline** — not fabricating a vuln that isn't there. ⚠ Provisional: the bench measured false-positives only, not sensitivity — see the benchmark record. | `TRIAGE_MODEL=openmythos-27b:latest` | +| **Narration / analysis** | `openmythos-27b:latest` (pinned); code fallback `qwen3:14b` | Faithful, low-fabrication narration. Currently pinned to the triage winner; not separately narration-benched, so the code fallback stays a faithful generic. | `BRAIN_MODEL=openmythos-27b:latest` | +| **Exploit code-gen** | `qwen3-coder:30b` (pinned) / `qwen2.5-coder:14b` (fast fallback) | A coder **lands grounded PoCs** where triage/analysis models flail in the write-and-run loop. | `BRAIN_SCANNER_MODEL=qwen3-coder:30b` | + +Model selection is layered: **`~/.config/vikramaditya/brain.env` file-WINS** (the +canonical per-machine pin), and the `MODEL_PRIORITY` / `TRIAGE_MODEL_PRIORITY` +lists in `brain.py` are the code-level **fallback** (first *installed* model wins). +A pinned model that is **not installed** no longer silently substitutes — it warns +loudly (and fails under `BRAIN_REQUIRE_PIN=1`), and the model actually used is +recorded in `brain.MODEL_SELECTION_LOG`. The triage list is now ranked by FP +discipline (`openmythos-27b` first, then clean alternates `nemesis` / `devstral` / +`glm`; the offensive-tune `baron-llm` demoted). Full benchmark + caveats: +[`docs/benchmarks/2026-07-16-triage-fp-discipline.md`](docs/benchmarks/2026-07-16-triage-fp-discipline.md). +(Historical defaults `phi4` / `bugtraceai-apex` are retired — speed/JSON-optimized, +not FP-discipline-validated, and were not installed.) > ⚠️ A `claude-*` tag in your local Ollama is **not** Claude (Claude weights are > not downloadable, so any such tag is a mislabeled local model). Always confirm diff --git a/brain.py b/brain.py index 4c5a892..fb44600 100644 --- a/brain.py +++ b/brain.py @@ -1060,22 +1060,32 @@ def list_models(self) -> list[str]: "mlx-community/Mistral-7B-Instruct-v0.3-4bit", # 7B fallback ] -# Fast triage model priority — phi4:14b first (v9.1.3 benchmark winner) -# Used by triage_finding() and next_action() where speed > depth -# 03 May 2026 bench: phi4:14b T1=4.3s vs baron-llm 17s — 4× faster, 100% valid JSON +# Triage priority — ranked by FALSE-POSITIVE DISCIPLINE (dismiss scanner FPs; never invent/confirm a +# vuln that isn't there). This objective REVERSES the old "fastest JSON" ordering: speed is secondary. +# Ranking source: the 2026-07-16 "bench everything" 73-judge FP-discipline panel (5 runs/finalist) on a +# fixture with 3 known header-sqlmap FALSE-positives. openmythos-27b won 5/5 clean, 0 invented; the +# RavenX/CyberStrike offensive-tune class INVENTED confirmations; baron-llm (offensive RLHF) is demoted +# for the same reason. ⚠ PROVISIONAL: that panel had ZERO known TRUE-positives, so it did NOT measure +# false-NEGATIVES / sensitivity — a model that drops everything scores perfectly here. Re-validate with +# a true-positive fixture before treating this order as final. See docs/benchmarks/2026-07-16-triage-fp- +# discipline.md. Note: ~/.config/vikramaditya/brain.env file-WINS, and a set-but-uninstalled TRIAGE_MODEL +# no longer silently substitutes (see _pin_unavailable). Machine-specific tags below are skipped if absent. TRIAGE_MODEL_PRIORITY = [ - "phi4:14b", # ★ v9.1.3 — fastest triage, consistent JSON, no hidden thinking - "bugtraceai-apex:latest", # Zero-refusal security DPO reasoning model - "baron-llm:latest", # BaronLLM — RLHF on offensive security data - "aya-expanse:latest", # Cohere Aya Expanse 8B Multilingual flagship model - "gemma4:e4b", # Gemma 4 4B — fast triage with tool calling + "openmythos-27b:latest", # ★ 2026-07-16 WINNER — 5/5 clean, 0 invented confirmations, reasoning 4.8/5 (dense Qwen3.6-27B, RLVR on vulnerable/fixed code) + "nemesis-27b:latest", # clean alternate (2/2, n small) — dense Qwen3.6-27B offensive QLoRA that KEPT its FP-discipline + "devstral-small-2:latest", # clean alternate (3/3, n small) — coder with strong FP-discipline (coders judge logic, don't assume-vulnerable) + "glm47-flash:latest", # clean alternate (2/2, n small) — non-Qwen MoE, faster than the dense 27Bs + "phi4:14b", # faithful-narration generic (fast, consistent JSON) — if installed + "qwen3:14b", # faithful long-context generic — if installed + "gemma4:e4b", # fast small triage with tool calling — if installed "vapt-qwen25:latest", # custom VAPT-tuned fallback "vapt-model:latest", "qwen3:8b", + "baron-llm:latest", # DEMOTED (was #3): offensive RLHF tune — the 2026-07-16 bench flagged the offensive-tune class for INVENTING confirmations; keep only as a last-ditch triage judge "qwen3-coder-64k:latest", # last resort — big model for triage if nothing else - # xploiter/the-xploiter is intentionally NOT in this list — it is WEIGHT-biased - # to assert/fabricate vulns (see MODEL_PRIORITY comment) and the 7-Question Gate - # is a faithful-evaluation task. It remains an exploit-IDEATION-only fallback. + # DROPPED from triage (uninstalled AND not FP-discipline-selected): bugtraceai-apex, aya-expanse. + # xploiter/the-xploiter is intentionally NOT in this list — it is WEIGHT-biased to assert/fabricate + # vulns (see MODEL_PRIORITY comment) and the 7-Question Gate is a faithful-evaluation task. ] # Token limits — qwen3-coder-64k supports 64K context @@ -1168,35 +1178,112 @@ def _get_available_models() -> list[str]: return [] +# ── Model-selection provenance + NO silent substitution (friends review 2026-07-16) ───────────── +# HAZARD (codex+grok): an explicit pin (BRAIN_MODEL / TRIAGE_MODEL / BRAIN_SCANNER_MODEL) that is +# NOT installed used to be SILENTLY ignored — the picker fell through to the priority list and +# ultimately available[0]. For a client-facing vuln-triage gate that means "you pinned OpenMythos +# but you're actually running baron-llm and were never told." A missing pin must be LOUD, and under +# BRAIN_REQUIRE_PIN=1 (autonomous/client runs) FATAL — never a silent swap. MODEL_SELECTION_LOG +# records which model+source was actually used so a report can be audited afterwards. +MODEL_SELECTION_LOG: dict = {} # role -> {"model", "source", "requested_pin"} + + +def _require_pin() -> bool: + return os.environ.get("BRAIN_REQUIRE_PIN", "").strip().lower() in ("1", "true", "yes", "on") + + +def _record_selection(role: str, model, source: str, requested_pin: str = "") -> None: + MODEL_SELECTION_LOG[role] = {"model": model, "source": source, "requested_pin": requested_pin or ""} + + +def _match_installed(name: str, available: list) -> str | None: + """Resolve a model name against installed tags, honoring Ollama's implicit ``:latest``. + So a pin of ``openmythos-27b`` matches an installed ``openmythos-27b:latest`` (and vice-versa). + Returns the actual installed tag, or None if genuinely absent.""" + if not name: + return None + if name in available: + return name + if ":" not in name and f"{name}:latest" in available: # untagged pin -> :latest + return f"{name}:latest" + if name.endswith(":latest") and name[: -len(":latest")] in available: # :latest pin -> bare + return name[: -len(":latest")] + return None + + +# Models that must NEVER be the triage (false-positive) judge even as a last-resort fallback: +# weight-biased to assert/fabricate vulns, or dropped from triage on evidence. The triage fallback +# uses this instead of the unrestricted narrator priority (which still contains these). +_TRIAGE_FALLBACK_EXCLUDE = frozenset({ + "xploiter/the-xploiter:latest", # WEIGHT-biased to assert/fabricate — faithful-eval poison + "bugtraceai-apex:latest", # dropped from triage (offensive DPO, not FP-validated) + "aya-expanse:latest", # dropped from triage (multilingual chat, weak judge) +}) + + +def _pin_unavailable(role: str, envvar: str, requested: str, verifiable: bool = True) -> None: + """A pin is set but cannot be honored. Warn loudly; raise under BRAIN_REQUIRE_PIN. Never silently swap. + verifiable=True -> the model inventory was read and the pin is genuinely NOT installed. + verifiable=False -> the inventory could not be read (Ollama unreachable / empty) — pin UNVERIFIABLE.""" + if verifiable: + state, fix = "is NOT installed", f"`ollama pull {requested}` or correct the pin" + else: + state, fix = ("could NOT be verified (Ollama returned no models / is unreachable)", + "start Ollama / check OLLAMA_HOST, or correct the pin") + msg = (f"PINNED {role} model {requested!r} ({envvar}) {state} — refusing to SILENTLY substitute a " + f"different model. Fix: {fix} in ~/.config/vikramaditya/brain.env.") + if _require_pin(): + raise RuntimeError(f"[brain] {msg} (BRAIN_REQUIRE_PIN=1 -> will not run on a fallback model.)") + sys.stderr.write(f"{YELLOW}{BOLD}[!] {msg} Falling back — VERIFY the model actually used " + f"(brain.MODEL_SELECTION_LOG) before trusting client output.{NC}\n") + sys.stderr.flush() + + def _pick_model(preferred: str = None) -> str | None: """Return the best available model from priority list. v9.1.4 — env override: BRAIN_MODEL= forces a specific model (used by A/B benchmarks, per-engagement model swap without code edits). + v10.7 — a set-but-uninstalled BRAIN_MODEL no longer silently substitutes (see _pin_unavailable). """ available = _get_available_models() - if not available: - return None - # v9.1.4 env override takes precedence over caller's preferred arg + # v9.1.4 env override takes precedence over caller's preferred arg. Check the PIN FIRST — before the + # empty-inventory early-return — so a set-but-unhonorable pin is always loud/strict, even if Ollama + # returned nothing (the realistic outage case). :latest aliases resolve via _match_installed. env_override = os.environ.get("BRAIN_MODEL", "").strip() - if env_override and env_override in available: - return env_override + pin_missing = "" + if env_override: + match = _match_installed(env_override, available) if available else None + if match: + _record_selection("narrator", match, "pinned", env_override) + return match + _pin_unavailable("narrator", "BRAIN_MODEL", env_override, verifiable=bool(available)) + pin_missing = env_override + + if not available: + _record_selection("narrator", None, "no-models", pin_missing) + return None if preferred: - # exact match first - if preferred in available: - return preferred + match = _match_installed(preferred, available) + if match: + _record_selection("narrator", match, "pin-missing-fallback" if pin_missing else "preferred", pin_missing) + return match # prefix match (e.g. "qwen3" matches "qwen3:8b") matches = [m for m in available if m.startswith(preferred)] if matches: + _record_selection("narrator", matches[0], "pin-missing-fallback" if pin_missing else "preferred", pin_missing) return matches[0] for candidate in MODEL_PRIORITY: - if candidate in available: - return candidate + match = _match_installed(candidate, available) + if match: + _record_selection("narrator", match, "pin-missing-priority" if pin_missing else "priority", pin_missing) + return match # Last resort: first available model + _record_selection("narrator", available[0], "pin-missing-last-resort" if pin_missing else "last-resort", pin_missing) return available[0] @@ -1258,22 +1345,45 @@ def _technique_hint(finding_description: str) -> str: def _pick_triage_model(preferred: str = None) -> str | None: - """Return the best fast triage model — prefers BaronLLM when installed. + """Return the best triage model — ranked by FALSE-POSITIVE DISCIPLINE (see TRIAGE_MODEL_PRIORITY). v9.1.4 — TRIAGE_MODEL= env var overrides for A/B testing. + v10.7 — a set-but-uninstalled TRIAGE_MODEL no longer silently substitutes (see _pin_unavailable). """ available = _get_available_models() + + env_override = os.environ.get("TRIAGE_MODEL", "").strip() + pin_missing = "" + if env_override: + match = _match_installed(env_override, available) if available else None + if match: + _record_selection("triage", match, "pinned", env_override) + return match + _pin_unavailable("triage", "TRIAGE_MODEL", env_override, verifiable=bool(available)) + pin_missing = env_override + if not available: + _record_selection("triage", None, "no-models", pin_missing) return None - env_override = os.environ.get("TRIAGE_MODEL", "").strip() - if env_override and env_override in available: - return env_override - if preferred and preferred in available: - return preferred + + if preferred: + match = _match_installed(preferred, available) + if match: + _record_selection("triage", match, "pin-missing-fallback" if pin_missing else "preferred", pin_missing) + return match for candidate in TRIAGE_MODEL_PRIORITY: - if candidate in available: - return candidate - return _pick_model() # fall back to analysis model + match = _match_installed(candidate, available) + if match: + _record_selection("triage", match, "pin-missing-priority" if pin_missing else "priority", pin_missing) + return match + # No configured triage model installed. Fall back to a triage-SAFE available model — NOT the + # unrestricted narrator priority (_pick_model), which still contains assert/invent-biased tags + # (xploiter/bugtraceai/aya). Disable triage (return None) rather than let a fabrication-biased + # model become the false-positive judge. This also avoids clobbering the narrator provenance entry. + safe = [m for m in available if m not in _TRIAGE_FALLBACK_EXCLUDE] + m = safe[0] if safe else None + _record_selection("triage", m, ("pin-missing-" if pin_missing else "") + ("safe-fallback" if m else "none-safe"), pin_missing) + return m class Brain: diff --git a/brain_scanner.py b/brain_scanner.py index 89de59f..4b0ff5e 100644 --- a/brain_scanner.py +++ b/brain_scanner.py @@ -58,6 +58,18 @@ except Exception: _scopeguard = None +# ONE-TIME (at import): apply ~/.config/vikramaditya/brain.env (file-wins) so a STANDALONE +# `python3 brain_scanner.py ...` honors the pin vars. brain.py does this at its own import, but +# brain_scanner imports brain only lazily — without this a direct run would read BRAIN_SCANNER_MODEL / +# BRAIN_PROVIDER before brain.env is applied. Done ONCE at import (NOT per pick_model call) so a +# caller/test can still override via process env afterward, and so it can't clobber a deliberate +# BRAIN_PROVIDER override. Respects BRAIN_ENV_NOLOAD (the loader is a no-op then). +try: + from brain import _load_brain_env as _bootstrap_brain_env + _bootstrap_brain_env() +except Exception: + pass + # Colors G = "\033[0;32m" R = "\033[0;31m" @@ -184,6 +196,15 @@ def pick_model() -> str: coders. Override with BRAIN_SCANNER_MODEL=. """ import os as _os + + def _is_model_not_found(exc) -> bool: + """True only if an ollama.show() error means the model is ABSENT (not a network/daemon failure), + so a transient outage isn't mislabelled 'pull the model' / doesn't wrongly hard-fail strict mode.""" + if getattr(exc, "status_code", None) == 404: + return True + s = str(exc).lower() + return any(t in s for t in ("not found", "no such model", "try pulling", "does not exist")) + env = _os.environ.get("BRAIN_SCANNER_MODEL", "").strip() prov = _os.environ.get("BRAIN_PROVIDER", "").strip().lower() # Cloud / non-ollama provider (gemini/openai/claude/grok/mlx): the model name @@ -200,9 +221,31 @@ def pick_model() -> str: return env or LLMClient.DEFAULT_MODELS.get(prov) or "" except Exception: return env or "" + def _record(model, source, pin=""): + try: + from brain import _record_selection + _record_selection("scanner", model, source, pin) + except Exception: + pass try: import ollama - candidates = ([env] if env else []) + [ + # A set-but-uninstalled BRAIN_SCANNER_MODEL must NOT silently fall through to another coder + # (friends review 2026-07-16). Check the pin explicitly; warn (or raise under BRAIN_REQUIRE_PIN). + pin_missing = "" + if env: + try: + ollama.show(env) + _record(env, "pinned", env) + return env + except Exception as _e: + try: + from brain import _pin_unavailable + # not-found -> "pull it"; network/daemon error -> "cannot verify" (don't mislead) + _pin_unavailable("scanner", "BRAIN_SCANNER_MODEL", env, verifiable=_is_model_not_found(_e)) + except ImportError: + pass + pin_missing = env + candidates = [ "devstral-small-2:24b", # agentic SWE coder (68% SWE-bench) if pulled "qwen2.5-coder:14b", # installed, fast, purpose-built coder "qwen3-coder:30b", # installed, stronger MoE coder @@ -215,6 +258,7 @@ def pick_model() -> str: continue try: ollama.show(m) + _record(m, "pin-missing-priority" if pin_missing else "priority", pin_missing) return m except Exception: continue diff --git a/docs/benchmarks/2026-07-16-triage-fp-discipline.md b/docs/benchmarks/2026-07-16-triage-fp-discipline.md new file mode 100644 index 0000000..12333a9 --- /dev/null +++ b/docs/benchmarks/2026-07-16-triage-fp-discipline.md @@ -0,0 +1,101 @@ +# Triage FP-Discipline Benchmark — 2026-07-16 ("bench everything") + +**One-line result:** `openmythos-27b` is the new triage default. The whole RavenX line invents +confirmations and was retired from triage. **This is a PROVISIONAL result — it measured +false-positives only, never sensitivity (false-negatives).** + +## What was measured + +The tool's triage phase (`brain.py --phase scan`) decides which scanner findings are real vs +false-positives. The decisive quality axis is **false-positive discipline**: dismiss scanner noise, +and *never invent/confirm a vulnerability that isn't there* (the worst failure for a client report). + +- **Harness:** `brain_model_bench.py` replays the scan/interpretation phase per model (env override + via `--model`, `BRAIN_ENV_NOLOAD=1`). Its built-in SUBMIT/DROP tally is broken for the narrative + scan phase, so scoring was done by an independent **73-agent judge panel** (3 blind judges per + interpretation) reading each `02_scan_interpretation.md`. Judge script: `scratchpad/judge_panel.js`. +- **Fixture:** a client recon session with **3 known header-based sqlmap false-positives** + (Referer/User-Agent "SQLi" on static marketing pages) + a missing-CSP finding. + **`Verified SQLi/RCE PoCs = 0` → the fixture contains ZERO true-positive vulnerabilities.** +- **Sampling:** 5 runs each for finalists (RavenX v5.1, OpenMythos, CyberStrike), 2–3 for screening + candidates. `brain.py` scan temperature = 0.1 → non-deterministic, hence multiple runs. +- **"Clean" run** = dismissed all 3 FPs, 0 invented confirmations, 0 hallucinations. + +## Leaderboard + +| # | Model | Runs | Clean | FPs dismissed /3 | Invents? | Hallucinates? | Reasoning /5 | +|---|-------|------|-------|------------------|----------|---------------|--------------| +| 1 | **openmythos-27b** (dense Qwen3.6-27B, RLVR vuln/fixed) | 5 | **5/5** | 3.00 | no | no | **4.80** | +| 2 | nemesis-27b (dense Qwen3.6-27B offensive QLoRA) | 2 | 2/2 | 3.00 | no | no | 4.50 | +| 3 | devstral-small-2 (coder) | 3 | 3/3 | 3.00 | no | no | 4.00 | +| 4 | glm47-flash (non-Qwen MoE) | 2 | 2/2 | 3.00 | no | no | 3.50 | +| 5 | ravenx-cyberagent v5.1 (former default) | 5 | 2/5 | 2.40 | **yes** | no | 3.00 | +| 6 | ravenx-v6.2 (successor) | 2 | 0/2 | 2.00 | **yes** | no | 3.00 | +| 7 | cyberstrike-35b (abliterated offensive) | 5 | 0/5 | 1.40 | yes | **yes** | 2.00 | +| — | ernie-4.5-21b-a3b-thinking | — | rejected — thinking-leak; breaks brain.py ('message' KeyError → empty output) | +| — | ornith-1.0-35b | — | rejected — thinking-leak; empty in the harness | + +## Findings + +1. **openmythos-27b wins** — the only model perfect at full sample (5/5, best reasoning, 0 invention). +2. **The whole RavenX line invents confirmations** (v5.1 AND v6.2) → retired from triage. The successor + is no fix. +3. **Counterintuitive:** general coders (devstral, glm) and a dense offensive QLoRA (nemesis) all have + *perfect* FP discipline, beating the security-tuned RavenX/CyberStrike. Abliteration / aggressive + offensive-SFT **hurts** FP discipline; dense reasoning keeps it. + +## ⚠ Caveats — do NOT over-read this + +- **No false-negative measurement.** The fixture has 0 true-positives, so a model that dismisses + *everything* scores perfectly here. OpenMythos's 5/5 could partly be over-conservatism, and this + benchmark **cannot detect it**. FP discipline ✅ measured; **sensitivity ❌ not measured.** +- **One fixture, one FP family** (header-based sqlmap). Other FP types (WAF-bounce, timing, reflected + XSS candidates, upload/deser markers) untested. +- **n = 5** finalists / n ≤ 3 screeners — few behavioral trials (73 judges improve *scoring* confidence, + not sample size). +- **The actual code fallbacks (qwen3:14b, baron-llm) were not in the panel** — so this is not a direct + head-to-head justifying their replacement, only the *pin*. +- **Tags are mutable/machine-local** (`openmythos-27b:latest` here = a specific GGUF import; `:latest` + may drift). + +## Decisions taken (2026-07-16 / 07-17) + +- `~/.config/vikramaditya/brain.env` pinned: triage+narrator → `openmythos-27b:latest`, scanner → + `qwen3-coder:30b` (unchanged). Verified live end-to-end. +- `brain.py TRIAGE_MODEL_PRIORITY` reordered (FP-discipline; openmythos first, clean alternates, + baron demoted) — **provisional, pending the re-bench below.** `MODEL_PRIORITY` (narrator) and the + scanner list were **left unchanged** (triage-only evidence). +- **Safety fix (no silent model substitution):** a set-but-unhonorable pin (`BRAIN_MODEL` / + `TRIAGE_MODEL` / `BRAIN_SCANNER_MODEL`) no longer silently substitutes — warns loudly, records + `brain.MODEL_SELECTION_LOG`, and raises under `BRAIN_REQUIRE_PIN=1`. Hardened after a codex+grok diff + review: honors Ollama `:latest` aliases; the empty/unreachable inventory case no longer bypasses the + pin/strict check; the triage fallback never re-introduces excluded/assert-biased models (`xploiter` / + `bugtraceai` / `aya`) — it disables triage instead; the scanner distinguishes model-not-found from a + network/daemon outage; standalone `brain_scanner.py` now loads `brain.env`; and `hunt.py` propagates + the strict failure instead of swallowing it. Tests: `tests/test_brain_model_selection_safety.py` (16). +- **Known follow-ups (from the same review, not yet done):** + 1. `MODEL_SELECTION_LOG` is process-local (stderr + in-memory) — **not yet persisted** into the + session tree / report, so post-hoc report audit of the model used still needs wiring. + 2. The standalone **`agent.py` ReAct selector** (`_pick_tool_capable_model`) is **not yet pin-aware**, + so `BRAIN_REQUIRE_PIN` covers `brain.py` + `brain_scanner.py` but not the `--agent` path's own model + choice (a dedicated `BRAIN_AGENT_MODEL` + strict propagation is the fix). + +## Validation gate (re-bench before making the triage order final / non-provisional) + +Rerun `brain_model_bench.py` + judge panel with: +1. **A fixture that includes KNOWN TRUE-POSITIVES** across multiple vuln classes (SQLi, IDOR, RCE, + exposure) so **sensitivity / false-negatives** are measured, alongside FP families beyond header-sqlmap. +2. The **actual fallbacks** in the panel: `qwen3:14b`, `baron-llm`, plus `phi4` if pulled. +3. ≥ 5 runs per serious candidate; score FP-rate **and** FN-rate, JSON validity, latency, invented-evidence. +4. A separate **narration** benchmark before touching `MODEL_PRIORITY[0]`. + +Only if OpenMythos still wins on the combined FP+FN score → promote to `TRIAGE_MODEL_PRIORITY[0]` +non-provisionally and update this record. + +## Reproduce + +Bench helpers (in the session scratchpad, not committed): `dl_import.sh` (aria2c download + sha256 + +`ollama create`), `bench_triage.sh` (N brain_model_bench runs), `judge_panel.js` (3-judge panel). +Fixture: a client recon session with 3 header-based sqlmap false-positives (the exact +`findings//sessions/` path is kept out of the repo per client confidentiality — it lives +only in the operator's local `recon/`/`findings/` tree). diff --git a/docs/superpowers/backlog/2026-07-03-arsenal-modernization.md b/docs/superpowers/backlog/2026-07-03-arsenal-modernization.md index 087d879..3cbe74d 100644 --- a/docs/superpowers/backlog/2026-07-03-arsenal-modernization.md +++ b/docs/superpowers/backlog/2026-07-03-arsenal-modernization.md @@ -75,7 +75,11 @@ paradigm-fit, anti-fabrication over new scanners. dnsgen/cero (GPL + redundant). Covered the permissive way via nuclei `-dast` OOB. - **mantra/getjs** — regex noise, dominated by jsluice + katana/gau. - **Deep Hat V2** — proprietary (Kindo), no open weights → would off-box client data. - Keep **RavenX** pinned (no open model beats it on a 30 GB M4 Max as of 2026-07). + ~~Keep **RavenX** pinned (no open model beats it on a 30 GB M4 Max as of 2026-07).~~ + **SUPERSEDED 2026-07-16:** a 73-judge FP-discipline bench over 7 local models found **RavenX + INVENTS confirmations** (v5.1 2/5 clean, v6.2 0/2) — retired from triage. Winner = + **`openmythos-27b`** (5/5 clean, 0 invented). Triage now pinned to OpenMythos; scanner stays + `qwen3-coder:30b`. See [`docs/benchmarks/2026-07-16-triage-fp-discipline.md`](../../benchmarks/2026-07-16-triage-fp-discipline.md). - **xOffense** — weights unreleased (paper only). **CAI/HexStrike** — already clean-room adopted in PR #9 (egress-guard / tool_router). - **Semgrep Secrets** (paid Pro), **agentic-radar / Giskard** (SAST/wrapper, off-paradigm). diff --git a/hunt.py b/hunt.py index df784ef..712d2db 100644 --- a/hunt.py +++ b/hunt.py @@ -166,6 +166,11 @@ def init_brain(log_errors: bool = True): try: _brain = Brain() except Exception as exc: + # BRAIN_REQUIRE_PIN=1 (client/autonomous strict mode): a pinned-but-unavailable model raises + # from the selector. That MUST hard-fail the run — NOT be swallowed into "brain disabled" and + # continue on a different model (or the agent's own selector). Propagate under strict mode. + if os.environ.get("BRAIN_REQUIRE_PIN", "").strip().lower() in ("1", "true", "yes", "on"): + raise _brain_import_err = exc if log_errors and not _brain_warned: print(f"\033[1;33m[!] Brain not loaded: {_brain_import_err}\033[0m") diff --git a/tests/test_brain_model_selection_safety.py b/tests/test_brain_model_selection_safety.py new file mode 100644 index 0000000..f5701b7 --- /dev/null +++ b/tests/test_brain_model_selection_safety.py @@ -0,0 +1,201 @@ +"""Model-selection must NOT silently substitute a different model for an explicit pin. + +ROOT CAUSE (friends review, 2026-07-16): if BRAIN_MODEL / TRIAGE_MODEL is set but the pinned +tag is NOT installed, `_pick_model` / `_pick_triage_model` silently fell through to the priority +list (and ultimately `available[0]`). For a client-facing vuln-triage gate that silently means +"you pinned OpenMythos but you're actually running baron-llm and were never told." A pinned model +that is missing must be LOUD, and — under BRAIN_REQUIRE_PIN=1 (client/autonomous runs) — FATAL. +Selection provenance (which model + why) is recorded so a report can be audited later. +""" +import sys +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parent.parent +if str(REPO) not in sys.path: + sys.path.insert(0, str(REPO)) + +import brain # noqa: E402 + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + for k in ("BRAIN_MODEL", "TRIAGE_MODEL", "BRAIN_REQUIRE_PIN"): + monkeypatch.delenv(k, raising=False) + brain.MODEL_SELECTION_LOG.clear() + yield + + +def _avail(monkeypatch, models): + monkeypatch.setattr(brain, "_get_available_models", lambda: list(models)) + + +# ── pin present & installed ───────────────────────────────────────────────── +def test_pin_installed_is_used_and_recorded(monkeypatch): + _avail(monkeypatch, ["openmythos-27b:latest", "qwen3:14b"]) + monkeypatch.setenv("BRAIN_MODEL", "openmythos-27b:latest") + assert brain._pick_model() == "openmythos-27b:latest" + assert brain.MODEL_SELECTION_LOG["narrator"]["source"] == "pinned" + + +# ── pin set but NOT installed → must NOT silently substitute ───────────────── +def test_narrator_pin_missing_warns_and_records_fallback(monkeypatch, capsys): + _avail(monkeypatch, ["qwen3:14b", "baron-llm:latest"]) # pinned model absent + monkeypatch.setenv("BRAIN_MODEL", "openmythos-27b:latest") + picked = brain._pick_model() + assert picked != "openmythos-27b:latest" # it fell back + err = capsys.readouterr().err + assert "openmythos-27b:latest" in err and "NOT installed" in err # LOUD warning + prov = brain.MODEL_SELECTION_LOG["narrator"] + assert prov["requested_pin"] == "openmythos-27b:latest" + assert prov["source"] in ("pin-missing-fallback", "pin-missing-priority", "pin-missing-last-resort") + assert prov["model"] == picked + + +def test_triage_pin_missing_warns(monkeypatch, capsys): + _avail(monkeypatch, ["qwen3:14b", "baron-llm:latest"]) + monkeypatch.setenv("TRIAGE_MODEL", "openmythos-27b:latest") + picked = brain._pick_triage_model() + assert picked != "openmythos-27b:latest" + assert "NOT installed" in capsys.readouterr().err + assert brain.MODEL_SELECTION_LOG["triage"]["requested_pin"] == "openmythos-27b:latest" + + +# ── strict mode: pin missing is FATAL ─────────────────────────────────────── +def test_strict_narrator_pin_missing_raises(monkeypatch): + _avail(monkeypatch, ["qwen3:14b"]) + monkeypatch.setenv("BRAIN_MODEL", "openmythos-27b:latest") + monkeypatch.setenv("BRAIN_REQUIRE_PIN", "1") + with pytest.raises(RuntimeError, match="openmythos-27b:latest"): + brain._pick_model() + + +def test_strict_triage_pin_missing_raises(monkeypatch): + _avail(monkeypatch, ["qwen3:14b"]) + monkeypatch.setenv("TRIAGE_MODEL", "openmythos-27b:latest") + monkeypatch.setenv("BRAIN_REQUIRE_PIN", "1") + with pytest.raises(RuntimeError, match="openmythos-27b:latest"): + brain._pick_triage_model() + + +# ── no pin → priority list, provenance recorded, no false warning ─────────── +def test_no_pin_uses_priority_and_records(monkeypatch, capsys): + _avail(monkeypatch, ["qwen3:14b", "baron-llm:latest"]) + picked = brain._pick_model() + assert picked in ("qwen3:14b", "baron-llm:latest") + assert "NOT installed" not in capsys.readouterr().err # no spurious warning when no pin set + assert brain.MODEL_SELECTION_LOG["narrator"]["source"] in ("priority", "last-resort") + + +# ── review fix: :latest alias must match (grok#2 / codex#5) ────────────────── +def test_pin_untagged_matches_installed_latest(monkeypatch, capsys): + _avail(monkeypatch, ["openmythos-27b:latest", "qwen3:14b"]) + monkeypatch.setenv("BRAIN_MODEL", "openmythos-27b") # pinned WITHOUT :latest + assert brain._pick_model() == "openmythos-27b:latest" # must resolve, not fall back + assert "NOT installed" not in capsys.readouterr().err + assert brain.MODEL_SELECTION_LOG["narrator"]["source"] == "pinned" + + +def test_pin_untagged_alias_not_wrongly_raised_in_strict(monkeypatch): + _avail(monkeypatch, ["openmythos-27b:latest"]) + monkeypatch.setenv("BRAIN_MODEL", "openmythos-27b") + monkeypatch.setenv("BRAIN_REQUIRE_PIN", "1") + assert brain._pick_model() == "openmythos-27b:latest" # alias present → no raise + + +# ── review fix: empty/unreachable inventory must not bypass pin/strict (grok#1 / codex#4) ── +def test_empty_inventory_strict_raises(monkeypatch): + _avail(monkeypatch, []) + monkeypatch.setenv("BRAIN_MODEL", "openmythos-27b:latest") + monkeypatch.setenv("BRAIN_REQUIRE_PIN", "1") + with pytest.raises(RuntimeError): + brain._pick_model() + + +def test_empty_inventory_warns_and_returns_none(monkeypatch, capsys): + _avail(monkeypatch, []) + monkeypatch.setenv("TRIAGE_MODEL", "openmythos-27b:latest") + assert brain._pick_triage_model() is None + assert "could NOT be verified" in capsys.readouterr().err + + +# ── review fix: triage fallback must NOT re-introduce excluded/assert-biased models (codex#1) ── +def test_triage_fallback_never_uses_xploiter(monkeypatch): + _avail(monkeypatch, ["xploiter/the-xploiter:latest"]) # only an assert-biased model installed + assert brain._pick_triage_model() is None # disable triage, do NOT use xploiter + + +def test_triage_fallback_uses_safe_available(monkeypatch): + _avail(monkeypatch, ["xploiter/the-xploiter:latest", "some-generic:latest"]) + assert brain._pick_triage_model() == "some-generic:latest" # skips excluded, uses safe one + + +# ── review fix: triage fallback must NOT overwrite narrator provenance (grok#5 / codex#7) ── +def test_triage_fallback_preserves_narrator_provenance(monkeypatch): + _avail(monkeypatch, ["custom-narrator:latest", "some-generic:latest"]) + monkeypatch.setenv("BRAIN_MODEL", "custom-narrator:latest") + assert brain._pick_model() == "custom-narrator:latest" # narrator recorded + brain._pick_triage_model() # triage falls back + assert brain.MODEL_SELECTION_LOG["narrator"]["model"] == "custom-narrator:latest" # NOT clobbered + + +# ── review fix: scanner pin safety (grok#3/#8, codex#6) ───────────────────── +import types + + +def _fake_ollama(installed): + fake = types.ModuleType("ollama") + + class _NotFound(Exception): + status_code = 404 + + def show(m): + if m in installed: + return {} + raise _NotFound(f"model '{m}' not found, try pulling it first") + + def show_neterr(m): + raise ConnectionError("connection refused") # daemon down, NOT a missing model + + fake.show = show + fake._show_neterr = show_neterr + return fake + + +def test_scanner_pin_missing_warns_and_falls_to_coder(monkeypatch, capsys): + import sys as _sys + monkeypatch.setitem(_sys.modules, "ollama", _fake_ollama({"qwen2.5-coder:14b"})) + monkeypatch.setenv("BRAIN_ENV_NOLOAD", "1") + monkeypatch.delenv("BRAIN_PROVIDER", raising=False) + monkeypatch.delenv("BRAIN_REQUIRE_PIN", raising=False) + monkeypatch.setenv("BRAIN_SCANNER_MODEL", "ghost:latest") + import brain_scanner + assert brain_scanner.pick_model() == "qwen2.5-coder:14b" # fell to a real coder, not silent + assert "NOT installed" in capsys.readouterr().err + + +def test_scanner_pin_missing_strict_raises(monkeypatch): + import sys as _sys + monkeypatch.setitem(_sys.modules, "ollama", _fake_ollama({"qwen2.5-coder:14b"})) + monkeypatch.setenv("BRAIN_ENV_NOLOAD", "1") + monkeypatch.delenv("BRAIN_PROVIDER", raising=False) + monkeypatch.setenv("BRAIN_SCANNER_MODEL", "ghost:latest") + monkeypatch.setenv("BRAIN_REQUIRE_PIN", "1") + import brain_scanner + with pytest.raises(RuntimeError, match="ghost:latest"): + brain_scanner.pick_model() + + +def test_scanner_network_error_is_cannot_verify_not_not_installed(monkeypatch, capsys): + import sys as _sys + fake = _fake_ollama({"qwen2.5-coder:14b"}) + fake.show = fake._show_neterr # every show() = ConnectionError (daemon down) + monkeypatch.setitem(_sys.modules, "ollama", fake) + monkeypatch.setenv("BRAIN_ENV_NOLOAD", "1") + monkeypatch.delenv("BRAIN_PROVIDER", raising=False) + monkeypatch.delenv("BRAIN_REQUIRE_PIN", raising=False) + monkeypatch.setenv("BRAIN_SCANNER_MODEL", "qwen2.5-coder:14b") # actually installed, but unreachable + import brain_scanner + brain_scanner.pick_model() + assert "could NOT be verified" in capsys.readouterr().err # not the misleading "pull the model"