feat: Migrate to OpenAI-compatible backend + scope guard, evidence & verified-only learning

CHANGELOG.txt

@@ -1,5 +1,64 @@
 # Changelog
 
+## [v1.7.1-beta] - 2026-06-17
+
+### Backend: migrated to OpenAI-compatible (no more Ollama-only)
+- feat(llm): AIRecon now drives ANY OpenAI-compatible gateway (LiteLLM / vLLM / hosted / a local
+  gateway) via `openai_base_url` / `openai_api_key` / `openai_model`. The client speaks the OpenAI
+  /v1/chat/completions wire format directly over httpx (no `openai` SDK dependency).
+- feat(llm): reasoning is auto-detected at runtime — `auto` sends `reasoning_effort` and, on an
+  HTTP 400 "unsupported parameter", strips it and remembers per-model. No model-name hardcoding.
+- chore: removed the dead `ollama` dependency from pyproject.toml; renamed all `ollama_*` config
+  keys to `openai_*` / `llm_*`; removed every `ollama`/`9router` identifier and brand mention from
+  code, generated config.yaml, README and docs.
+
+### De-hardcoded recon/analysis/exploit
+- refactor: removed model/provider/technique name special-casing. Tech-payload augmentation,
+  url->priority-params, chain follow-ons, and the prelude expert hints are now data-driven
+  (fuzzer_data.json) or removed; context-pressure/compaction thresholds key off the effective
+  context window, not a `qwen`/`122b` name check.
+- refactor(novel_discovery): replaced random-canned tactics with target-specific, evidence-derived
+  + optional LLM-driven attack-vector generation.
+
+### Verification, evidence & reporting
+- feat(verify): VerificationEngine wired into the LLM finding path (type-aware, non-monotonous);
+  expanded confirmators (open_redirect / ssrf / xxe).
+- feat(report): persist machine-captured request/response evidence per finding (<slug>.evidence.json),
+  finding lifecycle labels (VALIDATED / SUSPECTED / INFORMATIONAL), CWE/OWASP classification, and a
+  Verification section in the .md.
+
+### Intelligence & memory (the brain)
+- feat: cross-session brain learns ONLY from verified/high-confidence findings
+  (`intelligence_learn_only_verified`) — compounds proven knowledge, not false positives.
+- feat: cold-start dataset knowledge (tech_correlations) injected from iteration 1; memory/dataset
+  recall is config-driven and more active (recall every 4 iters, correlation every 6).
+- feat(memory): compaction preserves protected context by-type; rolling-summary / compressor caps
+  scale with context window.
+
+### Platform & operability
+- feat(scope): scope guard + persistent audit log (~/.airecon/audit/audit.jsonl); enforced on the
+  agent `execute` path AND `/shell`; live management via the new `/scope` TUI command + `/api/scope`.
+- feat(config): data-driven scan profiles (quick/standard/deep/stealth/ctf/bugbounty).
+- feat(status): real sandbox tool-health probing + scope/scan-profile surfaced; progress timeline
+  (phase/last tool/last command/stuck); `/api/models` per-model reasoning capability.
+- feat(notify): completion webhook + COMPLETE.json flag.
+- feat(resume): session keeps a raw chat/tool-turn buffer so resume replays real history; `/api/history`
+  surfaces prior progress even after compaction.
+- feat(config): all new keys are now written into the generated config.yaml with sections/comments.
+
+### Fixes
+- fix(config): thread-safe `Config.load()` via atomic write (race could reset config to defaults).
+- fix(tui): quit (Ctrl+C -> yes) no longer hangs — stream worker cancelled, network calls bounded,
+  exit() always reached.
+- fix(docker): entrypoint no longer `chown -R /workspace` (which clobbered host files); only the
+  mount-point is made writable.
+- fix(startup): proxy import/startup failures now write the crash log and show the full path + reason.
+- chore: routed `distill_insights` through LLMClient; removed dead locals; fixed bare-except violations.
+
+### Validation
+- ~2200 tests passing across proxy/agent/tui suites; pyflakes 0 undefined names; package imports
+  clean; generated config.yaml free of ollama/9router.
+
 ## [v0.1.7-beta] - 2026-04-02
 
 ### Added

airecon/__main__.py

@@ -249,27 +249,27 @@ def _row(content: str = "") -> str:
         print(_row(f"  {D}v{version} — AI-Powered Security Reconnaissance{X}"))
         print(f"  {C}╠{'═' * W}╣{X}")
 
-        ollama_status = OFF
+        llm_status = OFF
         model_names: list[str] = []
-        active_model = cfg.ollama_model
+        active_model = cfg.openai_model
         try:
+            _headers = {}
+            if cfg.openai_api_key:
+                _headers["Authorization"] = f"Bearer {cfg.openai_api_key}"
             async with httpx.AsyncClient(timeout=5.0) as client:
-                try:
-                    resp = await client.get(f"{cfg.ollama_url}/api/tags")
-                    resp.raise_for_status()
-                except Exception:
-                    resp = await client.get(f"{cfg.ollama_url}/api-tags")
-                    resp.raise_for_status()
-
-                models = resp.json().get("models", [])
-                model_names = [m.get("name", "") for m in models if isinstance(m, dict)]
-                ollama_status = ON
+                resp = await client.get(
+                    f"{cfg.openai_base_url.rstrip('/')}/models", headers=_headers
+                )
+                resp.raise_for_status()
+                models = resp.json().get("data", [])
+                model_names = [m.get("id", "") for m in models if isinstance(m, dict)]
+                llm_status = ON
         except Exception as e:
-            logger.debug("Ollama status check failed: %s", e)
+            logger.debug("LLM status check failed: %s", e)
 
         print(_row())
-        print(_row(f"  {B}Ollama{X}        {ollama_status}"))
-        print(_row(f"  {D}Endpoint:{X}     {cfg.ollama_url}"))
+        print(_row(f"  {B}LLM Gateway{X}   {llm_status}"))
+        print(_row(f"  {D}Endpoint:{X}     {cfg.openai_base_url}"))
         print(_row(f"  {D}Active Model:{X} {Y}{active_model}{X}"))
         if model_names:
             label = f"  {D}Available:{X}    "
@@ -450,7 +450,6 @@ def _fmt_tokens(n: int) -> str:
         return
 
     proxy_url = f"http://{cfg.proxy_host}:{cfg.proxy_port}"
-    model = cfg.ollama_model
     _docker = shutil.which("docker") or "docker"
     session_info: dict = {}
     agent_stats: dict = {}
@@ -577,31 +576,7 @@ def _fmt_tokens(n: int) -> str:
     except Exception as e:
         logger.debug("SearXNG shutdown failed: %s", e)
 
-    print(_row(f"  {D}Unloading model…{X}"), end="\r", flush=True)
-    try:
-        ollama_url = cfg.ollama_url.rstrip("/")
-        cmd = [
-            "curl",
-            "-s",
-            "-X",
-            "POST",
-            f"{ollama_url}/api/generate",
-            "-d",
-            json.dumps({"model": model, "keep_alive": 0}),
-        ]
-        subprocess.run(  # nosec B603
-            cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=2
-        )
-    except Exception:
-        try:
-            data = json.dumps({"model": model, "keep_alive": 0}).encode()
-            req = urllib.request.Request(
-                f"{cfg.ollama_url.rstrip('/')}/api/generate", data=data, method="POST"
-            )
-            urllib.request.urlopen(req, timeout=2)  # nosec B310
-        except Exception as e:
-            logger.debug("Fallback unload via urllib failed: %s", e)
-    print(_svc(f"{G}✓{X}", "Ollama model", "VRAM released"))
+    # Remote OpenAI-compatible gateways are stateless — no local VRAM to release.
 
     print(_row())
     print(f"  {C}╚{'═' * (W + 2)}╝{X}")

airecon/_version.py

@@ -5,4 +5,4 @@
 
     __version__ = version("airecon")
 except Exception:
-    __version__ = "0.1.7-beta"
+    __version__ = "1.7.1-beta"

airecon/containers/Dockerfile

@@ -129,7 +129,7 @@ RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest && \
     go install -v github.com/tomnomnom/anew@latest 2>/dev/null || true
 
 # ── Copy Go binaries to /usr/local/bin so they are always in default PATH ──
-# Ollama subprocess calls (which/command -v) don't always inherit custom PATH
+# LLM gateway subprocess calls (which/command -v) don't always inherit custom PATH
 USER root
 RUN cp -n /home/pentester/go/bin/* /usr/local/bin/ 2>/dev/null || true && \
     # stratus-red-team Go install produces binary 'stratus'; alias to canonical meta name

airecon/containers/docker-entrypoint.sh

@@ -28,11 +28,20 @@ echo "[airecon-sandbox] Tools ready at: $(date)"
 #    --ignore-certificate-errors \
 #    > /dev/null 2>&1 &
 
-# Keep container alive
+# Ensure the sandbox user can create its output subdirectories WITHOUT rewriting
+# the ownership/permissions of existing files inside the host bind-mount.
+# A recursive chown/chmod here would clobber the user's real files (git objects,
+# secrets, source, @-referenced copies) on the host — so we only adjust the
+# mount-point directory itself (non-recursive). AIRecon creates and owns its own
+# per-target output subdirs (output/, command/, tools/, vulnerabilities/), which
+# are therefore writable without touching anything that was already there.
 if [ -d "/workspace" ]; then
-    sudo chown -R pentester:pentester /workspace 2>/dev/null || true
-    sudo chmod -R 775 /workspace 2>/dev/null || true
-    echo "[airecon-sandbox] Workspace permissions fixed."
+    # Make the mount point group-writable + group-owned by the sandbox user so
+    # the agent can mkdir its target folders. NOT recursive — existing host
+    # files keep their original ownership and permissions.
+    sudo chown pentester:pentester /workspace 2>/dev/null || true
+    sudo chmod 0775 /workspace 2>/dev/null || true
+    echo "[airecon-sandbox] Workspace mount-point writable (existing files untouched)."
 fi
 
 # Keep container alive

airecon/proxy/agent/adaptive_learning.py

@@ -852,14 +852,17 @@ def record_strategy_result(
 
     # ── Abstraction (LLM-assisted distillation) ──────────────────────────────
 
-    def distill_insights(self, ollama_url: str = "", model: str = "") -> list[LearnedInsight]:
-        """Ask Ollama to abstract patterns from recent observations.
+    def distill_insights(
+        self, base_url: str = "", model: str = "", api_key: str = ""
+    ) -> list[LearnedInsight]:
+        """Ask the LLM to abstract patterns from recent observations.
 
         Returns newly created insights (not the full catalog).
         This is how airecon 'learns' — raw data → generalized rules via LLM.
+        Routed through the OpenAI-compatible gateway (LiteLLM/vLLM/hosted).
         """
-        if not ollama_url or not model:
-            logger.debug("distill_insights: no ollama config, skipping")
+        if not base_url or not model:
+            logger.debug("distill_insights: no LLM config, skipping")
             return []
 
         # Only distill if we have enough new observations
@@ -893,23 +896,21 @@ def distill_insights(self, ollama_url: str = "", model: str = "") -> list[Learne
         )
 
         try:
-            import aiohttp
-
             async def _call():
-                payload = {
-                    "model": model,
-                    "prompt": prompt,
-                    "stream": False,
-                    "options": {"temperature": 0.1, "num_predict": 1024},
-                }
-                async with aiohttp.ClientSession() as session:
-                    async with session.post(
-                        f"{ollama_url.rstrip('/')}/api/generate",
-                        json=payload,
-                        timeout=aiohttp.ClientTimeout(total=120),
-                    ) as resp:
-                        data = await resp.json()
-                        return data.get("response", "").strip()
+                # Route through LLMClient so distillation reuses the shared
+                # retry/5xx-handling, reasoning-capability detection, and timeout
+                # plumbing instead of a bespoke aiohttp call.
+                from ..llm import LLMClient
+
+                client = LLMClient(base_url=base_url, model=model)
+                await client._async_init()
+                answer = await client.complete(
+                    [{"role": "user", "content": prompt}],
+                    max_retries=1,
+                    options={"temperature": 0.1, "num_predict": 1024},
+                    operation="compression",
+                )
+                return (answer or "").strip()
 
             try:
                 asyncio.get_running_loop()
@@ -1059,24 +1060,34 @@ def get_insights_for_context(
         if not self.learned_insights:
             return []
 
-        matched: list[LearnedInsight] = []
+        techs_lc = [t.lower() for t in (tech_stack or [])]
+        scored: list[tuple[float, LearnedInsight]] = []
         for insight in self.learned_insights:
             score = 0.0
             conds = insight.conditions
+            conds_lc = str(conds).lower()
 
-            if phase and conds.get("phase", "").lower() == phase.lower():
+            if phase and str(conds.get("phase", "")).lower() == phase.lower():
                 score += 0.5
 
-            if tech_stack:
-                for tech in tech_stack:
-                    if tech.lower() in str(conds).lower():
-                        score += 0.3
+            for tech in techs_lc:
+                if tech and tech in conds_lc:
+                    score += 0.3
 
+            # Keep context-specific matches AND generally-applicable
+            # (conditionless) insights, but rank the relevant ones first so the
+            # model sees its most pertinent past learning at the top of context.
             if score > 0 or not conds:
-                matched.append(insight)
-
-        matched.sort(key=lambda i: i.confidence, reverse=True)
-        return matched[:10]
+                # Blend relevance with how trustworthy/well-supported the
+                # insight is: confidence and the number of observations behind
+                # it. This stops a high-confidence-but-irrelevant insight from
+                # crowding out a directly-relevant one.
+                support = min(getattr(insight, "observation_count", 0) / 10.0, 0.5)
+                rank = score + (insight.confidence * 0.4) + support
+                scored.append((rank, insight))
+
+        scored.sort(key=lambda pair: pair[0], reverse=True)
+        return [insight for _, insight in scored[:10]]
 
     def should_avoid_tool(self, tool_name: str) -> tuple[bool, list[str]]:
         if tool_name in self.negative_patterns:

airecon/proxy/agent/agent_graph.py

@@ -44,9 +44,9 @@ class AgentEdge:
 
 
 class AgentGraph:
-    def __init__(self, target: str, ollama: Any = None, engine: Any = None):
+    def __init__(self, target: str, llm: Any = None, engine: Any = None):
         self.target = target
-        self.ollama = ollama
+        self.llm = llm
         self.engine = engine
         self.nodes: dict[str, AgentNode] = {}
         self.edges: list[AgentEdge] = []
@@ -106,7 +106,7 @@ async def execute(
                 type="agent_state", data={"status": f"Starting {node.id}..."}
             )
 
-            agent = AgentLoop(ollama=self.ollama, engine=self.engine)
+            agent = AgentLoop(llm=self.llm, engine=self.engine)
             await agent.initialize(target=self.target)
             agent._override_max_iterations = node.max_iterations
             agent._blocked_tools = set(MINI_AGENT_BLOCKED_TOOLS)
@@ -146,7 +146,7 @@ def create_default_graph(
     target: str, prompt: str = "", recon_mode: str = "full"
 ) -> AgentGraph:
 
-    g = AgentGraph(target, ollama=None, engine=None)
+    g = AgentGraph(target, llm=None, engine=None)
     if recon_mode == "full":
         return _build_full_pipeline(g, target, prompt)