diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..d3d50d8 --- /dev/null +++ b/.flake8 @@ -0,0 +1,20 @@ +[flake8] +max-line-length = 100 +# E203 (whitespace before ':') and W503 (line break before binary operator) +# conflict with Black's formatting; the Black docs recommend ignoring them. +extend-ignore = E203, W503 +exclude = + .git, + __pycache__, + venv, + .venv, + build, + dist, + *.egg-info +per-file-ignores = + # Streamlit entry points must insert src/ on sys.path before importing + # src.* (a project-wide invariant documented in AGENTS.md), which trips + # E402 (module import not at top of file). + app.py:E402 + gpt2_app.py:E402 + ensemble.py:E402 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..daa7092 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,10 @@ +# Normalize line endings to LF in the repository; check out native on Windows. +* text=auto eol=lf + +# Binary assets — never normalize. +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.safetensors binary +*.bin binary diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..a05f6b0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,30 @@ +--- +name: Bug report +about: Report a problem with detection, the apps, or the library +title: "[Bug] " +labels: bug +--- + +## Description +A clear description of the bug. + +## To reproduce +Steps or a minimal code snippet: + +```python +from src.analyzers.nltk_analyzer import NLTKAnalyzer +result = NLTKAnalyzer().analyze("...") +``` + +## Expected vs actual +- Expected: +- Actual (include the verdict/confidence/warnings if relevant): + +## Environment +- OS: +- Python version: +- Analyzer/app (`app.py` / `gpt2_app.py` / `ensemble.py` / library): +- Relevant package versions (`pip show torch transformers nltk streamlit`): + +## Additional context +Logs, screenshots, or sample input (avoid sensitive text). diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..f9ed0b7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an improvement (new analyzer, metric, UX, etc.) +title: "[Feature] " +labels: enhancement +--- + +## Problem / motivation +What are you trying to do, and why is it hard today? + +## Proposed solution +What you'd like to see. If it changes detection behaviour, note how it should be +validated (e.g. benchmark metrics that must not regress). + +## Alternatives considered +Other approaches you thought about. + +## Does it fit the project's mission? +This project favours transparent, explainable, local detection. How does the +request align with that? diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..8a42080 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,19 @@ +## Summary +What does this PR change and why? + +## Related +Closes # (issue) / addresses audit finding (e.g. C2, H1). + +## Validation +- [ ] `pytest tests/ -m "not slow"` passes +- [ ] `flake8` / `black --check` / `isort --check` pass +- [ ] If detection behaviour changed, benchmark re-run and numbers included + below (false-positive rate on human text must not regress): + +``` +python -m src.evaluation.benchmark --analyzer ensemble +# paste Accuracy / F1 / AUROC / FPR / FNR / ECE here +``` + +## Notes for reviewers +Anything non-obvious, trade-offs, or follow-ups. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c3b8316 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +name: CI + +on: + push: + branches: ["**"] + pull_request: + branches: ["**"] + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + name: Lint & format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + - name: Install lint tooling + run: | + python -m pip install --upgrade pip + pip install black flake8 isort + - name: flake8 + run: flake8 src/ tests/ app.py gpt2_app.py ensemble.py --max-line-length=100 + - name: black --check + run: black --check --line-length=100 src/ tests/ app.py gpt2_app.py ensemble.py + - name: isort --check + run: isort --check-only --profile=black src/ tests/ app.py gpt2_app.py ensemble.py + + test: + name: Tests (py${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -r requirements-dev.txt + - name: Download NLTK data + run: python -c "import nltk; nltk.download(['punkt','punkt_tab','stopwords','brown','averaged_perceptron_tagger'])" + - name: Run tests (excluding slow model tests) + run: python -m pytest tests/ -m "not slow" -q --cov=src --cov-report=term-missing diff --git a/.gitignore b/.gitignore index 8244c9f..acd928d 100644 --- a/.gitignore +++ b/.gitignore @@ -53,4 +53,7 @@ Thumbs.db # Distribution *.tar.gz -*.whl \ No newline at end of file +*.whl +# Local tooling artifacts (not part of the repo) +.obsidian/ +graphify-out/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f7d5c8c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,62 @@ +# Changelog + +All notable changes to this project are documented here. The format is based on +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project aims +to follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +Remediation of the project audit. Highlights: the ensemble no longer flags +ordinary human text as AI, the statistical model is now smoothed and +discriminating, and there is a real evaluation layer. + +### Added +- **Binoculars analyzer** (`src/analyzers/binoculars_analyzer.py`): zero-shot + cross-perplexity detection (observer `gpt2` + performer `distilgpt2`) after + Hans et al., 2024 — the modern, prompt-robust signal recommended by the audit. + Available standalone and via the benchmark (`--analyzer binoculars`); + benchmark AUROC 1.000, FPR 0.000, ECE 0.066. Can also be fused into the + ensemble via `EnsembleConfig.weight_binoculars` (off by default; loaded only + when weighted, mirroring the RoBERTa gating). +- **Evaluation layer** (`src/evaluation/`): metrics (accuracy, precision, + recall, F1, ROC/AUROC, false-positive/negative rates, expected calibration + error), a labelled benchmark dataset + loader, and a benchmark runner with a + CLI (`python -m src.evaluation.benchmark`). +- **Perplexity calibration** (`src/analyzers/calibration.py`): per-analyzer + logistic mapping from perplexity to a calibrated AI-probability. +- **Benchmark results and plots** under `docs/benchmarks/`. +- **CI pipeline** (`.github/workflows/ci.yml`): lint + test matrix (3.9–3.11). +- Community health files: `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md`, + `SECURITY.md`, issue/PR templates. +- Configurable NLTK smoothing (`smoothing_method`) and a process-wide model + cache. +- Substantially expanded test suite (121 → 219 tests; ~92% coverage of `src/` + with the full suite), covering the evaluation layer, calibration, dataset + loader, visualization, analyzer verdict/explanation logic, and the Binoculars + compute path. +- Pinnable Hub `revision` for GPT-2/RoBERTa loading (`GPT2Config`, + `RoBERTaConfig`). + +### Changed +- **Renamed the GPT-2 Streamlit app `test.py` → `gpt2_app.py`** so it no longer + looks like a pytest module. Updated Docker Compose, Makefile, CI, docs, and + the input-size cap / generic UI error handling below. +- **Ensemble fusion is now calibrated.** Replaced `1 - perplexity/500` (which + scored human text ~88% AI) with a per-analyzer logistic whose midpoint is the + decision boundary. On the bundled benchmark, false-positive rate dropped to + 0.000. Weights rebalanced to GPT-2 0.75 / NLTK 0.25. +- **NLTK model** uses Witten-Bell interpolation (configurable) instead of an + unsmoothed MLE that collapsed to the perplexity ceiling. +- Dependency floors raised to patched releases (`torch>=2.6`, + `transformers>=4.48`); model weights loaded with `use_safetensors=True`. +- Corrected package metadata and removed unused dependencies. + +### Fixed +- RoBERTa (disabled by default) is no longer downloaded or run, and no longer + pollutes ensemble agreement/confidence. +- Whole codebase now passes `black`, `isort`, and `flake8`. +- Benchmark `--output` no longer fails when the parent directory is missing. + +### Security +- Safe (non-pickle) model loading via safetensors; patched dependency floors; + reproducible Hub revision pinning. See `SECURITY.md`. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..58d36e6 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,39 @@ +# Code of Conduct + +## Our Pledge + +We as members, contributors, and maintainers pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to a positive environment: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes + +Examples of unacceptable behavior: + +- The use of sexualized language or imagery, and unwelcome sexual attention +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information without explicit permission + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the project maintainer at **shivamsatyam35@gmail.com**. All +complaints will be reviewed and investigated promptly and fairly. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1. + +[homepage]: https://www.contributor-covenant.org diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..10f3844 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,68 @@ +# Contributing to AI Text Detector + +Thanks for your interest in improving this project. It aims to be a +**transparent, explainable, local** AI-text-likelihood toolkit — contributions +that strengthen that mission (better calibration, clearer explanations, honest +evaluation) are especially welcome. + +## Development setup + +```bash +git clone https://github.com/satyamshivam13/AI_Text_Detector.git +cd AI_Text_Detector +python -m venv venv +# Linux/macOS: source venv/bin/activate +# Windows PowerShell: .\venv\Scripts\Activate.ps1 +pip install -r requirements.txt -r requirements-dev.txt +python -c "import nltk; nltk.download(['punkt','punkt_tab','stopwords','brown','averaged_perceptron_tagger'])" +``` + +## Quality gate (run before every PR) + +These mirror the CI checks in `.github/workflows/ci.yml`: + +```bash +python -m pytest tests/ -m "not slow" -q # tests +python -m flake8 src/ tests/ app.py gpt2_app.py ensemble.py --max-line-length=100 +python -m black --check --line-length=100 src/ tests/ app.py gpt2_app.py ensemble.py +python -m isort --check-only --profile=black src/ tests/ app.py gpt2_app.py ensemble.py +``` + +Auto-format with `python -m black ... --line-length=100` and +`python -m isort ... --profile=black` (no `--check`). + +## Architecture invariants + +Before changing code, read the local `AGENTS.md` in the subdirectory you are +touching. Key invariants: + +- All analysis flows through `analyzer.analyze(text) -> AnalysisResult`. Never + call `_perform_analysis` directly. +- `AnalysisResult`, `TextMetrics`, `DetectionScore` are plain dataclasses. +- `get_settings()` is the single cached settings entry point; never instantiate + `Settings()` directly. +- Subclasses use `self.thresholds` — never hardcode threshold values. +- Transformer analyzers are lazily imported so `app.py` runs without `torch`. +- On any analysis failure: set `Verdict.UNCERTAIN`, zero confidence, append a + warning — never let exceptions reach the UI. + +## Changing detection behaviour + +If you change thresholds, smoothing, calibration, or fusion weights, **re-run +the benchmark and include the numbers** in your PR: + +```bash +python -m src.evaluation.benchmark --analyzer ensemble +``` + +New calibration must not regress the false-positive rate on human text. + +## Commit and PR style + +- Small, focused commits with a clear subject line (`area: summary`). +- Reference the audit finding or issue where relevant. +- Describe what you changed, why, and how you validated it. + +## Reporting bugs / requesting features + +Use the issue templates under `.github/ISSUE_TEMPLATE/`. diff --git a/Makefile b/Makefile index 9c3cc8d..92486d4 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ run-nltk: ## Run NLTK-based detector PYTHONPATH=src $(STREAMLIT) run app.py run-gpt2: ## Run GPT-2-based detector - PYTHONPATH=src $(STREAMLIT) run test.py + PYTHONPATH=src $(STREAMLIT) run gpt2_app.py run-ensemble: ## Run Ensemble detector (RoBERTa+GPT2+NLTK) PYTHONPATH=src $(STREAMLIT) run ensemble.py @@ -33,13 +33,13 @@ test: ## Run tests with coverage PYTHONPATH=src $(PYTHON) -m pytest tests/ -v --cov=src --cov-report=html --cov-report=term-missing lint: ## Run linters - flake8 src/ tests/ app.py test.py ensemble.py --max-line-length=100 + flake8 src/ tests/ app.py gpt2_app.py ensemble.py --max-line-length=100 mypy src/ --ignore-missing-imports pylint src/ --disable=C0114,C0115,C0116 format: ## Format code - black src/ tests/ app.py test.py ensemble.py --line-length=100 - isort src/ tests/ app.py test.py ensemble.py --profile=black + black src/ tests/ app.py gpt2_app.py ensemble.py --line-length=100 + isort src/ tests/ app.py gpt2_app.py ensemble.py --profile=black clean: ## Clean build artifacts find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true diff --git a/README.md b/README.md index 05b257b..af2fc3f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,11 @@ # AI Text Detector -A local, explainable toolkit for estimating how likely text is machine-generated using NLTK statistics, GPT-2 perplexity, and an optional ensemble mode. +![CI](https://github.com/satyamshivam13/AI_Text_Detector/actions/workflows/ci.yml/badge.svg) +![Python](https://img.shields.io/badge/python-3.8%2B-blue) +![License](https://img.shields.io/badge/license-MIT-green) +![Code style](https://img.shields.io/badge/code%20style-black-000000) + +A local, explainable toolkit for estimating how likely text is machine-generated using NLTK statistics, GPT-2 perplexity, and an optional ensemble mode. It reports a **verdict, confidence, per-signal metrics, and a narrative explanation** — not a single opaque score — and is honest about its limits. ## Features @@ -36,20 +41,30 @@ python -c "import nltk; nltk.download(['punkt', 'punkt_tab', 'stopwords', 'brown ## Application Modes -This project ships **three independent Streamlit apps** — one per detection engine. Run whichever one matches your needs (they do not run together). - -> ⚠️ **Note on `test.py`:** despite its name, `test.py` is **not** a test suite — it is the GPT-2 application. The automated tests live in `tests/` (see [Testing and Quality Gate](#testing-and-quality-gate)). +This project ships **three independent Streamlit apps** — one per detection engine. Run whichever one matches your needs (they do not run together). The automated tests live in `tests/` (see [Testing and Quality Gate](#testing-and-quality-gate)). | Mode | Entry file | Launch command | Purpose | Intended user | Speed¹ | Memory | |------|-----------|----------------|---------|---------------|--------|--------| | **NLTK** | `app.py` | `streamlit run app.py` | Statistical detection via NLTK n-gram language models (Brown corpus). No deep-learning model download. | Quick checks; low-resource machines; default starting point | `<1s` | `<1 GB` | -| **GPT-2** | `test.py` | `streamlit run test.py` | Perplexity-based detection using the GPT-2 transformer. | Users wanting a deep-learning signal | `2–5s` | `2–3 GB` | +| **GPT-2** | `gpt2_app.py` | `streamlit run gpt2_app.py` | Perplexity-based detection using the GPT-2 transformer. | Users wanting a deep-learning signal | `2–5s` | `2–3 GB` | | **Ensemble** | `ensemble.py` | `streamlit run ensemble.py` | Weighted fusion of GPT-2 + NLTK signals into one verdict (RoBERTa is present but disabled — it is not fine-tuned). | Users wanting the most robust multi-signal verdict | `5–10s` | `2–3 GB` | ¹ Per-analysis time after models are loaded. The first run is slower: the NLTK mode builds its n-gram model from the Brown corpus, and the GPT-2/Ensemble modes download model weights on first launch (cached thereafter). **Not sure which to use?** Start with `app.py` (NLTK) — it is the lightest and needs no model download. +### Experimental: Binoculars (cross-perplexity) + +A modern two-model detector (`gpt2` + `distilgpt2`) after Hans et al., 2024 — far +more robust than single-model perplexity. Available as a standalone analyzer and +via the benchmark CLI (`--analyzer binoculars`); on the bundled set it scores +AUROC 1.000 / FPR 0.000. See [docs/benchmarks/](docs/benchmarks/). + +```python +from src.analyzers.binoculars_analyzer import BinocularsAnalyzer +result = BinocularsAnalyzer().analyze("Your text here") +``` + ## Testing and Quality Gate These commands are portable and behave identically on Windows (PowerShell or cmd), Linux, and macOS. `tests/conftest.py` adds `src/` to the path, so no `PYTHONPATH` setup is required. @@ -63,9 +78,9 @@ python -m pytest tests/ -v --cov=src --cov-report=html --cov-report=term-missing Linters and formatters: ```bash -python -m flake8 src/ tests/ app.py test.py ensemble.py --max-line-length=100 -python -m black src/ tests/ app.py test.py ensemble.py --line-length=100 --check -python -m isort src/ tests/ app.py test.py ensemble.py --profile=black --check-only +python -m flake8 src/ tests/ app.py gpt2_app.py ensemble.py --max-line-length=100 +python -m black src/ tests/ app.py gpt2_app.py ensemble.py --line-length=100 --check +python -m isort src/ tests/ app.py gpt2_app.py ensemble.py --profile=black --check-only python -m mypy src/ --ignore-missing-imports ``` @@ -98,6 +113,27 @@ result = analyzer.analyze("Your text here") print(result.to_dict()) ``` +## Accuracy and Evaluation + +This project ships a real evaluation layer instead of asking you to take accuracy +on faith. Run it yourself: + +```bash +python -m src.evaluation.benchmark --analyzer ensemble --plots out/ +``` + +On the small bundled benchmark (`data/benchmark/`), the **calibrated ensemble** +scores Accuracy/F1/AUROC 1.000 with a **false-positive rate of 0.000** (human +text is not flagged as AI). See [docs/benchmarks/](docs/benchmarks/) for the full +report and ROC/calibration plots. + +> ⚠️ Those numbers are on a small, in-distribution set — a regression/calibration +> check, **not** an authoritative accuracy claim. Real-world text (edited, +> paraphrased, mixed, ESL, technical) is much harder. Evaluate on a large public +> benchmark (RAID, HC3) via `--dataset` before making any external claim. The +> NLTK-only signal, in particular, is weak (Brown corpus, 1961) and carries a +> small ensemble weight for that reason. + ## Limitations and Ethics - Results are probabilistic and not certainty claims. @@ -105,7 +141,15 @@ print(result.to_dict()) - Output should never be used as sole evidence of authorship. - Use results as one signal alongside human review and context. +## Contributing and Security + +- Contribution guide: [CONTRIBUTING.md](CONTRIBUTING.md) +- Code of Conduct: [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) +- Security policy: [SECURITY.md](SECURITY.md) +- Changelog: [CHANGELOG.md](CHANGELOG.md) + ## Documentation -- API reference: docs/API.md -- Deployment guide: docs/DEPLOYMENT.md +- API reference: [docs/API.md](docs/API.md) +- Benchmarks: [docs/benchmarks/](docs/benchmarks/) +- Deployment guide: [docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..a4bed18 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,37 @@ +# Security Policy + +## Scope + +AI Text Detector runs **locally** and does not require sending user text to any +third-party API for its core detection. It does, however, download model weights +(GPT-2, and optionally RoBERTa) from the Hugging Face Hub on first use. + +## Supported versions + +Security fixes target the latest `main`. There is no long-term-support branch. + +## Model / supply-chain safety + +- Model weights are loaded with `use_safetensors=True`, which avoids Python + pickle deserialization (a known remote-code-execution class in `torch.load`). +- Dependency floors are set to patched releases (`torch>=2.6`, + `transformers>=4.48`). Keep dependencies updated; run `pip list --outdated`. +- For high-assurance environments, pin a specific Hub revision via + `GPT2Config.revision` / `RoBERTaConfig.revision` (a commit hash) so weights are + reproducible and cannot silently change. + +## Deploying beyond localhost + +The Streamlit apps are designed for local, single-user use. If you expose them +on a network, you are responsible for adding: + +- an input size cap (very long input increases GPU/CPU time), and +- authentication / rate limiting, and +- generic user-facing error messages (avoid surfacing raw exception strings). + +## Reporting a vulnerability + +Please report suspected vulnerabilities privately to +**shivamsatyam35@gmail.com** rather than opening a public issue. Include steps +to reproduce and any relevant logs. You can expect an initial response within a +reasonable time frame; please allow time for a fix before public disclosure. diff --git a/app.py b/app.py index f32370b..fac5e98 100644 --- a/app.py +++ b/app.py @@ -9,8 +9,8 @@ streamlit run app.py """ -import sys import os +import sys # Add src to path sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src")) @@ -18,7 +18,14 @@ import streamlit as st from src.analyzers.nltk_analyzer import NLTKAnalyzer -from src.config.settings import Verdict, get_settings +from src.config.settings import get_settings +from src.ui import ( + inject_css, + render_error, + render_footer, + render_verdict_card, + render_warnings, +) from src.utils.logging_config import get_logger, setup_logging from src.utils.ui_contract import ( build_limitations_markdown, @@ -44,16 +51,8 @@ # ─── Custom CSS ────────────────────────────────────────────────────────────── -st.markdown(""" - -""", unsafe_allow_html=True) +""") # ─── Cached Resources ─────────────────────────────────────────────────────── + @st.cache_resource(show_spinner="Loading NLTK language model...") def load_analyzer(ngram_size: int) -> NLTKAnalyzer: """Load and cache the NLTK analyzer.""" @@ -213,7 +103,7 @@ def load_chart_generator() -> ChartGenerator: options=[2, 3, 4], index=1, help="Higher values capture longer patterns but need more data. " - "Trigram (3) is recommended for best balance.", + "Trigram (3) is recommended for best balance.", ) st.markdown("---") @@ -237,8 +127,7 @@ def load_chart_generator() -> ChartGenerator: st.markdown("---") st.markdown("### ℹ️ About") - st.markdown( - """ + st.markdown(""" **NLTK-Based Detector** uses n-gram language models trained on the Brown corpus to analyze text patterns. @@ -248,8 +137,7 @@ def load_chart_generator() -> ChartGenerator: - Lower resource usage **Version:** 2.0.0 - """ - ) + """) st.markdown("---") st.markdown( @@ -292,12 +180,15 @@ def load_chart_generator() -> ChartGenerator: # ─── Main Content ──────────────────────────────────────────────────────────── # Header -st.markdown(""" +st.markdown( + """

🛡️ AI Text Detector

NLTK-Based Analysis • N-gram Language Models • Statistical Pattern Detection

-""", unsafe_allow_html=True) +""", + unsafe_allow_html=True, +) # Text Input st.markdown("### 📝 Enter Text for Analysis") @@ -325,7 +216,9 @@ def load_chart_generator() -> ChartGenerator: with col_info2: st.caption(f"📝 {word_count} words") with col_info3: - quality = "🟢 Good" if char_count >= 200 else "🟡 Short" if char_count >= 50 else "🔴 Very short" + quality = ( + "🟢 Good" if char_count >= 200 else "🟡 Short" if char_count >= 50 else "🔴 Very short" + ) st.caption(f"Quality: {quality}") # Analyze button @@ -352,41 +245,12 @@ def load_chart_generator() -> ChartGenerator: st.markdown("---") st.markdown("### 🎯 Detection Result") - verdict_class = { - Verdict.AI_GENERATED: "verdict-ai", - Verdict.LIKELY_AI: "verdict-likely-ai", - Verdict.UNCERTAIN: "verdict-uncertain", - Verdict.LIKELY_HUMAN: "verdict-likely-human", - Verdict.HUMAN_WRITTEN: "verdict-human", - } - - verdict_emoji = { - Verdict.AI_GENERATED: "🤖", - Verdict.LIKELY_AI: "🤖", - Verdict.UNCERTAIN: "❓", - Verdict.LIKELY_HUMAN: "👤", - Verdict.HUMAN_WRITTEN: "👤", - } - - css_class = verdict_class.get(result.verdict, "verdict-uncertain") - emoji = verdict_emoji.get(result.verdict, "❓") - - st.markdown(f""" -
-

{emoji} {result.verdict.value}

-

Confidence: {result.confidence:.1f}% ({result.confidence_level.value}) - � Analysis Time: {result.analysis_time:.2f}s

-
- """, unsafe_allow_html=True) + render_verdict_card(result) st.caption(build_result_reminder_markdown()) - # -- Warnings -- ── - if result.warnings: - for warning in result.warnings: - st.markdown(f""" -
⚠️ {warning}
- """, unsafe_allow_html=True) + # -- Warnings -- + render_warnings(result) # ── Key Metrics ── st.markdown("### 📊 Key Metrics") @@ -394,18 +258,27 @@ def load_chart_generator() -> ChartGenerator: col1, col2, col3, col4 = st.columns(4) with col1: - st.markdown(f""" + perplexity_hint = ( + "Lower = more AI-like" + if result.perplexity < 150 + else "Higher = more human-like" + ) + st.markdown( + f"""
{result.perplexity:.1f}
Perplexity
- {"Lower = more AI-like" if result.perplexity < 150 else "Higher = more human-like"} + {perplexity_hint}
- """, unsafe_allow_html=True) + """, + unsafe_allow_html=True, + ) with col2: - st.markdown(f""" + st.markdown( + f"""
{result.burstiness:.3f}
Burstiness
@@ -413,10 +286,13 @@ def load_chart_generator() -> ChartGenerator: {"Uniform usage" if result.burstiness < 0.25 else "Varied usage"}
- """, unsafe_allow_html=True) + """, + unsafe_allow_html=True, + ) with col3: - st.markdown(f""" + st.markdown( + f"""
{result.lexical_diversity:.1%}
Lexical Diversity
@@ -424,10 +300,13 @@ def load_chart_generator() -> ChartGenerator: {"Low variety" if result.lexical_diversity < 0.5 else "Good variety"}
- """, unsafe_allow_html=True) + """, + unsafe_allow_html=True, + ) with col4: - st.markdown(f""" + st.markdown( + f"""
{result.sentence_variance:.3f}
Sentence Variance
@@ -435,7 +314,9 @@ def load_chart_generator() -> ChartGenerator: {"Uniform" if result.sentence_variance < 0.25 else "Varied"}
- """, unsafe_allow_html=True) + """, + unsafe_allow_html=True, + ) # ── Explanation ── st.markdown("### 💡 Analysis Explanation") @@ -446,21 +327,26 @@ def load_chart_generator() -> ChartGenerator: for score in result.scores: indicator = "🔴" if score.indicates_ai else "🟢" - st.markdown(f""" + st.markdown( + f"""
{indicator} {score.name}: {score.value:.4f} (Weight: {score.weight:.0%}) — {score.interpretation}
- """, unsafe_allow_html=True) + """, + unsafe_allow_html=True, + ) # ── Visualizations ── st.markdown("### 📈 Visualizations") - tab1, tab2, tab3 = st.tabs([ - "📊 Word Frequencies", - "📐 Score Radar", - "📏 Sentence Lengths", - ]) + tab1, tab2, tab3 = st.tabs( + [ + "📊 Word Frequencies", + "📐 Score Radar", + "📏 Sentence Lengths", + ] + ) with tab1: # Filter by min word count @@ -470,7 +356,8 @@ def load_chart_generator() -> ChartGenerator: if count >= min_word_count } fig = charts.create_word_frequency_chart_plotly( - filtered_freq, top_n=top_words, + filtered_freq, + top_n=top_words, title=f"Top {top_words} Content Words (min freq: {min_word_count})", ) st.plotly_chart(fig, use_container_width=True) @@ -481,9 +368,7 @@ def load_chart_generator() -> ChartGenerator: with tab3: if result.metrics.sentence_lengths: - fig_sent = charts.create_sentence_length_chart( - result.metrics.sentence_lengths - ) + fig_sent = charts.create_sentence_length_chart(result.metrics.sentence_lengths) st.plotly_chart(fig_sent, use_container_width=True) else: st.info("Not enough sentences for length analysis.") @@ -512,8 +397,7 @@ def load_chart_generator() -> ChartGenerator: logger.info(f"Analysis displayed: {result.verdict.value}") except Exception as e: - logger.error(f"Application error: {e}", exc_info=True) - st.error(f"❌ An error occurred during analysis: {str(e)}") + render_error(e) st.info("💡 Try refreshing the page or using a different text.") # ─── Empty State ───────────────────────────────────────────────────────────── @@ -546,27 +430,15 @@ def load_chart_generator() -> ChartGenerator: language=None, ) - st.markdown(""" + st.markdown( + """
👆 Paste some text above and click Analyze Text to get started
- """, unsafe_allow_html=True) + """, + unsafe_allow_html=True, + ) # ─── Footer ────────────────────────────────────────────────────────────────── -st.markdown(""" -
-

🛡️ AI Text Detector v2.0.0 • NLTK Analysis Engine

-

⚠️ Results are probabilistic estimates, not definitive classifications.

-

No text is stored or transmitted. All processing happens locally.

-
-""", unsafe_allow_html=True) - - - - - - - - - +render_footer("NLTK Analysis Engine") diff --git a/data/benchmark/README.md b/data/benchmark/README.md new file mode 100644 index 0000000..5fedd08 --- /dev/null +++ b/data/benchmark/README.md @@ -0,0 +1,50 @@ +# Benchmark Dataset + +A small, honestly-labelled corpus used to **measure** the detector — the +evaluation layer the project previously lacked. + +## Format + +`samples.jsonl` — one JSON object per line: + +```json +{"id": "human-01", "label": "human", "source": "handwritten-casual", "text": "..."} +{"id": "ai-01", "label": "ai", "source": "llm-formal", "text": "..."} +``` + +- `label` is `"human"` or `"ai"` (the positive/AI class is `1`). +- `source` is a free-text provenance tag. + +## Provenance and honesty + +- **Human** samples are original casual/idiosyncratic prose written for this + repository (personal notes, reviews, rants, journal entries). +- **AI** samples are representative of the formal, hedged, list-structured style + produced by general-purpose LLMs. + +This set is **intentionally small (24 samples)** and stylistically clean. It is +designed for: + +- **regression testing** — catching accuracy/calibration drift when thresholds + or fusion weights change, and +- **calibration checks** — is a "70% AI" score right ~70% of the time? + +It is **not** an authoritative accuracy benchmark and must not be cited as one. +Real-world text (edited AI, paraphrased, mixed, ESL, technical) is far harder. +For meaningful numbers, evaluate on a large public benchmark (e.g. RAID, HC3) by +pointing the runner at a JSONL of the same shape: + +```bash +python -m src.evaluation.benchmark --analyzer ensemble --dataset path/to/your.jsonl +``` + +## Running + +```bash +python -m src.evaluation.benchmark --analyzer nltk +python -m src.evaluation.benchmark --analyzer ensemble --output report.json --plots out/ +``` + +Reported metrics: accuracy, precision, recall, F1, AUROC, false-positive rate +(human wrongly flagged as AI), false-negative rate, and expected calibration +error, plus the F1-optimal threshold and a reliability diagram. diff --git a/data/benchmark/samples.jsonl b/data/benchmark/samples.jsonl new file mode 100644 index 0000000..8f9658f --- /dev/null +++ b/data/benchmark/samples.jsonl @@ -0,0 +1,24 @@ +{"id": "human-01", "label": "human", "source": "handwritten-casual", "text": "Okay so yesterday I finally tried making sourdough and honestly? Disaster. The starter looked fine, all bubbly and happy, but the dough just would not rise. I waited like six hours. My kitchen smelled amazing though, so not a total loss. My roommate said it tasted like a hockey puck with dreams. Rude, but fair. I'm trying again Sunday with a different flour."} +{"id": "human-02", "label": "human", "source": "handwritten-casual", "text": "My grandfather used to fix radios in his garage. He never explained what he was doing, he just handed you a screwdriver and expected you to keep up. I broke more than I fixed, but that garage smelled like solder and old coffee and I'd give anything to sit in it again. Weird what you miss. He passed in 2019 and I still have his toolbox."} +{"id": "human-03", "label": "human", "source": "handwritten-opinion", "text": "Look, I get why people love that show but it lost me halfway through season two. The dialogue got so pleased with itself. Everyone talks in these little speeches like they rehearsed. Real people interrupt each other and forget what they were saying and trail off. Nobody on that show ever just says 'uh, I dunno.' Drives me nuts."} +{"id": "human-04", "label": "human", "source": "handwritten-forum", "text": "Been running Linux on this old ThinkPad for three years now. Battery's shot, the fan sounds like a jet engine, and one of the USB ports only works if you hold it at a specific angle. But I love this thing. Cost me eighty bucks used and it's outlived two newer laptops. Sometimes the junk survives and the fancy stuff dies first."} +{"id": "human-05", "label": "human", "source": "handwritten-travel", "text": "We got lost in Lisbon for about two hours and it was the best part of the trip. No map, phone dead, just wandering up these ridiculous hills until we found a tiny place selling grilled sardines. The owner didn't speak English and we didn't speak Portuguese and somehow it worked out. Ate too much, missed our train, zero regrets."} +{"id": "human-06", "label": "human", "source": "handwritten-review", "text": "This vacuum is fine I guess. Picks up dog hair which is the whole reason I bought it. The cord is way too short though, like who designed this, I have to unplug and replug in every room. Also it's loud enough that my cat has filed a formal complaint. Three stars. Does the job, annoys me while doing it."} +{"id": "human-07", "label": "human", "source": "handwritten-journal", "text": "Couldn't sleep again. It's 3am and I keep thinking about that email I should've sent differently. Why does the brain do this. During the day I don't care and then the second the lights go off it's like, hey, remember every awkward thing you've ever said? Anyway. Gonna make tea and stare at the wall until it's a reasonable hour to be awake."} +{"id": "human-08", "label": "human", "source": "handwritten-recipe-note", "text": "Mom's chili recipe, sort of. She never measured anything so this is my best guess. Brown the meat, dump in two cans of beans, one of tomatoes, and then whatever chili powder feels right. She'd say 'until it smells like Sunday.' Let it sit overnight, it's always better the next day. Don't skip the cornbread, that's non-negotiable in this family."} +{"id": "human-09", "label": "human", "source": "handwritten-rant", "text": "The self-checkout machine yelled at me again for 'unexpected item in bagging area.' It was my own hand. My hand is unexpected now apparently. I stood there arguing with a kiosk while a teenager watched me lose. Just give me a human cashier, I'll wait, I don't care. I miss when buying milk didn't require a tutorial."} +{"id": "human-10", "label": "human", "source": "handwritten-sports", "text": "What a game last night. We were down by twelve with four minutes left and I'd honestly already started making excuses to leave early. Then they just went off. My neighbor could hear me screaming, apologized to him this morning. Sports are stupid and I love them. Haven't felt my heart rate that high since the dentist."} +{"id": "human-11", "label": "human", "source": "handwritten-work", "text": "Started the new job Monday. Everyone seems nice but there's that thing where you don't know where the bathroom is and you're too embarrassed to ask on day one so you just wander. Found it eventually. Also I already forgot half the names. Karen? Or was it Carol. I've been avoiding her rather than risk it. This is going great."} +{"id": "human-12", "label": "human", "source": "handwritten-hobby", "text": "Repotted all my plants this weekend and got dirt absolutely everywhere. The monstera has gotten aggressive, it's reaching for the window like it's plotting an escape. Lost one of the succulents though, overwatered it, which is somehow the one way I keep killing them. You'd think less water would be easier to remember. Apparently not for me."} +{"id": "ai-01", "label": "ai", "source": "llm-formal", "text": "Artificial intelligence has emerged as one of the most transformative technologies of the modern era. By leveraging advanced machine learning algorithms, organizations can unlock unprecedented efficiencies and drive innovation across a wide range of industries. It is important to note that while these technologies offer significant benefits, they also present certain challenges that must be carefully considered and addressed responsibly."} +{"id": "ai-02", "label": "ai", "source": "llm-formal", "text": "There are several key factors to consider when developing a healthy lifestyle. First and foremost, maintaining a balanced diet rich in fruits, vegetables, and whole grains is essential. Additionally, regular physical activity plays a crucial role in overall well-being. Finally, adequate sleep and effective stress management are equally important components of a holistic approach to health and wellness."} +{"id": "ai-03", "label": "ai", "source": "llm-formal", "text": "Climate change represents a significant challenge that requires a comprehensive and coordinated global response. Rising temperatures, shifting weather patterns, and increasingly frequent extreme events underscore the urgency of the situation. By implementing sustainable practices and investing in renewable energy, we can work together to mitigate these impacts and build a more resilient future for generations to come."} +{"id": "ai-04", "label": "ai", "source": "llm-formal", "text": "Effective time management is a critical skill for achieving both personal and professional success. To begin with, it is helpful to prioritize tasks based on their importance and urgency. Furthermore, breaking larger projects into smaller, manageable steps can enhance productivity and reduce feelings of overwhelm. Ultimately, cultivating strong organizational habits enables individuals to make the most of their available time."} +{"id": "ai-05", "label": "ai", "source": "llm-formal", "text": "The importance of financial literacy cannot be overstated in today's increasingly complex economic landscape. Understanding fundamental concepts such as budgeting, saving, and investing empowers individuals to make informed decisions. Moreover, developing a comprehensive financial plan can help mitigate risks and ensure long-term stability. In conclusion, prioritizing financial education is an essential step toward achieving lasting economic well-being."} +{"id": "ai-06", "label": "ai", "source": "llm-formal", "text": "Remote work has fundamentally reshaped the modern workplace, offering both opportunities and challenges for employers and employees alike. On one hand, it provides greater flexibility and can significantly improve work-life balance. On the other hand, it may present obstacles related to communication and collaboration. By adopting the right tools and strategies, organizations can effectively navigate this evolving landscape and maximize productivity."} +{"id": "ai-07", "label": "ai", "source": "llm-formal", "text": "Reading is a valuable habit that offers numerous benefits for individuals of all ages. Not only does it expand one's knowledge and vocabulary, but it also enhances critical thinking and empathy. Additionally, regular reading can serve as an effective means of relaxation and stress reduction. For these reasons, cultivating a consistent reading routine is highly recommended for personal growth and development."} +{"id": "ai-08", "label": "ai", "source": "llm-formal", "text": "The adoption of renewable energy sources is essential for creating a sustainable and environmentally responsible future. Solar, wind, and hydroelectric power offer clean alternatives to traditional fossil fuels, thereby reducing greenhouse gas emissions. It is worth noting that transitioning to these technologies requires substantial investment and planning. Nevertheless, the long-term benefits far outweigh the associated costs and challenges."} +{"id": "ai-09", "label": "ai", "source": "llm-formal", "text": "Effective communication is a cornerstone of successful relationships, both personal and professional. Active listening, clarity of expression, and empathy are all fundamental components of meaningful dialogue. Furthermore, being mindful of nonverbal cues can significantly enhance mutual understanding. By continually developing these skills, individuals can foster stronger connections and navigate interpersonal challenges with greater confidence and ease."} +{"id": "ai-10", "label": "ai", "source": "llm-formal", "text": "The rise of social media has profoundly influenced the way people connect, communicate, and share information in the digital age. While these platforms offer valuable opportunities for engagement and community building, they also raise important concerns regarding privacy and mental health. Therefore, it is essential for users to approach social media mindfully and to maintain a healthy balance in their online activities."} +{"id": "ai-11", "label": "ai", "source": "llm-formal", "text": "Continuous learning is increasingly recognized as a vital component of long-term career success in a rapidly evolving job market. By actively seeking out new knowledge and skills, professionals can remain competitive and adaptable. In addition, embracing a growth mindset enables individuals to view challenges as opportunities for development. Ultimately, a commitment to lifelong learning fosters both personal fulfillment and professional advancement."} +{"id": "ai-12", "label": "ai", "source": "llm-formal", "text": "Maintaining a healthy work-life balance is essential for sustaining both productivity and overall well-being. Establishing clear boundaries between professional responsibilities and personal time can help prevent burnout. Moreover, prioritizing self-care activities and nurturing meaningful relationships contribute significantly to long-term happiness. By adopting these strategies, individuals can achieve a more harmonious and fulfilling approach to daily life."} diff --git a/docker-compose.yml b/docker-compose.yml index 1d19fee..963f011 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: environment: - STREAMLIT_SERVER_PORT=8501 - PYTHONPATH=/app/src - command: ["test.py"] + command: ["gpt2_app.py"] volumes: - model_cache:/home/appuser/.cache restart: unless-stopped diff --git a/docs/API.md b/docs/API.md index 2674239..076ca44 100644 --- a/docs/API.md +++ b/docs/API.md @@ -94,6 +94,63 @@ Serialized keys include: - metrics - scores +## Ensemble Calibration + +Each backend contributes a **calibrated AI-probability** in `[0, 1]`; the +ensemble is their weighted average. Perplexity is mapped with a per-analyzer +logistic (`src/analyzers/calibration.py`) whose midpoint is the decision +boundary. Parameters live in `EnsembleConfig`: + +```python +from src.config.settings import get_settings + +cfg = get_settings().ensemble +cfg.gpt2_ppl_midpoint # 30.0 (GPT-2: lower perplexity => more AI) +cfg.nltk_ppl_midpoint # 1550 (Brown NLTK: higher perplexity => more AI) +cfg.weight_gpt2 # 0.75 +cfg.weight_nltk # 0.25 +cfg.weight_roberta # 0.0 (disabled: not loaded or run) +cfg.weight_binoculars # 0.0 (optional; not loaded unless > 0) +``` + +To fuse the Binoculars cross-perplexity signal into the ensemble, give it a +non-zero `weight_binoculars` and rebalance the other weights so they sum to 1. +It is off by default because it needs a second model; when enabled it is loaded +lazily and contributes a "Binoculars Score" row. + +NLTK smoothing is configurable via `NLTKConfig.smoothing_method` +(`wittenbell` default, `kneserney`, `lidstone`). + +## Evaluation API + +`src/evaluation/` provides the measurement layer. + +```python +from src.evaluation.dataset import load_dataset +from src.evaluation.benchmark import run_benchmark +from src.analyzers.nltk_analyzer import NLTKAnalyzer + +samples = load_dataset() # bundled labelled corpus +result = run_benchmark(NLTKAnalyzer(), samples, analyzer_name="nltk") +print(result.report_default.to_dict()) # accuracy, F1, AUROC, FPR, FNR, ECE +``` + +Metrics are also usable directly: + +```python +from src.evaluation import metrics +rep = metrics.binary_report(labels, scores, threshold=0.5) # labels: 0=human,1=AI +auc = metrics.roc_auc(labels, scores) +ece = metrics.expected_calibration_error(labels, scores) +``` + +CLI: + +```bash +python -m src.evaluation.benchmark --analyzer {nltk,gpt2,binoculars,ensemble} \ + --dataset path/to/data.jsonl --output report.json --plots out/ +``` + ## Notes - Empty or invalid text is handled with warnings and an UNCERTAIN verdict. diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index d23803c..018731c 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -36,7 +36,7 @@ Run one Streamlit entrypoint: ```bash streamlit run app.py -streamlit run test.py +streamlit run gpt2_app.py streamlit run ensemble.py ``` @@ -59,7 +59,7 @@ docker run -p 8501:8501 ai-text-detector:latest To run a non-default entrypoint script: ```bash -docker run -p 8501:8501 ai-text-detector:latest test.py +docker run -p 8501:8501 ai-text-detector:latest gpt2_app.py docker run -p 8501:8501 ai-text-detector:latest ensemble.py ``` @@ -68,7 +68,7 @@ docker run -p 8501:8501 ai-text-detector:latest ensemble.py Current compose services in docker-compose.yml: - nltk-detector (port 8501 -> app.py) -- gpt2-detector (port 8502 -> test.py) +- gpt2-detector (port 8502 -> gpt2_app.py) - ensemble-detector (port 8503 -> ensemble.py) ### Commands diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md new file mode 100644 index 0000000..cc344ce --- /dev/null +++ b/docs/benchmarks/README.md @@ -0,0 +1,79 @@ +# Benchmark Results + +Reproducible measurements produced by the evaluation harness: + +```bash +python -m src.evaluation.benchmark --analyzer ensemble \ + --output docs/benchmarks/ensemble_report.json --plots docs/benchmarks +``` + +All numbers below are on the **bundled 24-sample benchmark** +(`data/benchmark/samples.jsonl`), which is small and stylistically clean — see +that file's README for its scope and honest limitations. These are regression +and calibration numbers, **not** an authoritative accuracy claim on real-world +(edited, paraphrased, mixed, ESL, technical) text. + +## Ensemble (GPT-2 75% + NLTK 25%, calibrated) + +| Metric | Value | +|--------|------:| +| Accuracy | 1.000 | +| Precision | 1.000 | +| Recall | 1.000 | +| F1 | 1.000 | +| AUROC | 1.000 | +| **False-positive rate** (human flagged as AI) | **0.000** | +| False-negative rate (AI missed) | 0.000 | +| Expected calibration error | 0.175 | + +![ROC](roc_ensemble.png) ![Calibration](calibration_ensemble.png) + +## Why this matters: the C2 fix + +The previous ensemble mapped every analyzer's perplexity to an AI-score with +`1 - perplexity / 500`. Human text has a GPT-2 perplexity around 58, which that +formula turned into `1 - 58/500 = 0.88` — **88% AI for ordinary human writing.** +That systematic false-positive bias is the audit's critical finding C2. + +The calibrated logistic (per-analyzer midpoint = decision boundary; see +`src/analyzers/calibration.py`) fixes the direction and scale for each analyzer. +On the benchmark, human text now sits well below the 0.5 boundary and the +false-positive rate is **0.000**. + +## Binoculars (cross-perplexity, modern) + +A two-model detector (observer `gpt2` + performer `distilgpt2`) after Hans et +al., 2024 — the SOTA-aligned modernization from the competitive audit. It scores +text by the ratio of the observer's log-perplexity to the observer/performer +cross-perplexity, which cancels the prompt/topic bias that makes single-model +GPT-2 perplexity brittle. + +| Metric | Value | +|--------|------:| +| Accuracy | 1.000 | +| F1 | 1.000 | +| AUROC | 1.000 | +| **False-positive rate** | **0.000** | +| Expected calibration error | 0.066 | + +![ROC](roc_binoculars.png) ![Calibration](calibration_binoculars.png) + +On the benchmark, human scores cluster ~0.88–1.05 and AI ~0.72–0.84 with a clean +gap; the decision midpoint (0.863) sits between the clusters. Reproduce with +`python -m src.evaluation.benchmark --analyzer binoculars`. It is available as a +standalone analyzer and is not enabled in the default ensemble (to keep the +default lightweight — it needs a second model). + +## Single-analyzer baselines + +| Analyzer | AUROC | Notes | +|----------|------:|-------| +| NLTK only | ~0.41 | Brown-corpus (1961) n-gram signal is weak/near-inverted for modern text; carries only 25% ensemble weight for this reason. | +| Ensemble | 1.000 | GPT-2 perplexity is the dominant, strongly-separating signal. | + +Run `python -m src.evaluation.benchmark --analyzer nltk` to reproduce the NLTK +baseline. The gap is exactly why the ensemble weights GPT-2 heavily. + +> These results reflect a clean, in-distribution set. Expect materially lower +> numbers on adversarial or edited text — evaluate on a large public benchmark +> (RAID, HC3) via `--dataset` before making any external accuracy claim. diff --git a/docs/benchmarks/binoculars_report.json b/docs/benchmarks/binoculars_report.json new file mode 100644 index 0000000..3c04f32 --- /dev/null +++ b/docs/benchmarks/binoculars_report.json @@ -0,0 +1,309 @@ +{ + "analyzer": "binoculars", + "n_samples": 24, + "metrics_at_0.5": { + "threshold": 0.5, + "n_samples": 24, + "n_positive": 12, + "n_negative": 12, + "true_positives": 12, + "false_positives": 0, + "true_negatives": 12, + "false_negatives": 0, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "specificity": 1.0, + "false_positive_rate": 0.0, + "false_negative_rate": 0.0, + "roc_auc": 1.0, + "expected_calibration_error": 0.06614583333333336 + }, + "best_f1_threshold": 0.8435, + "metrics_at_best_f1": { + "threshold": 0.8435, + "n_samples": 24, + "n_positive": 12, + "n_negative": 12, + "true_positives": 12, + "false_positives": 0, + "true_negatives": 12, + "false_negatives": 0, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "specificity": 1.0, + "false_positive_rate": 0.0, + "false_negative_rate": 0.0, + "roc_auc": 1.0, + "expected_calibration_error": 0.06614583333333336 + }, + "calibration_bins": [ + { + "bin_lower": 0.0, + "bin_upper": 0.1, + "count": 12, + "mean_predicted": 0.041666666666666685, + "observed_fraction": 0.0 + }, + { + "bin_lower": 0.1, + "bin_upper": 0.2, + "count": 0, + "mean_predicted": NaN, + "observed_fraction": NaN + }, + { + "bin_lower": 0.2, + "bin_upper": 0.30000000000000004, + "count": 0, + "mean_predicted": NaN, + "observed_fraction": NaN + }, + { + "bin_lower": 0.30000000000000004, + "bin_upper": 0.4, + "count": 0, + "mean_predicted": NaN, + "observed_fraction": NaN + }, + { + "bin_lower": 0.4, + "bin_upper": 0.5, + "count": 0, + "mean_predicted": NaN, + "observed_fraction": NaN + }, + { + "bin_lower": 0.5, + "bin_upper": 0.6000000000000001, + "count": 0, + "mean_predicted": NaN, + "observed_fraction": NaN + }, + { + "bin_lower": 0.6000000000000001, + "bin_upper": 0.7000000000000001, + "count": 0, + "mean_predicted": NaN, + "observed_fraction": NaN + }, + { + "bin_lower": 0.7000000000000001, + "bin_upper": 0.8, + "count": 0, + "mean_predicted": NaN, + "observed_fraction": NaN + }, + { + "bin_lower": 0.8, + "bin_upper": 0.9, + "count": 5, + "mean_predicted": 0.8554, + "observed_fraction": 1.0 + }, + { + "bin_lower": 0.9, + "bin_upper": 1.0, + "count": 7, + "mean_predicted": 0.9479285714285713, + "observed_fraction": 1.0 + } + ], + "predictions": [ + { + "id": "human-01", + "label": 0, + "source": "handwritten-casual", + "ai_probability": 0.025, + "verdict": "Human-Written", + "confidence": 95.0 + }, + { + "id": "human-02", + "label": 0, + "source": "handwritten-casual", + "ai_probability": 0.025, + "verdict": "Human-Written", + "confidence": 95.0 + }, + { + "id": "human-03", + "label": 0, + "source": "handwritten-opinion", + "ai_probability": 0.025, + "verdict": "Human-Written", + "confidence": 95.0 + }, + { + "id": "human-04", + "label": 0, + "source": "handwritten-forum", + "ai_probability": 0.025, + "verdict": "Human-Written", + "confidence": 95.0 + }, + { + "id": "human-05", + "label": 0, + "source": "handwritten-travel", + "ai_probability": 0.025, + "verdict": "Human-Written", + "confidence": 95.0 + }, + { + "id": "human-06", + "label": 0, + "source": "handwritten-review", + "ai_probability": 0.075, + "verdict": "Likely Human-Written", + "confidence": 85.0 + }, + { + "id": "human-07", + "label": 0, + "source": "handwritten-journal", + "ai_probability": 0.075, + "verdict": "Likely Human-Written", + "confidence": 85.0 + }, + { + "id": "human-08", + "label": 0, + "source": "handwritten-recipe-note", + "ai_probability": 0.075, + "verdict": "Likely Human-Written", + "confidence": 85.0 + }, + { + "id": "human-09", + "label": 0, + "source": "handwritten-rant", + "ai_probability": 0.025, + "verdict": "Human-Written", + "confidence": 95.0 + }, + { + "id": "human-10", + "label": 0, + "source": "handwritten-sports", + "ai_probability": 0.025, + "verdict": "Human-Written", + "confidence": 95.0 + }, + { + "id": "human-11", + "label": 0, + "source": "handwritten-work", + "ai_probability": 0.075, + "verdict": "Likely Human-Written", + "confidence": 85.0 + }, + { + "id": "human-12", + "label": 0, + "source": "handwritten-hobby", + "ai_probability": 0.025, + "verdict": "Human-Written", + "confidence": 95.0 + }, + { + "id": "ai-01", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.9405, + "verdict": "AI-Generated", + "confidence": 88.1 + }, + { + "id": "ai-02", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.9425, + "verdict": "AI-Generated", + "confidence": 88.5 + }, + { + "id": "ai-03", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.9665, + "verdict": "AI-Generated", + "confidence": 93.3 + }, + { + "id": "ai-04", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.856, + "verdict": "Likely AI-Generated", + "confidence": 71.2 + }, + { + "id": "ai-05", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.8465, + "verdict": "Likely AI-Generated", + "confidence": 69.3 + }, + { + "id": "ai-06", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.947, + "verdict": "AI-Generated", + "confidence": 89.4 + }, + { + "id": "ai-07", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.9475, + "verdict": "AI-Generated", + "confidence": 89.5 + }, + { + "id": "ai-08", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.9445, + "verdict": "AI-Generated", + "confidence": 88.9 + }, + { + "id": "ai-09", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.864, + "verdict": "Likely AI-Generated", + "confidence": 72.8 + }, + { + "id": "ai-10", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.947, + "verdict": "AI-Generated", + "confidence": 89.4 + }, + { + "id": "ai-11", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.867, + "verdict": "Likely AI-Generated", + "confidence": 73.4 + }, + { + "id": "ai-12", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.8435, + "verdict": "Likely AI-Generated", + "confidence": 68.7 + } + ] +} \ No newline at end of file diff --git a/docs/benchmarks/calibration_binoculars.png b/docs/benchmarks/calibration_binoculars.png new file mode 100644 index 0000000..9c917c5 Binary files /dev/null and b/docs/benchmarks/calibration_binoculars.png differ diff --git a/docs/benchmarks/calibration_ensemble.png b/docs/benchmarks/calibration_ensemble.png new file mode 100644 index 0000000..a1b54e4 Binary files /dev/null and b/docs/benchmarks/calibration_ensemble.png differ diff --git a/docs/benchmarks/ensemble_report.json b/docs/benchmarks/ensemble_report.json new file mode 100644 index 0000000..8bfa808 --- /dev/null +++ b/docs/benchmarks/ensemble_report.json @@ -0,0 +1,309 @@ +{ + "analyzer": "ensemble", + "n_samples": 24, + "metrics_at_0.5": { + "threshold": 0.5, + "n_samples": 24, + "n_positive": 12, + "n_negative": 12, + "true_positives": 12, + "false_positives": 0, + "true_negatives": 12, + "false_negatives": 0, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "specificity": 1.0, + "false_positive_rate": 0.0, + "false_negative_rate": 0.0, + "roc_auc": 1.0, + "expected_calibration_error": 0.17479692239902392 + }, + "best_f1_threshold": 0.7171, + "metrics_at_best_f1": { + "threshold": 0.7170979323317036, + "n_samples": 24, + "n_positive": 12, + "n_negative": 12, + "true_positives": 12, + "false_positives": 0, + "true_negatives": 12, + "false_negatives": 0, + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "specificity": 1.0, + "false_positive_rate": 0.0, + "false_negative_rate": 0.0, + "roc_auc": 1.0, + "expected_calibration_error": 0.17479692239902392 + }, + "calibration_bins": [ + { + "bin_lower": 0.0, + "bin_upper": 0.1, + "count": 5, + "mean_predicted": 0.0784992646892875, + "observed_fraction": 0.0 + }, + { + "bin_lower": 0.1, + "bin_upper": 0.2, + "count": 4, + "mean_predicted": 0.15334347487058422, + "observed_fraction": 0.0 + }, + { + "bin_lower": 0.2, + "bin_upper": 0.30000000000000004, + "count": 3, + "mean_predicted": 0.2481011239096301, + "observed_fraction": 0.0 + }, + { + "bin_lower": 0.30000000000000004, + "bin_upper": 0.4, + "count": 0, + "mean_predicted": NaN, + "observed_fraction": NaN + }, + { + "bin_lower": 0.4, + "bin_upper": 0.5, + "count": 0, + "mean_predicted": NaN, + "observed_fraction": NaN + }, + { + "bin_lower": 0.5, + "bin_upper": 0.6000000000000001, + "count": 0, + "mean_predicted": NaN, + "observed_fraction": NaN + }, + { + "bin_lower": 0.6000000000000001, + "bin_upper": 0.7000000000000001, + "count": 0, + "mean_predicted": NaN, + "observed_fraction": NaN + }, + { + "bin_lower": 0.7000000000000001, + "bin_upper": 0.8, + "count": 5, + "mean_predicted": 0.7510220293083606, + "observed_fraction": 1.0 + }, + { + "bin_lower": 0.8, + "bin_upper": 0.9, + "count": 7, + "mean_predicted": 0.828562472934184, + "observed_fraction": 1.0 + }, + { + "bin_lower": 0.9, + "bin_upper": 1.0, + "count": 0, + "mean_predicted": NaN, + "observed_fraction": NaN + } + ], + "predictions": [ + { + "id": "human-01", + "label": 0, + "source": "handwritten-casual", + "ai_probability": 0.1902, + "verdict": "Human-Written", + "confidence": 95 + }, + { + "id": "human-02", + "label": 0, + "source": "handwritten-casual", + "ai_probability": 0.0724, + "verdict": "Human-Written", + "confidence": 95 + }, + { + "id": "human-03", + "label": 0, + "source": "handwritten-opinion", + "ai_probability": 0.2048, + "verdict": "Human-Written", + "confidence": 95 + }, + { + "id": "human-04", + "label": 0, + "source": "handwritten-forum", + "ai_probability": 0.0942, + "verdict": "Human-Written", + "confidence": 95 + }, + { + "id": "human-05", + "label": 0, + "source": "handwritten-travel", + "ai_probability": 0.1468, + "verdict": "Human-Written", + "confidence": 95 + }, + { + "id": "human-06", + "label": 0, + "source": "handwritten-review", + "ai_probability": 0.1083, + "verdict": "Human-Written", + "confidence": 95 + }, + { + "id": "human-07", + "label": 0, + "source": "handwritten-journal", + "ai_probability": 0.2467, + "verdict": "Human-Written", + "confidence": 95 + }, + { + "id": "human-08", + "label": 0, + "source": "handwritten-recipe-note", + "ai_probability": 0.2928, + "verdict": "Human-Written", + "confidence": 95 + }, + { + "id": "human-09", + "label": 0, + "source": "handwritten-rant", + "ai_probability": 0.0682, + "verdict": "Human-Written", + "confidence": 95 + }, + { + "id": "human-10", + "label": 0, + "source": "handwritten-sports", + "ai_probability": 0.0972, + "verdict": "Human-Written", + "confidence": 95 + }, + { + "id": "human-11", + "label": 0, + "source": "handwritten-work", + "ai_probability": 0.168, + "verdict": "Human-Written", + "confidence": 95 + }, + { + "id": "human-12", + "label": 0, + "source": "handwritten-hobby", + "ai_probability": 0.0605, + "verdict": "Human-Written", + "confidence": 95 + }, + { + "id": "ai-01", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.7574, + "verdict": "AI-Generated", + "confidence": 86.6 + }, + { + "id": "ai-02", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.816, + "verdict": "AI-Generated", + "confidence": 88.1 + }, + { + "id": "ai-03", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.8317, + "verdict": "AI-Generated", + "confidence": 90.0 + }, + { + "id": "ai-04", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.7171, + "verdict": "AI-Generated", + "confidence": 86.5 + }, + { + "id": "ai-05", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.8109, + "verdict": "AI-Generated", + "confidence": 89.3 + }, + { + "id": "ai-06", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.8391, + "verdict": "AI-Generated", + "confidence": 90.2 + }, + { + "id": "ai-07", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.7397, + "verdict": "AI-Generated", + "confidence": 86.1 + }, + { + "id": "ai-08", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.8097, + "verdict": "AI-Generated", + "confidence": 87.9 + }, + { + "id": "ai-09", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.753, + "verdict": "AI-Generated", + "confidence": 87.6 + }, + { + "id": "ai-10", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.83, + "verdict": "AI-Generated", + "confidence": 89.9 + }, + { + "id": "ai-11", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.788, + "verdict": "AI-Generated", + "confidence": 88.6 + }, + { + "id": "ai-12", + "label": 1, + "source": "llm-formal", + "ai_probability": 0.8626, + "verdict": "AI-Generated", + "confidence": 90.9 + } + ] +} \ No newline at end of file diff --git a/docs/benchmarks/roc_binoculars.png b/docs/benchmarks/roc_binoculars.png new file mode 100644 index 0000000..2b717af Binary files /dev/null and b/docs/benchmarks/roc_binoculars.png differ diff --git a/docs/benchmarks/roc_ensemble.png b/docs/benchmarks/roc_ensemble.png new file mode 100644 index 0000000..1d82085 Binary files /dev/null and b/docs/benchmarks/roc_ensemble.png differ diff --git a/ensemble.py b/ensemble.py index bda15e3..aed1545 100644 --- a/ensemble.py +++ b/ensemble.py @@ -1,4 +1,4 @@ -""" +""" AI Text Detector — Ensemble Analysis ====================================== @@ -11,8 +11,8 @@ streamlit run ensemble.py """ -import sys import os +import sys # Add src to path sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src")) @@ -20,7 +20,14 @@ import streamlit as st from src.analyzers.ensemble_analyzer import EnsembleAnalyzer -from src.config.settings import Verdict, get_settings +from src.config.settings import get_settings +from src.ui import ( + inject_css, + render_error, + render_footer, + render_verdict_card, + render_warnings, +) from src.utils.logging_config import get_logger, setup_logging from src.utils.ui_contract import ( build_limitations_markdown, @@ -46,14 +53,9 @@ # ─── Custom CSS ────────────────────────────────────────────────────────────── -st.markdown(""" - -""", unsafe_allow_html=True) +""") # ─── Cached Resources ─────────────────────────────────────────────────────── -@st.cache_resource(show_spinner="Loading ensemble models... (this may take 2-3 minutes on first run)") + +@st.cache_resource( + show_spinner="Loading ensemble models... (this may take 2-3 minutes on first run)" +) def load_analyzer() -> EnsembleAnalyzer: """Load and cache the Ensemble analyzer.""" analyzer = EnsembleAnalyzer() - # Force model loading - _ = analyzer.roberta_analyzer + # Warm the analyzers that actually contribute to the blend. RoBERTa is + # intentionally NOT loaded here: it is disabled (weight 0) and warming it + # would trigger a large download and memory use for no contribution. + if analyzer.weights.get("roberta", 0.0) > 0: + _ = analyzer.roberta_analyzer _ = analyzer.gpt2_analyzer _ = analyzer.nltk_analyzer return analyzer @@ -236,30 +145,28 @@ def load_chart_generator() -> ChartGenerator: st.markdown("---") st.markdown("### ℹ️ About") - st.markdown( - """ - **Ensemble Analyzer** combines two powerful detection methods: - - 🧠 **GPT-2** (65% weight) - - Deep perplexity analysis - - Transformer-based patterns - - 📊 **NLTK** (35% weight) - - Statistical n-gram models - - Linguistic features - - ⚠️ **Note**: RoBERTa is disabled (requires fine-tuning). - See README for fine-tuning guide. - - **Benchmarks:** pending validation - - **Processing:** 5-10 seconds - - **Memory:** 2-3 GB RAM - - **Version:** 2.0.0 - """ - ) + st.markdown(""" +**Ensemble Analyzer** combines two calibrated detection signals: + +🧠 **GPT-2** (75% weight) +- Deep perplexity analysis +- Transformer-based patterns + +📊 **NLTK** (25% weight) +- Statistical n-gram models +- Linguistic features + +⚠️ **Note**: RoBERTa is disabled (weight 0, not loaded) until a fine-tuned +checkpoint is wired in. See README. + +**Benchmarks:** see `docs/benchmarks/` + +**Processing:** 5-10 seconds + +**Memory:** 2-3 GB RAM + +**Version:** 2.0.0 +""") st.markdown("---") st.markdown( @@ -273,18 +180,18 @@ def load_chart_generator() -> ChartGenerator: st.markdown("---") st.markdown("### ?? Why Ensemble?") - + st.markdown(""" - Combining multiple models provides: - - ✅ **Multi-signal** - Combines GPT-2 perplexity with NLTK statistics - - ✅ **Consensus** - Aggregates analyzer outputs into a single verdict - - ✅ **Transparent** - See how each analyzer votes - - ✅ **Weighted fusion** - GPT-2 65% / NLTK 35% (configurable) - """) +Combining multiple models provides: + +✅ **Multi-signal** - Combines GPT-2 perplexity with NLTK statistics + +✅ **Consensus** - Aggregates analyzer outputs into a single verdict + +✅ **Transparent** - See how each analyzer votes + +✅ **Weighted fusion** - GPT-2 75% / NLTK 25% (configurable) +""") st.markdown("---") st.markdown(build_limitations_markdown()) @@ -292,7 +199,8 @@ def load_chart_generator() -> ChartGenerator: # ─── Main Content ──────────────────────────────────────────────────────────── # Header -st.markdown(""" +st.markdown( + """

🎯 AI Text Detector — Ensemble

@@ -301,15 +209,21 @@ def load_chart_generator() -> ChartGenerator: • Multi-Signal Detection • Multi-Model Consensus • Experimental

-""", unsafe_allow_html=True) +""", + unsafe_allow_html=True, +) # Model loading notice -st.markdown(""" +st.markdown( + """
- 💡 First-time setup: The ensemble will download RoBERTa (~500MB), - GPT-2 (~500MB), and NLTK data (~50MB). Subsequent runs will be much faster. + 💡 First-time setup: The ensemble will download + GPT-2 (~500MB) and NLTK data (~50MB). Subsequent runs will be much faster. + (RoBERTa is disabled and not downloaded.)
-""", unsafe_allow_html=True) +""", + unsafe_allow_html=True, +) st.markdown("") @@ -382,46 +296,17 @@ def load_chart_generator() -> ChartGenerator: st.markdown("---") st.markdown("### 🎯 Detection Result") - verdict_class = { - Verdict.AI_GENERATED: "verdict-ai", - Verdict.LIKELY_AI: "verdict-likely-ai", - Verdict.UNCERTAIN: "verdict-uncertain", - Verdict.LIKELY_HUMAN: "verdict-likely-human", - Verdict.HUMAN_WRITTEN: "verdict-human", - } - - verdict_emoji = { - Verdict.AI_GENERATED: "🤖", - Verdict.LIKELY_AI: "🤖", - Verdict.UNCERTAIN: "❓", - Verdict.LIKELY_HUMAN: "👤", - Verdict.HUMAN_WRITTEN: "👤", - } - - css_class = verdict_class.get(result.verdict, "verdict-uncertain") - emoji = verdict_emoji.get(result.verdict, "❓") - - st.markdown(f""" -
-

{emoji} {result.verdict.value}

-

Confidence: {result.confidence:.1f}% ({result.confidence_level.value}) - • Analysis Time: {result.analysis_time:.2f}s

-
- """, unsafe_allow_html=True) + render_verdict_card(result) st.caption(build_result_reminder_markdown()) - # -- Confidence Gauge -- ── + # -- Confidence Gauge -- if show_gauge: fig_gauge = charts.create_metrics_gauge(result) st.plotly_chart(fig_gauge, use_container_width=True) # ── Warnings ── - if result.warnings: - for warning in result.warnings: - st.markdown(f""" -
⚠️ {warning}
- """, unsafe_allow_html=True) + render_warnings(result) # ── Explanation ── st.markdown("### 💡 Analysis Explanation") @@ -432,12 +317,15 @@ def load_chart_generator() -> ChartGenerator: for score in result.scores: indicator = "🔴" if score.indicates_ai else "🟢" - st.markdown(f""" + st.markdown( + f"""
{indicator} {score.name}: {score.value:.4f} (Weight: {score.weight:.0%}) — {score.interpretation}
- """, unsafe_allow_html=True) + """, + unsafe_allow_html=True, + ) # ── Visualizations ── st.markdown("### 📈 Visualizations") @@ -471,19 +359,24 @@ def load_chart_generator() -> ChartGenerator: if show_comparison: with tab_objects[tab_idx]: st.markdown("#### Individual Analyzer Results") - + # Extract individual scores scores_data = [] for score in result.scores[1:]: # Skip ensemble score - scores_data.append({ - "Analyzer": score.name.replace(" Score", "").replace(" Perplexity Score", "").replace(" Statistical Score", ""), - "Value": f"{score.value:.2%}", - "Weight": f"{score.weight:.0%}", - "Verdict": score.interpretation - }) - + scores_data.append( + { + "Analyzer": score.name.replace(" Score", "") + .replace(" Perplexity Score", "") + .replace(" Statistical Score", ""), + "Value": f"{score.value:.2%}", + "Weight": f"{score.weight:.0%}", + "Verdict": score.interpretation, + } + ) + if scores_data: import pandas as pd + df = pd.DataFrame(scores_data) st.dataframe(df, hide_index=True, use_container_width=True) @@ -512,13 +405,13 @@ def load_chart_generator() -> ChartGenerator: except Exception as e: progress_bar.empty() - logger.error(f"Application error: {e}", exc_info=True) - st.error(f"❌ An error occurred: {str(e)}") + render_error(e) st.info( "💡 This might be due to:\n" - "- Insufficient memory (ensemble requires 4-6GB RAM)\n" + "- Insufficient memory (ensemble requires 2-3GB RAM)\n" "- Network issues during model download\n" - "- Try individual analyzers (`streamlit run app.py` or `streamlit run test.py`) as alternatives" + "- Try individual analyzers (`streamlit run app.py` or " + "`streamlit run gpt2_app.py`) as alternatives" ) # ─── Empty State ───────────────────────────────────────────────────────────── @@ -564,11 +457,14 @@ def load_chart_generator() -> ChartGenerator: st.session_state["text_example"] = example_human st.rerun() - st.markdown(""" + st.markdown( + """
👆 Paste text above or use an example, then click Analyze with Ensemble
- """, unsafe_allow_html=True) + """, + unsafe_allow_html=True, + ) # Handle example text injection if "text_example" in st.session_state: @@ -576,18 +472,8 @@ def load_chart_generator() -> ChartGenerator: # ─── Footer ────────────────────────────────────────────────────────────────── -st.markdown(""" -
-

🎯 AI Text Detector v2.0.0 • Ensemble Analysis Engine (GPT-2 + NLTK)
- ⚠️ RoBERTa disabled (requires fine-tuning)

-

⚠️ Results are probabilistic estimates, not definitive classifications.

-

No text is stored or transmitted. All processing happens locally.

-
-""", unsafe_allow_html=True) - - - - - - - +render_footer( + "Ensemble Analysis Engine (GPT-2 + NLTK)", + icon="🎯", + note="RoBERTa disabled (requires fine-tuning)", +) diff --git a/test.py b/gpt2_app.py similarity index 75% rename from test.py rename to gpt2_app.py index 9251455..1878b38 100644 --- a/test.py +++ b/gpt2_app.py @@ -6,11 +6,11 @@ for advanced AI-generated text detection. Usage: - streamlit run test.py + streamlit run gpt2_app.py """ -import sys import os +import sys # Add src to path sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src")) @@ -18,7 +18,14 @@ import streamlit as st from src.analyzers.gpt2_analyzer import GPT2Analyzer -from src.config.settings import Verdict, get_settings +from src.config.settings import get_settings +from src.ui import ( + inject_css, + render_error, + render_footer, + render_verdict_card, + render_warnings, +) from src.utils.logging_config import get_logger, setup_logging from src.utils.ui_contract import ( build_limitations_markdown, @@ -44,14 +51,9 @@ # ─── Custom CSS ────────────────────────────────────────────────────────────── -st.markdown(""" - -""", unsafe_allow_html=True) +""") # ─── Cached Resources ─────────────────────────────────────────────────────── + @st.cache_resource(show_spinner="Loading GPT-2 model... (this may take a minute on first run)") def load_analyzer() -> GPT2Analyzer: """Load and cache the GPT-2 analyzer.""" @@ -218,8 +127,7 @@ def load_chart_generator() -> ChartGenerator: st.markdown("---") st.markdown("### ℹ️ About") - st.markdown( - """ + st.markdown(""" **GPT-2 Deep Analyzer** uses the GPT-2 transformer model (124M parameters) for advanced perplexity-based detection. @@ -231,14 +139,13 @@ def load_chart_generator() -> ChartGenerator: **Note:** First run downloads the GPT-2 model (~500MB). **Version:** 2.0.0 - """ - ) + """) st.markdown("---") st.markdown( build_mode_guidance_markdown( mode_label="GPT-2", - launch_command="streamlit run test.py", + launch_command="streamlit run gpt2_app.py", speed_hint="2-5s", memory_hint="2-3 GB", ) @@ -276,20 +183,26 @@ def load_chart_generator() -> ChartGenerator: # ─── Main Content ──────────────────────────────────────────────────────────── # Header -st.markdown(""" +st.markdown( + """

🧠 AI Text Detector

GPT-2 Deep Analysis • Transformer-Based Detection • Advanced Pattern Recognition

-""", unsafe_allow_html=True) +""", + unsafe_allow_html=True, +) # Model loading notice -st.markdown(""" +st.markdown( + """
💡 First-time setup: The GPT-2 model (~500MB) will be downloaded and cached automatically. Subsequent runs will be much faster.
-""", unsafe_allow_html=True) +""", + unsafe_allow_html=True, +) st.markdown("") @@ -362,46 +275,17 @@ def load_chart_generator() -> ChartGenerator: st.markdown("---") st.markdown("### 🎯 Detection Result") - verdict_class = { - Verdict.AI_GENERATED: "verdict-ai", - Verdict.LIKELY_AI: "verdict-likely-ai", - Verdict.UNCERTAIN: "verdict-uncertain", - Verdict.LIKELY_HUMAN: "verdict-likely-human", - Verdict.HUMAN_WRITTEN: "verdict-human", - } - - verdict_emoji = { - Verdict.AI_GENERATED: "🤖", - Verdict.LIKELY_AI: "🤖", - Verdict.UNCERTAIN: "❓", - Verdict.LIKELY_HUMAN: "👤", - Verdict.HUMAN_WRITTEN: "👤", - } - - css_class = verdict_class.get(result.verdict, "verdict-uncertain") - emoji = verdict_emoji.get(result.verdict, "❓") - - st.markdown(f""" -
-

{emoji} {result.verdict.value}

-

Confidence: {result.confidence:.1f}% ({result.confidence_level.value}) - • Analysis Time: {result.analysis_time:.2f}s

-
- """, unsafe_allow_html=True) + render_verdict_card(result) st.caption(build_result_reminder_markdown()) - # -- Confidence Gauge -- ── + # -- Confidence Gauge -- if show_gauge: fig_gauge = charts.create_metrics_gauge(result) st.plotly_chart(fig_gauge, use_container_width=True) # ── Warnings ── - if result.warnings: - for warning in result.warnings: - st.markdown(f""" -
⚠️ {warning}
- """, unsafe_allow_html=True) + render_warnings(result) # ── Key Metrics ── st.markdown("### 📊 Key Metrics") @@ -409,7 +293,8 @@ def load_chart_generator() -> ChartGenerator: col1, col2, col3, col4 = st.columns(4) with col1: - st.markdown(f""" + st.markdown( + f"""
{result.perplexity:.1f}
GPT-2 Perplexity
@@ -417,10 +302,13 @@ def load_chart_generator() -> ChartGenerator: {"Low = AI-like" if result.perplexity < 200 else "High = Human-like"}
- """, unsafe_allow_html=True) + """, + unsafe_allow_html=True, + ) with col2: - st.markdown(f""" + st.markdown( + f"""
{result.burstiness:.3f}
Burstiness
@@ -428,10 +316,13 @@ def load_chart_generator() -> ChartGenerator: {"Uniform" if result.burstiness < 0.25 else "Natural variation"}
- """, unsafe_allow_html=True) + """, + unsafe_allow_html=True, + ) with col3: - st.markdown(f""" + st.markdown( + f"""
{result.lexical_diversity:.1%}
Lexical Diversity
@@ -439,10 +330,13 @@ def load_chart_generator() -> ChartGenerator: {"Low" if result.lexical_diversity < 0.5 else "Good"} vocabulary variety
- """, unsafe_allow_html=True) + """, + unsafe_allow_html=True, + ) with col4: - st.markdown(f""" + st.markdown( + f"""
{result.sentence_variance:.3f}
Sentence Variance
@@ -450,7 +344,9 @@ def load_chart_generator() -> ChartGenerator: {"Uniform" if result.sentence_variance < 0.25 else "Varied"} structure
- """, unsafe_allow_html=True) + """, + unsafe_allow_html=True, + ) # ── Explanation ── st.markdown("### 💡 Analysis Explanation") @@ -461,12 +357,15 @@ def load_chart_generator() -> ChartGenerator: for score in result.scores: indicator = "🔴" if score.indicates_ai else "🟢" - st.markdown(f""" + st.markdown( + f"""
{indicator} {score.name}: {score.value:.4f} (Weight: {score.weight:.0%}) — {score.interpretation}
- """, unsafe_allow_html=True) + """, + unsafe_allow_html=True, + ) # ── Visualizations ── st.markdown("### 📈 Visualizations") @@ -500,9 +399,7 @@ def load_chart_generator() -> ChartGenerator: if show_sentence_chart: with tab_objects[tab_idx]: if result.metrics.sentence_lengths: - fig_sent = charts.create_sentence_length_chart( - result.metrics.sentence_lengths - ) + fig_sent = charts.create_sentence_length_chart(result.metrics.sentence_lengths) st.plotly_chart(fig_sent, use_container_width=True) else: st.info("Not enough sentences for length analysis.") @@ -532,8 +429,7 @@ def load_chart_generator() -> ChartGenerator: except Exception as e: progress_bar.empty() - logger.error(f"Application error: {e}", exc_info=True) - st.error(f"❌ An error occurred: {str(e)}") + render_error(e) st.info( "💡 This might be due to:\n" "- Insufficient memory for GPT-2 model\n" @@ -584,11 +480,14 @@ def load_chart_generator() -> ChartGenerator: st.session_state["text_example"] = example_human st.rerun() - st.markdown(""" + st.markdown( + """
👆 Paste text above or use an example, then click Deep Analyze with GPT-2
- """, unsafe_allow_html=True) + """, + unsafe_allow_html=True, + ) # Handle example text injection if "text_example" in st.session_state: @@ -596,17 +495,4 @@ def load_chart_generator() -> ChartGenerator: # ─── Footer ────────────────────────────────────────────────────────────────── -st.markdown(""" -
-

🧠 AI Text Detector v2.0.0 • GPT-2 Deep Analysis Engine

-

⚠️ Results are probabilistic estimates, not definitive classifications.

-

No text is stored or transmitted. All processing happens locally.

-
-""", unsafe_allow_html=True) - - - - - - - +render_footer("GPT-2 Deep Analysis Engine", icon="🧠") diff --git a/requirements.txt b/requirements.txt index 83199be..7276be4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,11 @@ streamlit>=1.28.0,<2.0.0 # NLP Libraries nltk>=3.8.1,<4.0.0 -transformers>=4.35.0,<5.0.0 -torch>=2.0.0,<3.0.0 +# transformers 4.48+ / torch 2.6+ raise the floor above known deserialization +# advisories and enable safe (safetensors) weight loading by default. See +# docs/SECURITY considerations and https://github.com/huggingface/transformers/issues/38464 +transformers>=4.48.0,<5.0.0 +torch>=2.6.0,<3.0.0 # Data Processing numpy>=1.24.0,<2.0.0 @@ -12,11 +15,4 @@ pandas>=2.0.0,<3.0.0 # Visualization matplotlib>=3.7.0,<4.0.0 -plotly>=5.18.0,<6.0.0 - -# Utilities -pydantic>=2.0.0,<3.0.0 -python-dotenv>=1.0.0,<2.0.0 - -# Logging -structlog>=23.1.0,<24.0.0 \ No newline at end of file +plotly>=5.18.0,<6.0.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 538c895..06f896a 100644 --- a/setup.py +++ b/setup.py @@ -15,12 +15,15 @@ setup( name="ai-text-detector", version="2.0.0", - author="AI Detection Community", - author_email="support@ai-text-detector.dev", - description="Production-ready AI text detection system", + author="Satyam Shivam", + author_email="shivamsatyam35@gmail.com", + description=( + "Explainable, local, multi-signal toolkit for estimating how likely " + "text is AI-generated" + ), long_description=long_description, long_description_content_type="text/markdown", - url="https://github.com/yourusername/ai-text-detector", + url="https://github.com/satyamshivam13/AI_Text_Detector", packages=find_packages(where="src"), package_dir={"": "src"}, classifiers=[ diff --git a/src/__init__.py b/src/__init__.py index 4b69dac..72f9adf 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -14,4 +14,4 @@ __version__ = "2.0.0" __author__ = "AI Detection Community" -__license__ = "MIT" \ No newline at end of file +__license__ = "MIT" diff --git a/src/analyzers/AGENTS.md b/src/analyzers/AGENTS.md new file mode 100644 index 0000000..b98ebec --- /dev/null +++ b/src/analyzers/AGENTS.md @@ -0,0 +1,48 @@ +# src/analyzers — Detection Pipeline + +## Purpose +Owns all text-to-`AnalysisResult` logic. Provides `BaseAnalyzer` (abstract template), +three concrete backends (NLTK, GPT-2, RoBERTa), and `EnsembleAnalyzer` (multi-model fusion). +Does **not** own UI rendering, chart generation, or settings — those live in `utils/` and `config/`. + +## Entry Points +- `base_analyzer.py` — Abstract base; `analyze()` is the only public method callers use +- `nltk_analyzer.py` — Brown-corpus n-gram perplexity; fast, no GPU needed +- `gpt2_analyzer.py` — GPT-2 token-loss perplexity; requires `torch` + `transformers` +- `roberta_analyzer.py` — RoBERTa sequence classifier; currently disabled (needs fine-tuning) +- `binoculars_analyzer.py` — Cross-perplexity (observer+performer) detector; optional, off by default +- `calibration.py` — `logistic_ai_probability`: maps perplexity → calibrated AI-probability +- `ensemble_analyzer.py` — Fuses GPT-2 (75%) + NLTK (25%), with optional RoBERTa and Binoculars slots +- `__init__.py` — Eagerly exports `BaseAnalyzer`, `NLTKAnalyzer`; lazily exports the torch-backed analyzers + +## Contracts & Invariants +- **Only** call `analyzer.analyze(text: str) → AnalysisResult`. Never call `_perform_analysis` directly from outside this package. +- `_perform_analysis(text, result)` is the single hook subclasses implement — it receives cleaned text and a partially-populated result, and must return the same result object with `perplexity`, `burstiness`, `lexical_diversity`, `sentence_variance`, and `scores` populated. +- `BaseAnalyzer._determine_verdict` is the shared scoring engine. Do not duplicate its logic in subclasses — call `super()` or override with care. +- `EnsembleAnalyzer` **overrides** `analyze()` entirely (not just `_perform_analysis`) to run sub-analyzers in sequence and fuse. The weights are in `EnsembleAnalyzer` itself — change them there, not in individual analyzers. +- Lazy import contract in `__init__.py`: `GPT2Analyzer`, `RoBERTaAnalyzer`, `BinocularsAnalyzer`, `EnsembleAnalyzer` are loaded via `__getattr__` using `_LAZY_MODULES`. Do not add them to the eager import block — `app.py` must stay torch-free. +- Error contract: all analysis exceptions must be caught inside `BaseAnalyzer.analyze()`. Subclasses should let exceptions bubble up from `_perform_analysis` — the base catches them and sets `Verdict.UNCERTAIN`. + +## Patterns +To add a new analyzer backend: +1. Create `src/analyzers/my_analyzer.py` with a class extending `BaseAnalyzer` +2. Implement `_perform_analysis(text, result) → AnalysisResult` only +3. Populate `result.perplexity`, `result.burstiness`, `result.lexical_diversity`, `result.sentence_variance` +4. Use `result.add_score(DetectionScore(...))` for per-signal scores +5. Add to `_LAZY_MODULES` in `__init__.py` if it requires torch/transformers +6. Add a `@st.cache_resource` loader in any Streamlit entry point that uses it + +To extend the ensemble: +1. Add sub-analyzer instance in `EnsembleAnalyzer.__init__` +2. Run it in `analyze()` alongside existing sub-analyzers +3. Adjust weights in `EnsembleConfig` (currently GPT-2: 0.75, NLTK: 0.25; RoBERTa/Binoculars 0.0) + +## Anti-patterns +- Never hardcode threshold values (like `0.30`, `150.0`) in analyzer logic — always use `self.thresholds.*` from `ThresholdConfig` +- Never call `TextProcessor()` to instantiate per analysis — `self.processor` is set in `BaseAnalyzer.__init__`; use `TextProcessor.clean_text()` / `TextProcessor.compute_metrics()` as classmethods +- Don't add Streamlit imports or `st.*` calls here — analyzers are UI-agnostic + +## Related Context +- Result model: `src/models/AGENTS.md` +- Settings & thresholds: `src/models/AGENTS.md` +- Shared text utilities: `src/utils/AGENTS.md` diff --git a/src/analyzers/__init__.py b/src/analyzers/__init__.py index b053225..5032da5 100644 --- a/src/analyzers/__init__.py +++ b/src/analyzers/__init__.py @@ -16,6 +16,7 @@ from src.analyzers.nltk_analyzer import NLTKAnalyzer if TYPE_CHECKING: # pragma: no cover - import-time typing only + from src.analyzers.binoculars_analyzer import BinocularsAnalyzer from src.analyzers.ensemble_analyzer import EnsembleAnalyzer from src.analyzers.gpt2_analyzer import GPT2Analyzer from src.analyzers.roberta_analyzer import RoBERTaAnalyzer @@ -25,12 +26,14 @@ "NLTKAnalyzer", "GPT2Analyzer", "RoBERTaAnalyzer", + "BinocularsAnalyzer", "EnsembleAnalyzer", ] _LAZY_MODULES = { "GPT2Analyzer": "src.analyzers.gpt2_analyzer", "RoBERTaAnalyzer": "src.analyzers.roberta_analyzer", + "BinocularsAnalyzer": "src.analyzers.binoculars_analyzer", "EnsembleAnalyzer": "src.analyzers.ensemble_analyzer", } diff --git a/src/analyzers/base_analyzer.py b/src/analyzers/base_analyzer.py index 4a92fa4..02f2944 100644 --- a/src/analyzers/base_analyzer.py +++ b/src/analyzers/base_analyzer.py @@ -9,15 +9,9 @@ import time from abc import ABC, abstractmethod -from typing import Optional - -from src.config.settings import ( - ConfidenceLevel, - ThresholdConfig, - Verdict, - get_settings, -) -from src.models.result import AnalysisResult, DetectionScore + +from src.config.settings import ConfidenceLevel, Verdict, get_settings +from src.models.result import AnalysisResult from src.utils.logging_config import get_logger from src.utils.text_processing import TextProcessor @@ -47,6 +41,7 @@ def analyze(self, text: str) -> AnalysisResult: # Validate input cleaned_text = TextProcessor.clean_text(text) + cleaned_text = self._apply_input_cap(cleaned_text, result) result.text_length = len(cleaned_text) if not cleaned_text: @@ -89,7 +84,8 @@ def analyze(self, text: str) -> AnalysisResult: result.verdict = Verdict.UNCERTAIN result.confidence = 0.0 result.confidence_level = ConfidenceLevel.VERY_LOW - result.add_warning(f"Analysis error: {str(e)}") + # Generic, non-leaking message; the full traceback is logged above. + result.add_warning("An error occurred during analysis (details logged server-side).") result.explanation = "An error occurred during analysis." result.analysis_time = round(time.time() - start_time, 3) @@ -103,6 +99,21 @@ def analyze(self, text: str) -> AnalysisResult: return result + def _apply_input_cap(self, cleaned_text: str, result: AnalysisResult) -> str: + """Truncate over-long input to the configured cap, adding a warning. + + Bounds worst-case compute (notably the GPT-2 sliding window) on very + long input. Returns the possibly-truncated text. + """ + max_chars = self.thresholds.max_input_chars + if len(cleaned_text) > max_chars: + result.add_warning( + f"Input was truncated to {max_chars} characters for analysis " + f"(received {len(cleaned_text)})." + ) + return cleaned_text[:max_chars] + return cleaned_text + @abstractmethod def _perform_analysis(self, text: str, result: AnalysisResult) -> AnalysisResult: """ @@ -181,12 +192,14 @@ def _determine_verdict(self, result: AnalysisResult) -> AnalysisResult: scores.append(("sentence_variance", 0.0, False)) # Weighted average - weights = {"perplexity": 0.40, "burstiness": 0.25, - "lexical_diversity": 0.15, "sentence_variance": 0.20} - - weighted_sum = sum( - score * weights.get(name, 0.25) for name, score, _ in scores - ) + weights = { + "perplexity": 0.40, + "burstiness": 0.25, + "lexical_diversity": 0.15, + "sentence_variance": 0.20, + } + + weighted_sum = sum(score * weights.get(name, 0.25) for name, score, _ in scores) total_weight = sum(weights.get(name, 0.25) for name, _, _ in scores) ai_probability = weighted_sum / total_weight if total_weight > 0 else 0.5 @@ -282,16 +295,12 @@ def _generate_explanation(self, result: AnalysisResult) -> str: # Sentence variance if result.sentence_variance < 0.15: - parts.append( - "Sentence lengths are very uniform, a common AI characteristic." - ) + parts.append("Sentence lengths are very uniform, a common AI characteristic.") elif result.sentence_variance > 0.50: - parts.append( - "Sentence lengths show high variation, typical of natural writing." - ) + parts.append("Sentence lengths show high variation, typical of natural writing.") # Warnings if result.warnings: parts.append("⚠️ Note: " + " ".join(result.warnings)) - return " ".join(parts) \ No newline at end of file + return " ".join(parts) diff --git a/src/analyzers/binoculars_analyzer.py b/src/analyzers/binoculars_analyzer.py new file mode 100644 index 0000000..5e6751f --- /dev/null +++ b/src/analyzers/binoculars_analyzer.py @@ -0,0 +1,233 @@ +""" +Binoculars Analyzer +=================== + +Zero-shot AI-text detection via **cross-perplexity** between two language models, +after Hans et al., 2024 ("Spotting LLMs With Binoculars"). + +Intuition +--------- +A single model's perplexity conflates "how machine-like is this text" with "how +surprising is this prompt/topic". Binoculars cancels the prompt effect by +dividing an *observer* model's log-perplexity by the *cross-perplexity* between +the observer and a second *performer* model: + + score = log_perplexity_observer(text) / cross_perplexity(observer, performer) + +Human text tends to score **higher**; machine-generated text scores **lower**. +Because it is a ratio of two models, it is far more robust than the single-model +GPT-2 perplexity signal — the modernization recommended by the project audit. + +This implementation deliberately uses a small, CPU-friendly observer/performer +pair (``gpt2`` / ``distilgpt2``) that share the GPT-2 tokenizer, so it stays +local and dependency-light. The decision boundary is calibrated on the bundled +benchmark rather than hard-coded to the paper's Falcon-specific threshold. +""" + +from __future__ import annotations + +import math +from typing import Optional, Tuple + +import torch +from transformers import GPT2LMHeadModel, GPT2TokenizerFast + +from src.analyzers.base_analyzer import BaseAnalyzer +from src.analyzers.calibration import logistic_ai_probability +from src.config.settings import ConfidenceLevel, Verdict +from src.models.result import AnalysisResult, DetectionScore +from src.utils.logging_config import get_logger +from src.utils.text_processing import TextProcessor + +logger = get_logger(__name__) + + +class BinocularsAnalyzer(BaseAnalyzer): + """Cross-perplexity (two-model) AI-text detector.""" + + def __init__(self): + super().__init__() + self.config = self.settings.binoculars + self.method_name = "Binoculars (cross-perplexity)" + self._tokenizer: Optional[GPT2TokenizerFast] = None + self._observer: Optional[GPT2LMHeadModel] = None + self._performer: Optional[GPT2LMHeadModel] = None + self._device: Optional[torch.device] = None + + # ------------------------------------------------------------------ # + # Lazy resources + # ------------------------------------------------------------------ # + @property + def device(self) -> torch.device: + if self._device is None: + self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + logger.info(f"Binoculars using device: {self._device}") + return self._device + + @property + def tokenizer(self) -> GPT2TokenizerFast: + if self._tokenizer is None: + logger.info("Loading Binoculars tokenizer...") + self._tokenizer = GPT2TokenizerFast.from_pretrained( + self.config.observer_model, revision=self.config.revision + ) + return self._tokenizer + + def _load_model(self, name: str) -> GPT2LMHeadModel: + logger.info(f"Loading Binoculars model: {name}...") + model = GPT2LMHeadModel.from_pretrained( + name, revision=self.config.revision, use_safetensors=True + ) + model.to(self.device) + model.eval() + return model + + @property + def observer(self) -> GPT2LMHeadModel: + if self._observer is None: + self._observer = self._load_model(self.config.observer_model) + return self._observer + + @property + def performer(self) -> GPT2LMHeadModel: + if self._performer is None: + self._performer = self._load_model(self.config.performer_model) + return self._performer + + # ------------------------------------------------------------------ # + # Core computation + # ------------------------------------------------------------------ # + def _compute_binoculars(self, text: str) -> Tuple[float, float]: + """Return ``(binoculars_score, observer_perplexity)``. + + ``binoculars_score`` is log-perplexity(observer) / cross-perplexity; + lower values indicate machine-generated text. + """ + enc = self.tokenizer( + text, + return_tensors="pt", + truncation=True, + max_length=self.config.max_token_length, + ) + input_ids = enc.input_ids.to(self.device) + if input_ids.size(1) <= 1: + return 1.0, 100.0 # neutral-ish for degenerate input + + with torch.no_grad(): + obs_logits = self.observer(input_ids).logits[:, :-1, :] + perf_logits = self.performer(input_ids).logits[:, :-1, :] + targets = input_ids[:, 1:] + + obs_logp = torch.log_softmax(obs_logits, dim=-1) + perf_logp = torch.log_softmax(perf_logits, dim=-1) + obs_p = obs_logp.exp() + + # Observer cross-entropy against the true next token (nats). + tgt_logp = obs_logp.gather(-1, targets.unsqueeze(-1)).squeeze(-1) + observer_ce = -tgt_logp.mean().item() + + # Cross-perplexity: mean cross-entropy between observer and performer + # next-token distributions (nats). + cross_ce = -(obs_p * perf_logp).sum(dim=-1).mean().item() + + if cross_ce <= 0: + return 1.0, math.exp(min(observer_ce, 9.0)) + + binoculars_score = observer_ce / cross_ce + observer_ppl = math.exp(min(observer_ce, 9.0)) # cap to avoid overflow + return binoculars_score, observer_ppl + + def ai_probability(self, text: str) -> float: + """Calibrated AI-probability in ``[0, 1]`` for ``text`` (lower score => AI).""" + score, _ = self._compute_binoculars(text) + return logistic_ai_probability( + score, + midpoint=self.config.score_midpoint, + slope=self.config.score_slope, + direction="lower_is_ai", + ) + + # ------------------------------------------------------------------ # + # Analyzer contract + # ------------------------------------------------------------------ # + def _perform_analysis(self, text: str, result: AnalysisResult) -> AnalysisResult: + result.method = self.method_name + + score, observer_ppl = self._compute_binoculars(text) + ai_prob = logistic_ai_probability( + score, + midpoint=self.config.score_midpoint, + slope=self.config.score_slope, + direction="lower_is_ai", + ) + result.perplexity = observer_ppl + + # Primary calibrated score (index 0, mirrors the ensemble contract). + result.add_score( + DetectionScore( + name="Binoculars AI Score", + value=ai_prob, + weight=1.0, + interpretation=self._interpret(score, ai_prob), + indicates_ai=ai_prob > 0.5, + ) + ) + result.add_score( + DetectionScore( + name="Binoculars Ratio", + value=score, + weight=0.0, + interpretation=f"log-ppl / cross-ppl = {score:.3f} (lower => AI)", + indicates_ai=score < self.config.score_midpoint, + ) + ) + + # Supporting statistical metrics for transparency. + burstiness, _ = TextProcessor.compute_burstiness(text) + result.burstiness = burstiness + result.lexical_diversity = result.metrics.lexical_diversity + result.sentence_variance = TextProcessor.compute_sentence_variance(text) + return result + + def _determine_verdict(self, result: AnalysisResult) -> AnalysisResult: + """Map the calibrated Binoculars AI-probability to a verdict.""" + ai_prob = result.scores[0].value if result.scores else 0.5 + confidence = abs(ai_prob - 0.5) * 200 # 0-100 + + if ai_prob >= 0.75: + result.verdict = Verdict.AI_GENERATED + result.confidence = min(95.0, 80 + confidence * 0.15) + elif ai_prob >= 0.55: + result.verdict = Verdict.LIKELY_AI + result.confidence = min(85.0, 65 + confidence * 0.2) + elif ai_prob >= 0.45: + result.verdict = Verdict.UNCERTAIN + result.confidence = 50.0 + elif ai_prob >= 0.25: + result.verdict = Verdict.LIKELY_HUMAN + result.confidence = min(85.0, 65 + (1 - ai_prob) * 50) + else: + result.verdict = Verdict.HUMAN_WRITTEN + result.confidence = min(95.0, 80 + (1 - ai_prob) * 50) + + result.confidence = round(result.confidence, 1) + if result.confidence >= 80: + result.confidence_level = ConfidenceLevel.HIGH + elif result.confidence >= 60: + result.confidence_level = ConfidenceLevel.MEDIUM + elif result.confidence >= 40: + result.confidence_level = ConfidenceLevel.LOW + else: + result.confidence_level = ConfidenceLevel.VERY_LOW + return result + + def _interpret(self, score: float, ai_prob: float) -> str: + if ai_prob > 0.75: + return f"Low cross-perplexity ratio ({score:.3f}) — strong AI indicator" + if ai_prob > 0.55: + return f"Below-boundary ratio ({score:.3f}) — likely AI-generated" + if ai_prob > 0.45: + return f"Borderline ratio ({score:.3f}) — uncertain" + if ai_prob > 0.25: + return f"Above-boundary ratio ({score:.3f}) — likely human-written" + return f"High cross-perplexity ratio ({score:.3f}) — strong human indicator" diff --git a/src/analyzers/calibration.py b/src/analyzers/calibration.py new file mode 100644 index 0000000..d0d96d5 --- /dev/null +++ b/src/analyzers/calibration.py @@ -0,0 +1,70 @@ +""" +Perplexity Calibration +====================== + +Turns a raw perplexity into a **calibrated AI-probability** in ``[0, 1]`` via a +logistic (sigmoid) transform. + +Why this exists +--------------- +The previous ensemble mapped *both* GPT-2 and NLTK perplexity to an AI-score +with the single formula ``max(0, min(1, 1 - perplexity / 500))``. That is wrong +for two independent reasons: + +1. **It conflates incommensurable scales.** GPT-2 perplexity on ordinary English + is ~40-100; the Brown-corpus n-gram model produces perplexities in the + hundreds-to-thousands. One linear map cannot fit both. +2. **It has no calibrated decision boundary.** Human GPT-2 perplexity ~58 maps + to ``1 - 58/500 = 0.88`` — i.e. human text scored 88% AI. This is the + systematic AI-bias flagged in the audit (finding C2). + +A logistic with an explicit midpoint (the perplexity at which AI-probability is +0.5) and slope fixes both: each analyzer gets its own midpoint/slope, and the +midpoint *is* the decision boundary. + +The ``direction`` argument encodes the empirical relationship for each analyzer: + +* GPT-2 — lower perplexity means *more* predictable, hence more AI-like + (``direction="lower_is_ai"``). +* Brown-corpus NLTK — formal/modern text is atypical of the 1961 corpus and so + scores *higher* perplexity while also being more likely AI in practice + (``direction="higher_is_ai"``). This signal is weaker and mainly reflects + stylistic typicality; it is weighted accordingly in the ensemble. +""" + +from __future__ import annotations + +import math + + +def logistic_ai_probability( + perplexity: float, + midpoint: float, + slope: float, + direction: str = "lower_is_ai", +) -> float: + """Map a perplexity to a calibrated AI-probability in ``[0, 1]``. + + Args: + perplexity: Raw perplexity value (>= 0). + midpoint: Perplexity at which the AI-probability is 0.5 (the decision + boundary). + slope: Logistic steepness (> 0). Larger => sharper transition. + direction: ``"lower_is_ai"`` (default) or ``"higher_is_ai"``. + + Returns: + AI-probability in ``[0, 1]``. + """ + if slope <= 0: + raise ValueError("slope must be positive") + if direction not in ("lower_is_ai", "higher_is_ai"): + raise ValueError("direction must be 'lower_is_ai' or 'higher_is_ai'") + + # sign chosen so that the AI side of the midpoint maps above 0.5. + sign = 1.0 if direction == "lower_is_ai" else -1.0 + z = sign * slope * (midpoint - perplexity) + # Numerically stable logistic. + if z >= 0: + return 1.0 / (1.0 + math.exp(-z)) + ez = math.exp(z) + return ez / (1.0 + ez) diff --git a/src/analyzers/ensemble_analyzer.py b/src/analyzers/ensemble_analyzer.py index d7ac1be..f1cc44e 100644 --- a/src/analyzers/ensemble_analyzer.py +++ b/src/analyzers/ensemble_analyzer.py @@ -8,13 +8,15 @@ from __future__ import annotations import time -from typing import List, Dict +from typing import Optional from src.analyzers.base_analyzer import BaseAnalyzer +from src.analyzers.binoculars_analyzer import BinocularsAnalyzer +from src.analyzers.calibration import logistic_ai_probability from src.analyzers.gpt2_analyzer import GPT2Analyzer from src.analyzers.nltk_analyzer import NLTKAnalyzer from src.analyzers.roberta_analyzer import RoBERTaAnalyzer -from src.config.settings import ConfidenceLevel, Verdict, get_settings +from src.config.settings import ConfidenceLevel, Verdict from src.models.result import AnalysisResult, DetectionScore from src.utils.logging_config import get_logger from src.utils.text_processing import TextProcessor @@ -25,30 +27,41 @@ class EnsembleAnalyzer(BaseAnalyzer): """Ensemble analyzer combining multiple detection methods. - Default weights are RoBERTa=0.0, GPT-2=0.65, and NLTK=0.35. - The fused score is computed as: - ensemble_ai_score = sum(weight_i * ai_score_i) - where GPT-2 and NLTK AI scores are normalized from perplexity with - max(0, min(1, 1 - (perplexity / 500))). + Each backend contributes a **calibrated** AI-probability in ``[0, 1]``; the + ensemble is their weighted average:: + + ensemble_ai_score = sum(weight_i * ai_probability_i) + + GPT-2 and NLTK perplexity are mapped to probabilities with a per-analyzer + logistic (see :mod:`src.analyzers.calibration`) whose midpoint is the + decision boundary, rather than the previous single ``1 - perplexity / 500`` + formula that scored ordinary human text as ~88% AI. Weights and calibration + parameters come from :class:`~src.config.settings.EnsembleConfig`. + + RoBERTa is disabled by default (weight 0) and is *not loaded or run* in that + state — its classification head is untrained, so it would only add cost and + noise until a fine-tuned checkpoint is wired in. """ def __init__(self): super().__init__() self.method_name = "Ensemble (GPT2+NLTK)" - + self.ensemble_config = self.settings.ensemble + # Initialize analyzers logger.info("Initializing ensemble components...") self._roberta_analyzer = None self._gpt2_analyzer = None self._nltk_analyzer = None - - # Weights for ensemble voting - # RoBERTa is disabled (0.0) by default as it requires fine-tuning - # To enable: fine-tune the model, then set weight to 0.45 and adjust others + self._binoculars_analyzer = None + + # Fusion weights, sourced from config so calibration lives in one place. + cfg = self.ensemble_config self.weights = { - "roberta": 0.0, # RoBERTa (DISABLED - requires fine-tuning) - "gpt2": 0.65, # GPT-2 perplexity (increased weight) - "nltk": 0.35, # NLTK statistical (increased weight) + "roberta": cfg.weight_roberta, + "gpt2": cfg.weight_gpt2, + "nltk": cfg.weight_nltk, + "binoculars": cfg.weight_binoculars, } @property @@ -75,6 +88,14 @@ def nltk_analyzer(self) -> NLTKAnalyzer: self._nltk_analyzer = NLTKAnalyzer(ngram_size=3) return self._nltk_analyzer + @property + def binoculars_analyzer(self) -> BinocularsAnalyzer: + """Lazy-load Binoculars analyzer (only used when its weight > 0).""" + if self._binoculars_analyzer is None: + logger.info("Loading Binoculars analyzer...") + self._binoculars_analyzer = BinocularsAnalyzer() + return self._binoculars_analyzer + def analyze(self, text: str) -> AnalysisResult: """ Analyze text using ensemble of all three methods. @@ -90,6 +111,7 @@ def analyze(self, text: str) -> AnalysisResult: # Validate input cleaned_text = TextProcessor.clean_text(text) + cleaned_text = self._apply_input_cap(cleaned_text, result) result.text_length = len(cleaned_text) if not cleaned_text: @@ -118,26 +140,40 @@ def analyze(self, text: str) -> AnalysisResult: # Compute text metrics once result.metrics = TextProcessor.compute_metrics(cleaned_text) - # Run all three analyzers in parallel (conceptually) + # Run the analyzers. RoBERTa is skipped entirely when its weight is + # 0 (the default): loading and running an untrained roberta-base + # contributes nothing to the blend but costs a large download, RAM + # and latency. A neutral placeholder keeps the result shape stable. logger.info("Running ensemble analysis...") - - logger.info("1/3: Running RoBERTa analysis...") - roberta_result = self.roberta_analyzer.analyze(cleaned_text) - - logger.info("2/3: Running GPT-2 analysis...") + + if self.weights.get("roberta", 0.0) > 0: + logger.info("Running RoBERTa analysis...") + roberta_result = self.roberta_analyzer.analyze(cleaned_text) + else: + logger.info("Skipping RoBERTa (weight=0, not loaded).") + roberta_result = self._disabled_roberta_result() + + logger.info("Running GPT-2 analysis...") gpt2_result = self.gpt2_analyzer.analyze(cleaned_text) - - logger.info("3/3: Running NLTK analysis...") + + logger.info("Running NLTK analysis...") nltk_result = self.nltk_analyzer.analyze(cleaned_text) + # Binoculars is optional and off by default; only load/run it when + # it carries weight. + binoculars_ai = None + if self.weights.get("binoculars", 0.0) > 0: + logger.info("Running Binoculars analysis...") + binoculars_ai = self.binoculars_analyzer.ai_probability(cleaned_text) + # Combine results result = self._combine_results( - result, roberta_result, gpt2_result, nltk_result + result, roberta_result, gpt2_result, nltk_result, binoculars_ai ) # Determine final verdict result = self._determine_verdict(result) - + # Generate explanation result.explanation = self._generate_ensemble_explanation( result, roberta_result, gpt2_result, nltk_result @@ -148,7 +184,8 @@ def analyze(self, text: str) -> AnalysisResult: result.verdict = Verdict.UNCERTAIN result.confidence = 0.0 result.confidence_level = ConfidenceLevel.VERY_LOW - result.add_warning(f"Analysis error: {str(e)}") + # Generic, non-leaking message; the full traceback is logged above. + result.add_warning("An error occurred during analysis (details logged server-side).") result.explanation = "An error occurred during ensemble analysis." result.analysis_time = round(time.time() - start_time, 3) @@ -169,6 +206,7 @@ def _combine_results( roberta_result: AnalysisResult, gpt2_result: AnalysisResult, nltk_result: AnalysisResult, + binoculars_ai: Optional[float] = None, ) -> AnalysisResult: """ Combine results from all three analyzers. @@ -182,83 +220,124 @@ def _combine_results( Returns: Combined result. """ - # Extract key metrics with fallback defaults - # RoBERTa AI score (from its scores) + cfg = self.ensemble_config + + # RoBERTa AI score (from its scores); 0.5 (neutral) when disabled. roberta_ai_score = 0.5 for score in roberta_result.scores: if "RoBERTa" in score.name: roberta_ai_score = score.value break - # GPT-2 perplexity (normalized to 0-1, inverted so high = AI) + # Calibrated per-analyzer AI-probabilities. Each analyzer uses its own + # logistic (midpoint = decision boundary, correct direction) instead of + # one linear map that scored human text as ~88% AI. gpt2_perplexity = gpt2_result.perplexity - # Normalize: low perplexity = high AI probability - gpt2_ai_score = max(0, min(1, 1 - (gpt2_perplexity / 500))) + gpt2_ai_score = logistic_ai_probability( + gpt2_perplexity, + midpoint=cfg.gpt2_ppl_midpoint, + slope=cfg.gpt2_ppl_slope, + direction="lower_is_ai", + ) - # NLTK perplexity (normalized similarly) nltk_perplexity = nltk_result.perplexity - nltk_ai_score = max(0, min(1, 1 - (nltk_perplexity / 500))) + nltk_ai_score = logistic_ai_probability( + nltk_perplexity, + midpoint=cfg.nltk_ppl_midpoint, + slope=cfg.nltk_ppl_slope, + direction="higher_is_ai", + ) - # Weighted ensemble vote + # Weighted ensemble vote (roberta weight is 0 by default => no effect). ensemble_ai_score = ( - self.weights["roberta"] * roberta_ai_score + - self.weights["gpt2"] * gpt2_ai_score + - self.weights["nltk"] * nltk_ai_score + self.weights["roberta"] * roberta_ai_score + + self.weights["gpt2"] * gpt2_ai_score + + self.weights["nltk"] * nltk_ai_score ) + # Optional Binoculars term (only present when its weight > 0). + if binoculars_ai is not None: + ensemble_ai_score += self.weights.get("binoculars", 0.0) * binoculars_ai + # Guard against misconfigured weights (e.g. enabling Binoculars without + # rebalancing) pushing the "probability" outside [0, 1]. + ensemble_ai_score = max(0.0, min(1.0, ensemble_ai_score)) logger.info( f"Ensemble scores - RoBERTa: {roberta_ai_score:.3f}, " f"GPT-2: {gpt2_ai_score:.3f}, NLTK: {nltk_ai_score:.3f}, " + f"Binoculars: {binoculars_ai if binoculars_ai is not None else 'off'}, " f"Combined: {ensemble_ai_score:.3f}" ) - # Add ensemble score - result.add_score(DetectionScore( - name="Ensemble AI Score", - value=ensemble_ai_score, - weight=1.0, - interpretation=self._interpret_ensemble_score(ensemble_ai_score), - indicates_ai=ensemble_ai_score > 0.5, - )) - - # Add individual analyzer scores for transparency - result.add_score(DetectionScore( - name="RoBERTa Score", - value=roberta_ai_score, - weight=self.weights["roberta"], - interpretation=f"RoBERTa: {roberta_result.verdict.value}", - indicates_ai=roberta_ai_score > 0.5, - )) - - result.add_score(DetectionScore( - name="GPT-2 Perplexity Score", - value=gpt2_ai_score, - weight=self.weights["gpt2"], - interpretation=f"GPT-2: {gpt2_result.verdict.value} (PPL: {gpt2_perplexity:.1f})", - indicates_ai=gpt2_ai_score > 0.5, - )) - - result.add_score(DetectionScore( - name="NLTK Statistical Score", - value=nltk_ai_score, - weight=self.weights["nltk"], - interpretation=f"NLTK: {nltk_result.verdict.value} (PPL: {nltk_perplexity:.1f})", - indicates_ai=nltk_ai_score > 0.5, - )) - - # Combine other metrics (averages) - result.perplexity = (gpt2_perplexity + nltk_perplexity) / 2 - result.burstiness = ( - roberta_result.burstiness + - gpt2_result.burstiness + - nltk_result.burstiness - ) / 3 + # Add ensemble score (always index 0 by contract). + result.add_score( + DetectionScore( + name="Ensemble AI Score", + value=ensemble_ai_score, + weight=1.0, + interpretation=self._interpret_ensemble_score(ensemble_ai_score), + indicates_ai=ensemble_ai_score > 0.5, + ) + ) + + # Individual analyzer scores for transparency. The RoBERTa row carries + # weight 0 when disabled so downstream agreement logic can exclude it. + roberta_note = ( + f"RoBERTa: {roberta_result.verdict.value}" + if self.weights["roberta"] > 0 + else "RoBERTa: disabled (weight 0, not run)" + ) + result.add_score( + DetectionScore( + name="RoBERTa Score", + value=roberta_ai_score, + weight=self.weights["roberta"], + interpretation=roberta_note, + indicates_ai=roberta_ai_score > 0.5, + ) + ) + + result.add_score( + DetectionScore( + name="GPT-2 Perplexity Score", + value=gpt2_ai_score, + weight=self.weights["gpt2"], + interpretation=f"GPT-2: {gpt2_result.verdict.value} (PPL: {gpt2_perplexity:.1f})", + indicates_ai=gpt2_ai_score > 0.5, + ) + ) + + result.add_score( + DetectionScore( + name="NLTK Statistical Score", + value=nltk_ai_score, + weight=self.weights["nltk"], + interpretation=f"NLTK: {nltk_result.verdict.value} (PPL: {nltk_perplexity:.1f})", + indicates_ai=nltk_ai_score > 0.5, + ) + ) + + # Optional Binoculars row (only when enabled), for transparency. + if binoculars_ai is not None: + result.add_score( + DetectionScore( + name="Binoculars Score", + value=binoculars_ai, + weight=self.weights.get("binoculars", 0.0), + interpretation=f"Binoculars cross-perplexity: {binoculars_ai:.2f} AI-prob", + indicates_ai=binoculars_ai > 0.5, + ) + ) + + # Aggregate reported metrics from the analyzers that actually run. + # Report GPT-2 perplexity only: GPT-2 (~10-100) and Brown-corpus NLTK + # (~1000-3000) perplexities are on incommensurable scales, so averaging + # them would produce a misleading "Average Perplexity" in the UI. + result.perplexity = gpt2_perplexity + result.burstiness = (gpt2_result.burstiness + nltk_result.burstiness) / 2 result.lexical_diversity = result.metrics.lexical_diversity result.sentence_variance = ( - roberta_result.sentence_variance + - gpt2_result.sentence_variance + - nltk_result.sentence_variance - ) / 3 + gpt2_result.sentence_variance + nltk_result.sentence_variance + ) / 2 return result @@ -272,30 +351,34 @@ def _determine_verdict(self, result: AnalysisResult) -> AnalysisResult: Returns: Result with verdict and confidence. """ - # Get ensemble AI score + cfg = self.ensemble_config + + # Get ensemble AI score (index 0 by contract). ensemble_score = result.scores[0].value if result.scores else 0.5 - # Agreement level (check if analyzers agree) - ai_votes = sum(1 for s in result.scores[1:] if s.indicates_ai) - total_votes = len(result.scores) - 1 # Exclude ensemble score itself + # Agreement only over analyzers that actually contribute (weight > 0): + # a disabled/zero-weight RoBERTa must not sway confidence or narrative. + voters = [s for s in result.scores[1:] if s.weight > 0] + ai_votes = sum(1 for s in voters if s.indicates_ai) + total_votes = len(voters) agreement = ai_votes / total_votes if total_votes > 0 else 0.5 - # Base confidence on ensemble score and agreement + # Base confidence on distance from the 0.5 boundary and voter agreement. base_confidence = abs(ensemble_score - 0.5) * 200 # 0-100 scale agreement_boost = agreement if ensemble_score > 0.5 else (1 - agreement) confidence = base_confidence * (0.7 + 0.3 * agreement_boost) - # Determine verdict - if ensemble_score > 0.75: + # Determine verdict against calibrated, configurable thresholds. + if ensemble_score >= cfg.ai_threshold: result.verdict = Verdict.AI_GENERATED result.confidence = min(95, 80 + confidence * 0.15) - elif ensemble_score > 0.60: + elif ensemble_score >= cfg.likely_ai_threshold: result.verdict = Verdict.LIKELY_AI result.confidence = min(85, 65 + confidence * 0.2) - elif ensemble_score > 0.40: + elif ensemble_score >= cfg.likely_human_threshold: result.verdict = Verdict.UNCERTAIN result.confidence = 50.0 - elif ensemble_score > 0.25: + elif ensemble_score >= cfg.human_threshold: result.verdict = Verdict.LIKELY_HUMAN result.confidence = min(85, 65 + (1 - ensemble_score) * 50) else: @@ -365,26 +448,46 @@ def _generate_ensemble_explanation( # Key metrics parts.append( f"\n📈 **Key Metrics**:\n" - f"• Average Perplexity: {result.perplexity:.1f}\n" + f"• GPT-2 Perplexity: {result.perplexity:.1f}\n" f"• Burstiness: {result.burstiness:.3f}\n" f"• Lexical Diversity: {result.lexical_diversity:.1%}" ) - # Agreement analysis - ai_votes = sum(1 for s in result.scores[1:] if s.indicates_ai) - total_votes = len(result.scores) - 1 - if ai_votes == total_votes: - parts.append("\n✅ **All analyzers agree** on AI detection.") + # Agreement analysis over contributing analyzers only (weight > 0). + voters = [s for s in result.scores[1:] if s.weight > 0] + ai_votes = sum(1 for s in voters if s.indicates_ai) + total_votes = len(voters) + if total_votes and ai_votes == total_votes: + parts.append("\n✅ **All contributing analyzers agree** on AI detection.") elif ai_votes == 0: - parts.append("\n✅ **All analyzers agree** on human authorship.") + parts.append("\n✅ **All contributing analyzers agree** on human authorship.") else: parts.append( - f"\n⚖️ **Mixed signals**: {ai_votes}/{total_votes} analyzers " - f"indicate AI-generated content." + f"\n⚖️ **Mixed signals**: {ai_votes}/{total_votes} contributing " + f"analyzers indicate AI-generated content." ) return " ".join(parts) + def _disabled_roberta_result(self) -> AnalysisResult: + """Neutral placeholder used when RoBERTa is disabled (weight 0). + + Returns a result carrying a neutral (0.5) RoBERTa score without loading + or running the model, so the ensemble result shape stays stable while + avoiding a needless download, memory and latency. + """ + placeholder = AnalysisResult(verdict=Verdict.UNCERTAIN, confidence=0.0) + placeholder.add_score( + DetectionScore( + name="RoBERTa AI Probability", + value=0.5, + weight=0.0, + interpretation="RoBERTa disabled (weight 0, not run)", + indicates_ai=False, + ) + ) + return placeholder + def _perform_analysis(self, text: str, result: AnalysisResult) -> AnalysisResult: """Not used in ensemble - override analyze() instead.""" return result diff --git a/src/analyzers/gpt2_analyzer.py b/src/analyzers/gpt2_analyzer.py index f6c14d3..ed14cc4 100644 --- a/src/analyzers/gpt2_analyzer.py +++ b/src/analyzers/gpt2_analyzer.py @@ -10,12 +10,10 @@ import math from typing import Optional -import numpy as np import torch from transformers import GPT2LMHeadModel, GPT2TokenizerFast from src.analyzers.base_analyzer import BaseAnalyzer -from src.config.settings import get_settings from src.models.result import AnalysisResult, DetectionScore from src.utils.logging_config import get_logger from src.utils.text_processing import TextProcessor @@ -54,6 +52,7 @@ def tokenizer(self) -> GPT2TokenizerFast: self._tokenizer = GPT2TokenizerFast.from_pretrained( self.settings.gpt2.model_name, cache_dir=self.settings.gpt2.cache_dir, + revision=self.settings.gpt2.revision, ) return self._tokenizer @@ -62,9 +61,13 @@ def model(self) -> GPT2LMHeadModel: """Lazy-load the GPT-2 model.""" if self._model is None: logger.info("Loading GPT-2 model...") + # use_safetensors avoids loading pickled weights (a known RCE class + # in torch.load); revision pins a reproducible Hub commit. self._model = GPT2LMHeadModel.from_pretrained( self.settings.gpt2.model_name, cache_dir=self.settings.gpt2.cache_dir, + revision=self.settings.gpt2.revision, + use_safetensors=True, ) self._model.to(self.device) self._model.eval() @@ -220,62 +223,72 @@ def _perform_analysis(self, text: str, result: AnalysisResult) -> AnalysisResult logger.info("Computing GPT-2 perplexity...") result.perplexity = self._compute_perplexity_gpt2(text) - result.add_score(DetectionScore( - name="GPT-2 Perplexity", - value=result.perplexity, - weight=0.40, - interpretation=self._interpret_gpt2_perplexity(result.perplexity), - indicates_ai=result.perplexity < self.thresholds.perplexity_medium, - )) + result.add_score( + DetectionScore( + name="GPT-2 Perplexity", + value=result.perplexity, + weight=0.40, + interpretation=self._interpret_gpt2_perplexity(result.perplexity), + indicates_ai=result.perplexity < self.thresholds.perplexity_medium, + ) + ) # 2. Compute burstiness logger.info("Computing burstiness...") burstiness, _ = TextProcessor.compute_burstiness(text) result.burstiness = burstiness - result.add_score(DetectionScore( - name="Burstiness", - value=result.burstiness, - weight=0.20, - interpretation=self._interpret_burstiness(result.burstiness), - indicates_ai=result.burstiness < self.thresholds.burstiness_medium, - )) + result.add_score( + DetectionScore( + name="Burstiness", + value=result.burstiness, + weight=0.20, + interpretation=self._interpret_burstiness(result.burstiness), + indicates_ai=result.burstiness < self.thresholds.burstiness_medium, + ) + ) # 3. Compute lexical diversity logger.info("Computing lexical diversity...") result.lexical_diversity = result.metrics.lexical_diversity - result.add_score(DetectionScore( - name="Lexical Diversity", - value=result.lexical_diversity, - weight=0.15, - interpretation=self._interpret_lexical_diversity(result.lexical_diversity), - indicates_ai=result.lexical_diversity < self.thresholds.lexical_diversity_medium, - )) + result.add_score( + DetectionScore( + name="Lexical Diversity", + value=result.lexical_diversity, + weight=0.15, + interpretation=self._interpret_lexical_diversity(result.lexical_diversity), + indicates_ai=result.lexical_diversity < self.thresholds.lexical_diversity_medium, + ) + ) # 4. Compute sentence variance logger.info("Computing sentence variance...") result.sentence_variance = TextProcessor.compute_sentence_variance(text) - result.add_score(DetectionScore( - name="Sentence Variance", - value=result.sentence_variance, - weight=0.15, - interpretation=self._interpret_sentence_variance(result.sentence_variance), - indicates_ai=result.sentence_variance < 0.25, - )) + result.add_score( + DetectionScore( + name="Sentence Variance", + value=result.sentence_variance, + weight=0.15, + interpretation=self._interpret_sentence_variance(result.sentence_variance), + indicates_ai=result.sentence_variance < 0.25, + ) + ) # 5. Compute token-level entropy (GPT-2 specific) logger.info("Computing token entropy...") try: entropy = self._compute_token_entropy(text) - result.add_score(DetectionScore( - name="Token Entropy", - value=entropy, - weight=0.10, - interpretation=self._interpret_entropy(entropy), - indicates_ai=entropy < 6.0, - )) + result.add_score( + DetectionScore( + name="Token Entropy", + value=entropy, + weight=0.10, + interpretation=self._interpret_entropy(entropy), + indicates_ai=entropy < 6.0, + ) + ) except Exception as e: logger.warning(f"Token entropy computation failed: {e}") @@ -338,4 +351,4 @@ def _interpret_entropy(self, value: float) -> str: elif value < 8.0: return "Moderate entropy — natural range" else: - return "High entropy — unpredictable patterns" \ No newline at end of file + return "High entropy — unpredictable patterns" diff --git a/src/analyzers/nltk_analyzer.py b/src/analyzers/nltk_analyzer.py index c0df08c..fc7424d 100644 --- a/src/analyzers/nltk_analyzer.py +++ b/src/analyzers/nltk_analyzer.py @@ -8,17 +8,20 @@ from __future__ import annotations import math -from collections import Counter -from typing import List, Optional, Tuple +from typing import Optional -import nltk from nltk.corpus import brown -from nltk.lm import MLE +from nltk.lm import ( + KneserNeyInterpolated, + Lidstone, + Vocabulary, + WittenBellInterpolated, +) +from nltk.lm.api import LanguageModel from nltk.lm.preprocessing import padded_everygram_pipeline from nltk.util import ngrams from src.analyzers.base_analyzer import BaseAnalyzer -from src.config.settings import get_settings from src.models.result import AnalysisResult, DetectionScore from src.utils.logging_config import get_logger from src.utils.text_processing import TextProcessor @@ -29,32 +32,90 @@ class NLTKAnalyzer(BaseAnalyzer): """Analyzer using NLTK n-gram language models.""" + # Process-wide cache of trained models keyed by (smoothing config, n). + # Building a model from the Brown corpus takes ~20s, so sharing it across + # analyzer instances turns N rebuilds into one (critical for tests and for + # Streamlit reruns that construct fresh analyzers). + _MODEL_CACHE: dict = {} + def __init__(self, ngram_size: int = 3): super().__init__() self.ngram_size = ngram_size - self._model: Optional[MLE] = None + self._model: Optional[LanguageModel] = None self._model_ngram_size: Optional[int] = None self.method_name = "NLTK N-gram" + def _model_cache_key(self, n: int) -> tuple: + """Identity of a trained model: smoothing settings + n-gram order.""" + cfg = self.settings.nltk + return ( + cfg.smoothing_method, + cfg.smoothing_discount, + cfg.smoothing_gamma, + cfg.unk_cutoff, + n, + ) + @property - def model(self) -> MLE: - """Lazy-load the language model.""" + def model(self) -> LanguageModel: + """Lazy-load the language model, reusing any process-cached instance.""" if self._model is None or self._model_ngram_size != self.ngram_size: - self._model = self._build_model(self.ngram_size) + key = self._model_cache_key(self.ngram_size) + cached = NLTKAnalyzer._MODEL_CACHE.get(key) + if cached is None: + cached = self._build_model(self.ngram_size) + NLTKAnalyzer._MODEL_CACHE[key] = cached + self._model = cached self._model_ngram_size = self.ngram_size return self._model - def _build_model(self, n: int) -> MLE: + def _make_smoothed_model(self, n: int) -> LanguageModel: + """Construct an (untrained) smoothed language model from settings. + + The smoothing method is selected via ``NLTKConfig.smoothing_method`` so + the configured smoothing parameters are actually honoured: + + * ``wittenbell`` — Witten-Bell interpolation (default; fast + accurate). + * ``kneserney`` — Kneser-Ney interpolation using ``smoothing_discount``. + * ``lidstone`` — additive add-k smoothing using ``smoothing_gamma``. """ - Build an n-gram language model from the Brown corpus. + cfg = self.settings.nltk + vocab = Vocabulary(unk_cutoff=cfg.unk_cutoff) + method = cfg.smoothing_method.lower() + + if method == "kneserney": + return KneserNeyInterpolated(order=n, discount=cfg.smoothing_discount, vocabulary=vocab) + if method == "lidstone": + return Lidstone(cfg.smoothing_gamma, order=n, vocabulary=vocab) + if method != "wittenbell": + logger.warning( + "Unknown smoothing_method %r; falling back to 'wittenbell'.", + cfg.smoothing_method, + ) + return WittenBellInterpolated(order=n, vocabulary=vocab) + + def _build_model(self, n: int) -> LanguageModel: + """ + Build a smoothed n-gram language model from the Brown corpus. + + Uses a **smoothed, interpolated** language model rather than a bare + maximum-likelihood estimate (``MLE``). An unsmoothed ``MLE`` assigns + probability 0 to every n-gram unseen in the training corpus, so almost any + real input collapsed to the perplexity ceiling and the "statistical" + signal carried no discriminating information. Interpolated smoothing + redistributes probability mass to unseen n-grams via lower-order back-off, + yielding perplexities that actually separate predictable text from varied + text. The concrete smoothing method and its parameters come from + ``NLTKConfig`` (see :meth:`_make_smoothed_model`). Args: n: N-gram size. Returns: - Trained MLE model. + Trained smoothed language model. """ - logger.info(f"Building {n}-gram language model from Brown corpus...") + cfg = self.settings.nltk + logger.info(f"Building {n}-gram {cfg.smoothing_method} model from Brown corpus...") TextProcessor.ensure_nltk_data() @@ -65,21 +126,18 @@ def _build_model(self, n: int) -> MLE: raise RuntimeError("Could not load Brown corpus. Run: nltk.download('brown')") from e # Preprocess corpus sentences - processed_sents = [ - [word.lower() for word in sent] - for sent in corpus_sents - ] + processed_sents = [[word.lower() for word in sent] for sent in corpus_sents] # Build padded n-gram pipeline train_data, padded_vocab = padded_everygram_pipeline(n, processed_sents) - # Train model - model = MLE(n) + # Train a smoothed model. ``unk_cutoff`` folds rare tokens into the + # ```` class so out-of-vocabulary input tokens receive a real, + # non-zero probability instead of underflowing to the epsilon floor. + model = self._make_smoothed_model(n) model.fit(train_data, padded_vocab) - logger.info( - f"Model built successfully. Vocabulary size: {len(model.vocab)}" - ) + logger.info(f"Model built successfully. Vocabulary size: {len(model.vocab)}") return model @@ -104,9 +162,7 @@ def _compute_perplexity(self, text: str) -> float: Returns: Perplexity score. """ - words = TextProcessor.tokenize_words( - text, remove_punctuation=True, lowercase=True - ) + words = TextProcessor.tokenize_words(text, remove_punctuation=True, lowercase=True) if len(words) < self.ngram_size: return 100.0 # Default for very short texts @@ -115,14 +171,16 @@ def _compute_perplexity(self, text: str) -> float: n = self.ngram_size # Generate n-grams from input text - text_ngrams = list(ngrams( - words, - n, - pad_left=True, - pad_right=True, - left_pad_symbol="", - right_pad_symbol="", - )) + text_ngrams = list( + ngrams( + words, + n, + pad_left=True, + pad_right=True, + left_pad_symbol="", + right_pad_symbol="", + ) + ) if not text_ngrams: return 100.0 @@ -180,50 +238,58 @@ def _perform_analysis(self, text: str, result: AnalysisResult) -> AnalysisResult logger.info("Computing perplexity...") result.perplexity = self._compute_perplexity(text) - result.add_score(DetectionScore( - name="Perplexity", - value=result.perplexity, - weight=0.40, - interpretation=self._interpret_perplexity(result.perplexity), - indicates_ai=result.perplexity < self.thresholds.perplexity_medium, - )) + result.add_score( + DetectionScore( + name="Perplexity", + value=result.perplexity, + weight=0.40, + interpretation=self._interpret_perplexity(result.perplexity), + indicates_ai=result.perplexity < self.thresholds.perplexity_medium, + ) + ) # 2. Compute burstiness logger.info("Computing burstiness...") burstiness, word_burstiness = TextProcessor.compute_burstiness(text) result.burstiness = burstiness - result.add_score(DetectionScore( - name="Burstiness", - value=result.burstiness, - weight=0.25, - interpretation=self._interpret_burstiness(result.burstiness), - indicates_ai=result.burstiness < self.thresholds.burstiness_medium, - )) + result.add_score( + DetectionScore( + name="Burstiness", + value=result.burstiness, + weight=0.25, + interpretation=self._interpret_burstiness(result.burstiness), + indicates_ai=result.burstiness < self.thresholds.burstiness_medium, + ) + ) # 3. Compute lexical diversity logger.info("Computing lexical diversity...") result.lexical_diversity = result.metrics.lexical_diversity - result.add_score(DetectionScore( - name="Lexical Diversity", - value=result.lexical_diversity, - weight=0.15, - interpretation=self._interpret_lexical_diversity(result.lexical_diversity), - indicates_ai=result.lexical_diversity < self.thresholds.lexical_diversity_medium, - )) + result.add_score( + DetectionScore( + name="Lexical Diversity", + value=result.lexical_diversity, + weight=0.15, + interpretation=self._interpret_lexical_diversity(result.lexical_diversity), + indicates_ai=result.lexical_diversity < self.thresholds.lexical_diversity_medium, + ) + ) # 4. Compute sentence variance logger.info("Computing sentence variance...") result.sentence_variance = TextProcessor.compute_sentence_variance(text) - result.add_score(DetectionScore( - name="Sentence Variance", - value=result.sentence_variance, - weight=0.20, - interpretation=self._interpret_sentence_variance(result.sentence_variance), - indicates_ai=result.sentence_variance < 0.25, - )) + result.add_score( + DetectionScore( + name="Sentence Variance", + value=result.sentence_variance, + weight=0.20, + interpretation=self._interpret_sentence_variance(result.sentence_variance), + indicates_ai=result.sentence_variance < 0.25, + ) + ) return result @@ -276,4 +342,4 @@ def _interpret_sentence_variance(self, value: float) -> str: elif value < 0.50: return "Moderate variation — natural range" else: - return "High variation — human writing pattern" \ No newline at end of file + return "High variation — human writing pattern" diff --git a/src/analyzers/roberta_analyzer.py b/src/analyzers/roberta_analyzer.py index 2253fc6..4a4d6ae 100644 --- a/src/analyzers/roberta_analyzer.py +++ b/src/analyzers/roberta_analyzer.py @@ -8,10 +8,9 @@ from __future__ import annotations import torch -from transformers import RobertaTokenizer, RobertaForSequenceClassification +from transformers import RobertaForSequenceClassification, RobertaTokenizer from src.analyzers.base_analyzer import BaseAnalyzer -from src.config.settings import get_settings from src.models.result import AnalysisResult, DetectionScore from src.utils.logging_config import get_logger from src.utils.text_processing import TextProcessor @@ -45,10 +44,11 @@ def tokenizer(self) -> RobertaTokenizer: """Lazy-load the RoBERTa tokenizer.""" if self._tokenizer is None: logger.info("Loading RoBERTa tokenizer...") - # Using a pre-trained AI detection model - # Note: You can replace this with a fine-tuned model for AI detection + # Note: replace with a fine-tuned AI-detection model to enable this + # analyzer in the ensemble (it is disabled by default). self._tokenizer = RobertaTokenizer.from_pretrained( - "roberta-base" + self.settings.roberta.model_name, + revision=self.settings.roberta.revision, ) return self._tokenizer @@ -63,12 +63,13 @@ def model(self) -> RobertaForSequenceClassification: "For production use, fine-tune this model on an AI detection dataset or " "use a pre-trained AI detector model." ) - # Using base model - REQUIRES FINE-TUNING for accurate AI detection - # To use a fine-tuned model, replace 'roberta-base' with your model path - # Example: 'your-username/roberta-ai-detector' + # Using base model - REQUIRES FINE-TUNING for accurate AI detection. + # use_safetensors avoids pickled-weight loading (a known RCE class). self._model = RobertaForSequenceClassification.from_pretrained( - "roberta-base", - num_labels=2 # Binary classification: AI vs Human + self.settings.roberta.model_name, + num_labels=2, # Binary classification: AI vs Human + revision=self.settings.roberta.revision, + use_safetensors=True, ) self._model.to(self.device) self._model.eval() @@ -87,13 +88,9 @@ def _compute_roberta_score(self, text: str) -> tuple[float, float]: """ # Tokenize with truncation and padding inputs = self.tokenizer( - text, - return_tensors="pt", - truncation=True, - max_length=512, - padding=True + text, return_tensors="pt", truncation=True, max_length=512, padding=True ) - + # Move to device inputs = {k: v.to(self.device) for k, v in inputs.items()} @@ -101,7 +98,7 @@ def _compute_roberta_score(self, text: str) -> tuple[float, float]: with torch.no_grad(): outputs = self.model(**inputs) logits = outputs.logits - + # Convert to probabilities probs = torch.softmax(logits, dim=1) ai_prob = probs[0][1].item() # Probability of AI class @@ -126,50 +123,58 @@ def _perform_analysis(self, text: str, result: AnalysisResult) -> AnalysisResult logger.info("Computing RoBERTa classification...") ai_prob, confidence = self._compute_roberta_score(text) - result.add_score(DetectionScore( - name="RoBERTa AI Score", - value=ai_prob, - weight=0.50, - interpretation=self._interpret_roberta_score(ai_prob), - indicates_ai=ai_prob > 0.5, - )) + result.add_score( + DetectionScore( + name="RoBERTa AI Score", + value=ai_prob, + weight=0.50, + interpretation=self._interpret_roberta_score(ai_prob), + indicates_ai=ai_prob > 0.5, + ) + ) # 2. Compute burstiness logger.info("Computing burstiness...") burstiness, _ = TextProcessor.compute_burstiness(text) result.burstiness = burstiness - result.add_score(DetectionScore( - name="Burstiness", - value=result.burstiness, - weight=0.20, - interpretation=self._interpret_burstiness(result.burstiness), - indicates_ai=result.burstiness < self.thresholds.burstiness_medium, - )) + result.add_score( + DetectionScore( + name="Burstiness", + value=result.burstiness, + weight=0.20, + interpretation=self._interpret_burstiness(result.burstiness), + indicates_ai=result.burstiness < self.thresholds.burstiness_medium, + ) + ) # 3. Compute lexical diversity logger.info("Computing lexical diversity...") result.lexical_diversity = result.metrics.lexical_diversity - result.add_score(DetectionScore( - name="Lexical Diversity", - value=result.lexical_diversity, - weight=0.15, - interpretation=self._interpret_lexical_diversity(result.lexical_diversity), - indicates_ai=result.lexical_diversity < self.thresholds.lexical_diversity_medium, - )) + result.add_score( + DetectionScore( + name="Lexical Diversity", + value=result.lexical_diversity, + weight=0.15, + interpretation=self._interpret_lexical_diversity(result.lexical_diversity), + indicates_ai=result.lexical_diversity < self.thresholds.lexical_diversity_medium, + ) + ) # 4. Compute sentence variance logger.info("Computing sentence variance...") result.sentence_variance = TextProcessor.compute_sentence_variance(text) - result.add_score(DetectionScore( - name="Sentence Variance", - value=result.sentence_variance, - weight=0.15, - interpretation=self._interpret_sentence_variance(result.sentence_variance), - indicates_ai=result.sentence_variance < 0.25, - )) + result.add_score( + DetectionScore( + name="Sentence Variance", + value=result.sentence_variance, + weight=0.15, + interpretation=self._interpret_sentence_variance(result.sentence_variance), + indicates_ai=result.sentence_variance < 0.25, + ) + ) # Store perplexity placeholder (not computed by RoBERTa) result.perplexity = 0.0 diff --git a/src/config/__init__.py b/src/config/__init__.py index 3eabeeb..9f61181 100644 --- a/src/config/__init__.py +++ b/src/config/__init__.py @@ -2,4 +2,4 @@ from src.config.settings import Settings, get_settings -__all__ = ["Settings", "get_settings"] \ No newline at end of file +__all__ = ["Settings", "get_settings"] diff --git a/src/config/settings.py b/src/config/settings.py index 2482156..dba7013 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -2,8 +2,9 @@ Application Settings ==================== -Centralized configuration management using Pydantic. -All settings can be overridden via environment variables. +Centralized configuration using frozen ``dataclasses`` with an ``lru_cache`` +singleton (:func:`get_settings`). A few runtime values can be overridden via +environment variables (see :meth:`Settings.__post_init__`). """ from __future__ import annotations @@ -17,12 +18,14 @@ class DetectionMethod(str, Enum): """Available detection methods.""" + NLTK = "nltk" GPT2 = "gpt2" class ConfidenceLevel(str, Enum): """Confidence level categories.""" + HIGH = "High" MEDIUM = "Medium" LOW = "Low" @@ -31,6 +34,7 @@ class ConfidenceLevel(str, Enum): class Verdict(str, Enum): """Detection verdict categories.""" + AI_GENERATED = "AI-Generated" LIKELY_AI = "Likely AI-Generated" UNCERTAIN = "Uncertain" @@ -64,6 +68,11 @@ class ThresholdConfig: recommended_text_length: int = 200 optimal_text_length: int = 500 + # Hard cap on analyzed input length. Very long input inflates GPT-2 + # sliding-window compute (a DoS vector if exposed beyond localhost), so + # input above this is truncated with a warning rather than processed whole. + max_input_chars: int = 50000 + @dataclass(frozen=True) class NLTKConfig: @@ -72,7 +81,14 @@ class NLTKConfig: corpus_name: str = "brown" default_ngram_size: int = 3 max_ngram_size: int = 5 - smoothing_discount: float = 0.75 + # Smoothing for the n-gram language model. ``wittenbell`` (interpolated + # back-off) is the default: it discriminates well and scores in + # milliseconds. ``kneserney`` gives slightly better modelling but is far + # slower in NLTK; ``lidstone`` is additive add-k smoothing. + smoothing_method: str = "wittenbell" # one of: wittenbell, kneserney, lidstone + smoothing_discount: float = 0.75 # used when smoothing_method == "kneserney" + smoothing_gamma: float = 0.1 # used when smoothing_method == "lidstone" + unk_cutoff: int = 2 # tokens seen fewer times fold into the class vocabulary_size: int = 50000 required_data: tuple = ( "punkt", @@ -93,6 +109,40 @@ class GPT2Config: device: Optional[str] = None batch_size: int = 1 cache_dir: Optional[str] = None + # Pin a specific Hugging Face Hub revision (commit hash or tag) for + # reproducible, supply-chain-safe loading. None tracks the default branch; + # set to a commit hash in production. + revision: Optional[str] = None + + +@dataclass(frozen=True) +class RoBERTaConfig: + """Configuration for the (optional) RoBERTa analyzer.""" + + model_name: str = "roberta-base" + max_token_length: int = 512 + revision: Optional[str] = None + + +@dataclass(frozen=True) +class BinocularsConfig: + """Configuration for the Binoculars-style cross-perplexity analyzer. + + Binoculars (Hans et al., 2024) scores text by the ratio of an observer + model's log-perplexity to the cross-perplexity between the observer and a + second "performer" model. Machine-generated text yields a *lower* ratio. + A small observer/performer pair sharing one tokenizer keeps it local and + CPU-friendly; the decision midpoint is calibrated on the bundled benchmark. + """ + + observer_model: str = "gpt2" + performer_model: str = "distilgpt2" + max_token_length: int = 512 + revision: Optional[str] = None + # Lower ratio => more AI. Midpoint calibrated from the benchmark separation + # (human scores ~0.88-1.05, AI ~0.72-0.84; boundary between the clusters). + score_midpoint: float = 0.863 + score_slope: float = 20.0 @dataclass(frozen=True) @@ -111,6 +161,44 @@ class VisualizationConfig: color_secondary: str = "#764ba2" +@dataclass(frozen=True) +class EnsembleConfig: + """Calibration and fusion configuration for the ensemble analyzer. + + Perplexity is mapped to a calibrated AI-probability with a per-analyzer + logistic (see :mod:`src.analyzers.calibration`). The midpoint is the + perplexity at which AI-probability is 0.5 — i.e. the decision boundary — + and was chosen from the observed human/AI perplexity separation on the + bundled benchmark. Re-run ``python -m src.evaluation.benchmark`` after + changing these. + """ + + # GPT-2: lower perplexity => more AI. Human ~40-106, AI ~10-25 on benchmark. + gpt2_ppl_midpoint: float = 30.0 + gpt2_ppl_slope: float = 0.15 + + # Brown-corpus NLTK: higher perplexity => more AI (formal/atypical text). + # Human median ~1200, AI median ~1900 on the benchmark, with overlap — a + # deliberately weak signal, so it carries a smaller weight below. + nltk_ppl_midpoint: float = 1550.0 + nltk_ppl_slope: float = 0.0015 + + # Fusion weights. RoBERTa stays disabled until a fine-tuned checkpoint is + # wired in; GPT-2 is the strongest signal and dominates the blend. + # Binoculars is available but off by default (it needs a second model); to + # enable it, give it a non-zero weight and rebalance so the weights sum to 1. + weight_roberta: float = 0.0 + weight_gpt2: float = 0.75 + weight_nltk: float = 0.25 + weight_binoculars: float = 0.0 + + # Verdict thresholds on the fused, calibrated AI-probability. + ai_threshold: float = 0.70 + likely_ai_threshold: float = 0.55 + likely_human_threshold: float = 0.45 + human_threshold: float = 0.30 + + @dataclass class Settings: """Main application settings.""" @@ -123,6 +211,9 @@ class Settings: thresholds: ThresholdConfig = field(default_factory=ThresholdConfig) nltk: NLTKConfig = field(default_factory=NLTKConfig) gpt2: GPT2Config = field(default_factory=GPT2Config) + roberta: RoBERTaConfig = field(default_factory=RoBERTaConfig) + binoculars: BinocularsConfig = field(default_factory=BinocularsConfig) + ensemble: EnsembleConfig = field(default_factory=EnsembleConfig) visualization: VisualizationConfig = field(default_factory=VisualizationConfig) def __post_init__(self): @@ -134,4 +225,4 @@ def __post_init__(self): @lru_cache(maxsize=1) def get_settings() -> Settings: """Get cached application settings singleton.""" - return Settings() \ No newline at end of file + return Settings() diff --git a/src/evaluation/__init__.py b/src/evaluation/__init__.py new file mode 100644 index 0000000..11bebb1 --- /dev/null +++ b/src/evaluation/__init__.py @@ -0,0 +1,30 @@ +"""Evaluation and benchmarking for AI-text detectors. + +This package provides the measurement layer the detector previously lacked: + +* :mod:`src.evaluation.metrics` — classification and calibration metrics + (accuracy, precision/recall/F1, ROC/AUROC, false-positive/negative rates, + reliability bins and expected calibration error), implemented in pure NumPy. +* :mod:`src.evaluation.dataset` — loader for the bundled, honestly-labelled + benchmark corpus under ``data/benchmark/``. +* :mod:`src.evaluation.benchmark` — runs an analyzer over a labelled dataset and + produces a structured, reproducible report. + +The bundled dataset is intentionally small and is meant for regression and +calibration checks, not as an authoritative accuracy claim. See +``data/benchmark/README.md``. +""" + +from src.evaluation.metrics import ( + BinaryClassificationReport, + binary_report, + expected_calibration_error, + roc_auc, +) + +__all__ = [ + "BinaryClassificationReport", + "binary_report", + "expected_calibration_error", + "roc_auc", +] diff --git a/src/evaluation/benchmark.py b/src/evaluation/benchmark.py new file mode 100644 index 0000000..67f887f --- /dev/null +++ b/src/evaluation/benchmark.py @@ -0,0 +1,271 @@ +""" +Benchmark Runner +================ + +Runs an analyzer over a labelled dataset and produces a reproducible report: +per-sample AI-probabilities, classification metrics at the default and +F1-optimal thresholds, ROC points and reliability bins. + +Usage (CLI):: + + python -m src.evaluation.benchmark --analyzer nltk + python -m src.evaluation.benchmark --analyzer ensemble --output report.json --plots out/ + +The analyzer is addressed by name so the heavy transformer analyzers are only +imported when actually requested. +""" + +from __future__ import annotations + +import argparse +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Optional, Protocol + +from src.config.settings import Verdict +from src.evaluation import metrics +from src.evaluation.dataset import Sample, load_dataset +from src.models.result import AnalysisResult +from src.utils.logging_config import get_logger + +logger = get_logger(__name__) + +# Continuous AI-probability contributed by verdict direction, scaled by +# confidence. Kept monotone so ROC/AUROC are meaningful even for analyzers that +# do not expose an explicit probability. +_AI_VERDICTS = (Verdict.AI_GENERATED, Verdict.LIKELY_AI) +_HUMAN_VERDICTS = (Verdict.HUMAN_WRITTEN, Verdict.LIKELY_HUMAN) + + +class _Analyzer(Protocol): + def analyze(self, text: str) -> AnalysisResult: ... + + +def result_to_ai_probability(result: AnalysisResult) -> float: + """Map an :class:`AnalysisResult` to a single AI-probability in ``[0, 1]``. + + Prefers an explicit ensemble AI score; otherwise derives a monotone score + from the verdict direction and confidence. + """ + for score in result.scores: + if score.name == "Ensemble AI Score": + return max(0.0, min(1.0, float(score.value))) + + conf = max(0.0, min(1.0, result.confidence / 100.0)) + if result.verdict in _AI_VERDICTS: + return 0.5 + 0.5 * conf + if result.verdict in _HUMAN_VERDICTS: + return 0.5 - 0.5 * conf + return 0.5 # UNCERTAIN + + +@dataclass +class SamplePrediction: + id: str + label: int + source: str + ai_probability: float + verdict: str + confidence: float + + def to_dict(self) -> Dict: + return { + "id": self.id, + "label": self.label, + "source": self.source, + "ai_probability": round(self.ai_probability, 4), + "verdict": self.verdict, + "confidence": round(self.confidence, 2), + } + + +@dataclass +class BenchmarkResult: + analyzer_name: str + n_samples: int + predictions: List[SamplePrediction] + report_default: metrics.BinaryClassificationReport + report_best_f1: metrics.BinaryClassificationReport + best_f1_threshold: float + calibration_bins: List[Dict[str, float]] = field(default_factory=list) + + def to_dict(self) -> Dict: + return { + "analyzer": self.analyzer_name, + "n_samples": self.n_samples, + "metrics_at_0.5": self.report_default.to_dict(), + "best_f1_threshold": round(self.best_f1_threshold, 4), + "metrics_at_best_f1": self.report_best_f1.to_dict(), + "calibration_bins": self.calibration_bins, + "predictions": [p.to_dict() for p in self.predictions], + } + + +def run_benchmark( + analyzer: _Analyzer, + samples: List[Sample], + analyzer_name: str = "analyzer", + threshold: float = 0.5, +) -> BenchmarkResult: + """Run ``analyzer`` over ``samples`` and compute metrics.""" + predictions: List[SamplePrediction] = [] + for sample in samples: + result = analyzer.analyze(sample.text) + predictions.append( + SamplePrediction( + id=sample.id, + label=sample.label, + source=sample.source, + ai_probability=result_to_ai_probability(result), + verdict=result.verdict.value, + confidence=result.confidence, + ) + ) + + labels = [p.label for p in predictions] + scores = [p.ai_probability for p in predictions] + + report_default = metrics.binary_report(labels, scores, threshold=threshold) + best_t, _ = metrics.best_threshold_by_f1(labels, scores) + report_best = metrics.binary_report(labels, scores, threshold=best_t) + + return BenchmarkResult( + analyzer_name=analyzer_name, + n_samples=len(samples), + predictions=predictions, + report_default=report_default, + report_best_f1=report_best, + best_f1_threshold=best_t, + calibration_bins=metrics.calibration_bins(labels, scores), + ) + + +def _build_analyzer(name: str) -> _Analyzer: + """Construct an analyzer by name (heavy analyzers imported lazily).""" + name = name.lower() + if name == "nltk": + from src.analyzers.nltk_analyzer import NLTKAnalyzer + + return NLTKAnalyzer(ngram_size=3) + if name == "gpt2": + from src.analyzers.gpt2_analyzer import GPT2Analyzer + + return GPT2Analyzer() + if name == "binoculars": + from src.analyzers.binoculars_analyzer import BinocularsAnalyzer + + return BinocularsAnalyzer() + if name == "ensemble": + from src.analyzers.ensemble_analyzer import EnsembleAnalyzer + + return EnsembleAnalyzer() + raise ValueError(f"Unknown analyzer {name!r}; expected nltk, gpt2, binoculars, or ensemble") + + +def save_plots(result: BenchmarkResult, output_dir: Path) -> List[Path]: + """Save ROC and reliability-diagram PNGs. Requires matplotlib. + + Returns the list of written files (empty if matplotlib is unavailable). + """ + try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except Exception as exc: # pragma: no cover - optional dependency path + logger.warning("matplotlib unavailable; skipping plots: %s", exc) + return [] + + output_dir.mkdir(parents=True, exist_ok=True) + labels = [p.label for p in result.predictions] + scores = [p.ai_probability for p in result.predictions] + written: List[Path] = [] + + # ROC curve. + fpr, tpr, _ = metrics.roc_curve(labels, scores) + auc = result.report_default.roc_auc + fig, ax = plt.subplots(figsize=(5, 5)) + ax.plot(fpr, tpr, label=f"AUROC = {auc:.3f}") + ax.plot([0, 1], [0, 1], linestyle="--", color="gray", label="chance") + ax.set_xlabel("False Positive Rate") + ax.set_ylabel("True Positive Rate") + ax.set_title(f"ROC — {result.analyzer_name}") + ax.legend(loc="lower right") + roc_path = output_dir / f"roc_{result.analyzer_name}.png" + fig.savefig(roc_path, dpi=120, bbox_inches="tight") + plt.close(fig) + written.append(roc_path) + + # Reliability diagram. + bins = [b for b in result.calibration_bins if b["count"] > 0] + fig, ax = plt.subplots(figsize=(5, 5)) + ax.plot([0, 1], [0, 1], linestyle="--", color="gray", label="perfect") + if bins: + ax.plot( + [b["mean_predicted"] for b in bins], + [b["observed_fraction"] for b in bins], + marker="o", + label=f"ECE = {result.report_default.expected_calibration_error:.3f}", + ) + ax.set_xlabel("Mean predicted AI-probability") + ax.set_ylabel("Observed AI fraction") + ax.set_title(f"Calibration — {result.analyzer_name}") + ax.legend(loc="upper left") + cal_path = output_dir / f"calibration_{result.analyzer_name}.png" + fig.savefig(cal_path, dpi=120, bbox_inches="tight") + plt.close(fig) + written.append(cal_path) + + return written + + +def _format_summary(result: BenchmarkResult) -> str: + r = result.report_default + return ( + f"\n=== Benchmark: {result.analyzer_name} ({result.n_samples} samples) ===\n" + f"Accuracy : {r.accuracy:.3f}\n" + f"Precision : {r.precision:.3f}\n" + f"Recall : {r.recall:.3f}\n" + f"F1 : {r.f1:.3f}\n" + f"AUROC : {r.roc_auc:.3f}\n" + f"FPR (human flagged as AI): {r.false_positive_rate:.3f}\n" + f"FNR (AI missed) : {r.false_negative_rate:.3f}\n" + f"ECE (calibration error) : {r.expected_calibration_error:.3f}\n" + f"Best-F1 threshold : {result.best_f1_threshold:.3f} " + f"(F1 ={result.report_best_f1.f1:.3f})\n" + ) + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description="Benchmark an AI-text detector.") + parser.add_argument( + "--analyzer", default="nltk", choices=["nltk", "gpt2", "binoculars", "ensemble"] + ) + parser.add_argument("--dataset", default=None, help="Path to a JSONL dataset") + parser.add_argument("--threshold", type=float, default=0.5) + parser.add_argument("--output", default=None, help="Write full report JSON here") + parser.add_argument("--plots", default=None, help="Directory for ROC/calibration PNGs") + args = parser.parse_args(argv) + + samples = load_dataset(args.dataset) + analyzer = _build_analyzer(args.analyzer) + result = run_benchmark(analyzer, samples, analyzer_name=args.analyzer, threshold=args.threshold) + + print(_format_summary(result)) + + if args.output: + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(result.to_dict(), indent=2), encoding="utf-8") + print(f"Wrote report to {args.output}") + if args.plots: + written = save_plots(result, Path(args.plots)) + for path in written: + print(f"Wrote plot {path}") + + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/src/evaluation/dataset.py b/src/evaluation/dataset.py new file mode 100644 index 0000000..8c132a8 --- /dev/null +++ b/src/evaluation/dataset.py @@ -0,0 +1,90 @@ +""" +Benchmark Dataset Loader +======================== + +Loads the bundled, labelled benchmark corpus (JSONL) used to measure the +detector. Each line is an object: ``{"id", "label", "source", "text"}`` where +``label`` is ``"human"`` or ``"ai"``. + +The bundled corpus is deliberately small and is meant for regression and +calibration checks, not as an authoritative accuracy benchmark. Point the loader +at a larger JSONL of the same shape to evaluate on your own data. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Union + +# data/benchmark/samples.jsonl relative to the repository root. +DEFAULT_DATASET_PATH = Path(__file__).resolve().parents[2] / "data" / "benchmark" / "samples.jsonl" + +_LABEL_TO_INT = {"human": 0, "ai": 1} + + +@dataclass(frozen=True) +class Sample: + """One labelled benchmark example.""" + + id: str + label: int # 0 = human, 1 = AI + source: str + text: str + + @property + def is_ai(self) -> bool: + return self.label == 1 + + +def load_dataset(path: Optional[Union[Path, str]] = None) -> List[Sample]: + """Load labelled samples from a JSONL file. + + Args: + path: Path to a JSONL dataset. Defaults to the bundled corpus. + + Returns: + List of :class:`Sample`. + + Raises: + FileNotFoundError: If the dataset file does not exist. + ValueError: If a record is malformed or has an unknown label. + """ + dataset_path = Path(path) if path is not None else DEFAULT_DATASET_PATH + if not dataset_path.exists(): + raise FileNotFoundError(f"Benchmark dataset not found: {dataset_path}") + + samples: List[Sample] = [] + with dataset_path.open("r", encoding="utf-8") as fh: + for line_no, line in enumerate(fh, start=1): + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON on line {line_no}: {exc}") from exc + + label_raw = str(record.get("label", "")).lower() + if label_raw not in _LABEL_TO_INT: + raise ValueError( + f"Unknown label {record.get('label')!r} on line {line_no}; " + f"expected one of {sorted(_LABEL_TO_INT)}" + ) + text = record.get("text", "") + if not isinstance(text, str) or not text.strip(): + raise ValueError(f"Empty or non-string text on line {line_no}") + + samples.append( + Sample( + id=str(record.get("id", f"sample-{line_no}")), + label=_LABEL_TO_INT[label_raw], + source=str(record.get("source", "unknown")), + text=text, + ) + ) + + if not samples: + raise ValueError(f"Dataset {dataset_path} contained no samples") + return samples diff --git a/src/evaluation/metrics.py b/src/evaluation/metrics.py new file mode 100644 index 0000000..e58ca97 --- /dev/null +++ b/src/evaluation/metrics.py @@ -0,0 +1,245 @@ +""" +Evaluation Metrics +================== + +Pure-NumPy classification and calibration metrics for AI-text detection. + +The positive class is **AI-generated** (label ``1``); the negative class is +**human-written** (label ``0``). Scores are AI-probabilities in ``[0, 1]``. + +No scikit-learn dependency is introduced — every metric here is implemented +directly so the evaluation layer stays lightweight and auditable. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Dict, List, Sequence, Tuple + +import numpy as np + + +@dataclass +class BinaryClassificationReport: + """Structured metrics for a binary detector at a fixed threshold.""" + + threshold: float + n_samples: int + n_positive: int + n_negative: int + + true_positives: int + false_positives: int + true_negatives: int + false_negatives: int + + accuracy: float + precision: float + recall: float # a.k.a. true positive rate / sensitivity + f1: float + specificity: float # true negative rate + + false_positive_rate: float # human text wrongly flagged as AI + false_negative_rate: float # AI text missed + + roc_auc: float + expected_calibration_error: float + + def to_dict(self) -> Dict: + return asdict(self) + + +def _as_arrays(labels: Sequence[int], scores: Sequence[float]) -> Tuple[np.ndarray, np.ndarray]: + y = np.asarray(labels, dtype=int) + s = np.asarray(scores, dtype=float) + if y.shape != s.shape: + raise ValueError("labels and scores must have the same length") + if y.size == 0: + raise ValueError("cannot compute metrics on an empty dataset") + if not np.all((y == 0) | (y == 1)): + raise ValueError("labels must be 0 (human) or 1 (AI)") + if np.any((s < 0) | (s > 1)): + raise ValueError("scores must be AI-probabilities in [0, 1]") + return y, s + + +def confusion_counts( + labels: Sequence[int], scores: Sequence[float], threshold: float = 0.5 +) -> Tuple[int, int, int, int]: + """Return (tp, fp, tn, fn) for ``score >= threshold`` => predicted AI.""" + y, s = _as_arrays(labels, scores) + pred = (s >= threshold).astype(int) + tp = int(np.sum((pred == 1) & (y == 1))) + fp = int(np.sum((pred == 1) & (y == 0))) + tn = int(np.sum((pred == 0) & (y == 0))) + fn = int(np.sum((pred == 0) & (y == 1))) + return tp, fp, tn, fn + + +def roc_curve( + labels: Sequence[int], scores: Sequence[float] +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Compute the ROC curve. + + Returns ``(fpr, tpr, thresholds)`` with points sorted by decreasing + threshold, suitable for plotting or AUROC integration. + """ + y, s = _as_arrays(labels, scores) + # Sort by score descending. + order = np.argsort(-s, kind="mergesort") + y = y[order] + s = s[order] + + p = np.sum(y == 1) + n = np.sum(y == 0) + if p == 0 or n == 0: + # Degenerate: only one class present. ROC is undefined; return trivial. + return np.array([0.0, 1.0]), np.array([0.0, 1.0]), np.array([1.0, 0.0]) + + tps = np.cumsum(y == 1) + fps = np.cumsum(y == 0) + + # Keep the last index of each distinct score (threshold boundaries). + distinct = np.where(np.diff(s) != 0)[0] + idx = np.r_[distinct, s.size - 1] + + tpr = np.r_[0.0, tps[idx] / p] + fpr = np.r_[0.0, fps[idx] / n] + thresholds = np.r_[np.inf, s[idx]] + return fpr, tpr, thresholds + + +def roc_auc(labels: Sequence[int], scores: Sequence[float]) -> float: + """Area under the ROC curve via the Mann-Whitney U statistic. + + Robust to ties. Returns ``0.5`` when only one class is present (undefined). + """ + y, s = _as_arrays(labels, scores) + pos = s[y == 1] + neg = s[y == 0] + if pos.size == 0 or neg.size == 0: + return 0.5 + # Rank-based AUROC (handles ties via average ranks). + order = np.argsort(s, kind="mergesort") + ranks = np.empty(s.size, dtype=float) + sorted_scores = s[order] + i = 0 + while i < s.size: + j = i + while j + 1 < s.size and sorted_scores[j + 1] == sorted_scores[i]: + j += 1 + avg_rank = (i + j) / 2.0 + 1.0 # 1-based average rank + ranks[order[i : j + 1]] = avg_rank + i = j + 1 + sum_ranks_pos = np.sum(ranks[y == 1]) + n_pos = pos.size + n_neg = neg.size + auc = (sum_ranks_pos - n_pos * (n_pos + 1) / 2.0) / (n_pos * n_neg) + return float(auc) + + +def calibration_bins( + labels: Sequence[int], scores: Sequence[float], n_bins: int = 10 +) -> List[Dict[str, float]]: + """Reliability-diagram bins. + + Each bin reports mean predicted AI-probability vs observed AI fraction. + """ + y, s = _as_arrays(labels, scores) + edges = np.linspace(0.0, 1.0, n_bins + 1) + bins: List[Dict[str, float]] = [] + for b in range(n_bins): + lo, hi = edges[b], edges[b + 1] + if b == n_bins - 1: + mask = (s >= lo) & (s <= hi) + else: + mask = (s >= lo) & (s < hi) + count = int(np.sum(mask)) + if count == 0: + bins.append( + { + "bin_lower": float(lo), + "bin_upper": float(hi), + "count": 0, + "mean_predicted": float("nan"), + "observed_fraction": float("nan"), + } + ) + continue + bins.append( + { + "bin_lower": float(lo), + "bin_upper": float(hi), + "count": count, + "mean_predicted": float(np.mean(s[mask])), + "observed_fraction": float(np.mean(y[mask])), + } + ) + return bins + + +def expected_calibration_error( + labels: Sequence[int], scores: Sequence[float], n_bins: int = 10 +) -> float: + """Expected Calibration Error (weighted |confidence - accuracy|).""" + y, s = _as_arrays(labels, scores) + total = y.size + ece = 0.0 + for b in calibration_bins(labels, scores, n_bins=n_bins): + if b["count"] == 0: + continue + ece += (b["count"] / total) * abs(b["mean_predicted"] - b["observed_fraction"]) + return float(ece) + + +def binary_report( + labels: Sequence[int], scores: Sequence[float], threshold: float = 0.5 +) -> BinaryClassificationReport: + """Compute a full metrics report at a fixed decision threshold.""" + y, _ = _as_arrays(labels, scores) + tp, fp, tn, fn = confusion_counts(labels, scores, threshold) + + n = tp + fp + tn + fn + n_pos = tp + fn + n_neg = tn + fp + + def _safe(numer: float, denom: float) -> float: + return float(numer / denom) if denom else 0.0 + + accuracy = _safe(tp + tn, n) + precision = _safe(tp, tp + fp) + recall = _safe(tp, tp + fn) + specificity = _safe(tn, tn + fp) + f1 = _safe(2 * precision * recall, precision + recall) + + return BinaryClassificationReport( + threshold=float(threshold), + n_samples=int(n), + n_positive=int(n_pos), + n_negative=int(n_neg), + true_positives=tp, + false_positives=fp, + true_negatives=tn, + false_negatives=fn, + accuracy=accuracy, + precision=precision, + recall=recall, + f1=f1, + specificity=specificity, + false_positive_rate=_safe(fp, fp + tn), + false_negative_rate=_safe(fn, fn + tp), + roc_auc=roc_auc(labels, scores), + expected_calibration_error=expected_calibration_error(labels, scores), + ) + + +def best_threshold_by_f1(labels: Sequence[int], scores: Sequence[float]) -> Tuple[float, float]: + """Return ``(threshold, f1)`` maximising F1 over candidate thresholds.""" + _as_arrays(labels, scores) + candidates = sorted(set([0.0] + list(np.asarray(scores, dtype=float)) + [1.0])) + best_t, best_f1 = 0.5, -1.0 + for t in candidates: + rep = binary_report(labels, scores, threshold=t) + if rep.f1 > best_f1: + best_f1, best_t = rep.f1, t + return float(best_t), float(best_f1) diff --git a/src/models/AGENTS.md b/src/models/AGENTS.md new file mode 100644 index 0000000..465f9a4 --- /dev/null +++ b/src/models/AGENTS.md @@ -0,0 +1,59 @@ +# src/models & src/config — Data Models and Settings + +## Purpose +`src/models/` owns the serializable result shape (`AnalysisResult`, `TextMetrics`, `DetectionScore`). +`src/config/` owns enums, frozen threshold/subsystem configs, and the `Settings` singleton. +Both directories are intentionally thin — no analysis logic, no I/O, no Streamlit imports. + +## Entry Points +- `src/models/result.py` — `AnalysisResult`, `TextMetrics`, `DetectionScore` dataclasses + `to_dict()` / `to_json()` +- `src/config/settings.py` — `Verdict`, `ConfidenceLevel`, `DetectionMethod` enums; `ThresholdConfig`, `NLTKConfig`, `GPT2Config`, `VisualizationConfig` frozen dataclasses; mutable `Settings` + `get_settings()` singleton + +## Contracts & Invariants + +### AnalysisResult +- Default state is `Verdict.UNCERTAIN`, `confidence=0.0`, `ConfidenceLevel.LOW` — always valid even if analysis fails +- `add_warning(msg)` is idempotent (deduplicates). Use it instead of directly appending to `result.warnings` +- `add_score(DetectionScore)` appends to `result.scores`. The order matters for display — add ensemble/overall score first, then individual analyzer scores +- `to_dict()` / `to_json()` produce stable serialized output consumed by the "Analysis Metadata" expander in the UI. Do not change key names without updating display code +- `is_ai_generated` and `is_human_written` are convenience properties; they do not cover `Verdict.UNCERTAIN` — check explicitly if uncertain handling matters + +### TextMetrics +- `lexical_diversity` is a computed `@property` (unique_words / total_words), not stored — do not assign it +- `word_frequencies` is a raw dict; downstream code (e.g. `ChartGenerator`) filters by min frequency — do not pre-filter here + +### DetectionScore +- `indicates_ai=True` drives the 🔴/🟢 indicator in the UI score rows — set it accurately +- `weight` should reflect the actual contribution to ensemble fusion if this score is used in weighted averaging + +### Settings & Config +- `get_settings()` is `@lru_cache(maxsize=1)` — the single instance is shared process-wide. Never instantiate `Settings()` directly +- Threshold values (`ThresholdConfig`) are `frozen=True` — do not attempt mutation; create a new instance if you need different thresholds in tests +- Environment overrides: `AI_DETECTOR_DEBUG=true` sets `settings.debug`; `AI_DETECTOR_LOG_LEVEL` sets `settings.log_level` — these are the only env vars the app reads +- `NLTKConfig.required_data` tuple is the authoritative list of NLTK corpora to download — update here if adding new NLTK resources + +## Patterns +To add a new field to `AnalysisResult`: +1. Add the field with a sensible default to the dataclass in `result.py` +2. Update `to_dict()` to include it (required for JSON export and test assertions) +3. Populate it in the relevant analyzer's `_perform_analysis` + +To add a new threshold: +1. Add the field to `ThresholdConfig` with a default value +2. Reference it via `self.thresholds.new_field` in the analyzer +3. Add test assertions in `tests/test_base_analyzer_contract.py` if it affects verdict logic + +To add a new verdict level: +1. Add to the `Verdict` enum +2. Update `BaseAnalyzer._determine_verdict` probability branches +3. Update verdict-to-CSS mapping in `app.py` and `ensemble.py` +4. Update `is_ai_generated` / `is_human_written` properties if needed + +## Anti-patterns +- Do not add business logic (scoring, thresholds, text analysis) to result dataclasses +- Do not import analyzer modules from `models/` or `config/` — the dependency direction is one-way: analyzers → models/config, never the reverse +- Do not call `get_settings()` inside dataclass `__post_init__` methods — pass config explicitly if needed + +## Related Context +- Consumers of these models: `src/analyzers/AGENTS.md` +- Visualization of results: `src/utils/AGENTS.md` diff --git a/src/models/__init__.py b/src/models/__init__.py index 7ac9cc0..556ee3e 100644 --- a/src/models/__init__.py +++ b/src/models/__init__.py @@ -1,5 +1,5 @@ """Data models module.""" -from src.models.result import AnalysisResult, TextMetrics, DetectionScore +from src.models.result import AnalysisResult, DetectionScore, TextMetrics __all__ = ["AnalysisResult", "TextMetrics", "DetectionScore"] diff --git a/src/models/result.py b/src/models/result.py index 1ee3bcb..110700e 100644 --- a/src/models/result.py +++ b/src/models/result.py @@ -8,9 +8,9 @@ from __future__ import annotations import json -from dataclasses import dataclass, field, asdict +from dataclasses import asdict, dataclass, field from datetime import datetime -from typing import Dict, List, Optional +from typing import Dict, List from src.config.settings import ConfidenceLevel, Verdict @@ -129,4 +129,4 @@ def to_dict(self) -> Dict: def to_json(self) -> str: """Convert to JSON string.""" - return json.dumps(self.to_dict(), indent=2) \ No newline at end of file + return json.dumps(self.to_dict(), indent=2) diff --git a/src/ui/__init__.py b/src/ui/__init__.py new file mode 100644 index 0000000..3b841ae --- /dev/null +++ b/src/ui/__init__.py @@ -0,0 +1,31 @@ +"""Shared Streamlit UI helpers. + +The three entry-point apps (``app.py``, ``gpt2_app.py``, ``ensemble.py``) previously +duplicated their CSS, verdict/emoji mappings, verdict-card, warning, and footer +rendering. Those shared pieces live here so a change is made once and stays +consistent across apps. + +- :mod:`src.ui.styles` — common CSS + ``inject_css``. +- :mod:`src.ui.components` — verdict/emoji maps and result-rendering helpers. +""" + +from src.ui.components import ( + render_error, + render_footer, + render_verdict_card, + render_warnings, + verdict_css_class, + verdict_emoji, +) +from src.ui.styles import BASE_CSS, inject_css + +__all__ = [ + "BASE_CSS", + "inject_css", + "render_error", + "render_footer", + "render_verdict_card", + "render_warnings", + "verdict_css_class", + "verdict_emoji", +] diff --git a/src/ui/components.py b/src/ui/components.py new file mode 100644 index 0000000..872bf31 --- /dev/null +++ b/src/ui/components.py @@ -0,0 +1,109 @@ +"""Shared Streamlit result-rendering components. + +The pure mapping helpers (:func:`verdict_css_class`, :func:`verdict_emoji`) are +unit-tested without Streamlit; the ``render_*`` helpers wrap ``st.markdown`` and +are exercised by the app smoke tests. + +All user-derived strings are HTML-escaped before being embedded in the +``unsafe_allow_html`` markup, closing the injection risk the audit flagged. +""" + +from __future__ import annotations + +import html + +import streamlit as st + +from src.config.settings import Verdict +from src.models.result import AnalysisResult +from src.utils.logging_config import get_logger + +_logger = get_logger(__name__) + +_VERDICT_CSS = { + Verdict.AI_GENERATED: "verdict-ai", + Verdict.LIKELY_AI: "verdict-likely-ai", + Verdict.UNCERTAIN: "verdict-uncertain", + Verdict.LIKELY_HUMAN: "verdict-likely-human", + Verdict.HUMAN_WRITTEN: "verdict-human", +} + +_VERDICT_EMOJI = { + Verdict.AI_GENERATED: "🤖", + Verdict.LIKELY_AI: "🤖", + Verdict.UNCERTAIN: "❓", + Verdict.LIKELY_HUMAN: "👤", + Verdict.HUMAN_WRITTEN: "👤", +} + + +def verdict_css_class(verdict: Verdict) -> str: + """CSS class for a verdict (falls back to the uncertain style).""" + return _VERDICT_CSS.get(verdict, "verdict-uncertain") + + +def verdict_emoji(verdict: Verdict) -> str: + """Emoji for a verdict (falls back to the uncertain glyph).""" + return _VERDICT_EMOJI.get(verdict, "❓") + + +def render_verdict_card(result: AnalysisResult) -> None: + """Render the verdict card (verdict, confidence, analysis time).""" + css_class = verdict_css_class(result.verdict) + emoji = verdict_emoji(result.verdict) + st.markdown( + f""" +
+

{emoji} {html.escape(result.verdict.value)}

+

Confidence: {result.confidence:.1f}% ({html.escape(result.confidence_level.value)}) + • Analysis Time: {result.analysis_time:.2f}s

+
+""", + unsafe_allow_html=True, + ) + + +def render_error( + exc: Exception, + user_message: str = ( + "Something went wrong during analysis. Please try again, " + "possibly with shorter or different text." + ), +) -> None: + """Show a generic error to the user and log the full exception server-side. + + Exception strings are never rendered to the UI (they can leak internal + detail); the traceback goes to the server log instead. + """ + _logger.error("Unhandled UI error: %s", exc, exc_info=True) + st.error(f"❌ {html.escape(user_message)}") + + +def render_warnings(result: AnalysisResult) -> None: + """Render each analysis warning in a warning box (HTML-escaped).""" + for warning in result.warnings: + st.markdown( + f'
⚠️ {html.escape(warning)}
', + unsafe_allow_html=True, + ) + + +def render_footer( + engine_label: str, icon: str = "🛡️", note: str = "", version: str = "2.0.0" +) -> None: + """Render the shared footer with an app-specific engine label and icon. + + ``note`` renders as an optional small second line (e.g. a disabled-analyzer + disclaimer); it is HTML-escaped. + """ + note_html = f"
⚠️ {html.escape(note)}" if note else "" + st.markdown( + f""" +
+

{icon} AI Text Detector v{html.escape(version)} • {html.escape(engine_label)}{note_html}

+

⚠️ Results are probabilistic estimates, not definitive classifications.

+

No text is stored or transmitted. All processing happens locally.

+
+""", + unsafe_allow_html=True, + ) diff --git a/src/ui/styles.py b/src/ui/styles.py new file mode 100644 index 0000000..6db2f32 --- /dev/null +++ b/src/ui/styles.py @@ -0,0 +1,142 @@ +"""Shared CSS for the Streamlit apps. + +``BASE_CSS`` holds the styling common to all three entry points (verdict cards, +metric cards, warning box, footer, layout, hidden branding). App-specific styles +(e.g. a page header gradient) are passed to :func:`inject_css` as ``extra_css``. +""" + +from __future__ import annotations + +import streamlit as st + +BASE_CSS = """ + /* Main container */ + .main .block-container { + padding-top: 2rem; + padding-bottom: 2rem; + max-width: 1200px; + } + + /* Result cards */ + .verdict-card { + padding: 1.5rem 2rem; + border-radius: 12px; + text-align: center; + margin: 1rem 0; + box-shadow: 0 4px 15px rgba(0, 0, 0, 0.2); + } + .verdict-ai { + background: linear-gradient(135deg, #ff416c, #ff4b2b); + border: 2px solid #ff416c; + } + .verdict-likely-ai { + background: linear-gradient(135deg, #f7971e, #ffd200); + border: 2px solid #f7971e; + } + .verdict-uncertain { + background: linear-gradient(135deg, #a8a8a8, #6c6c6c); + border: 2px solid #a8a8a8; + } + .verdict-likely-human { + background: linear-gradient(135deg, #56ab2f, #a8e063); + border: 2px solid #56ab2f; + } + .verdict-human { + background: linear-gradient(135deg, #11998e, #38ef7d); + border: 2px solid #11998e; + } + .verdict-card h2 { + color: white; + margin: 0; + font-size: 1.6rem; + text-shadow: 0 2px 4px rgba(0, 0, 0, 0.2); + } + .verdict-card p { + color: rgba(255, 255, 255, 0.9); + margin: 0.5rem 0 0 0; + font-size: 1rem; + } + + /* Metric cards */ + .metric-card { + background: rgba(255, 255, 255, 0.05); + border: 1px solid rgba(255, 255, 255, 0.1); + padding: 1.2rem; + border-radius: 10px; + text-align: center; + transition: transform 0.2s ease; + } + .metric-card:hover { + transform: translateY(-2px); + border-color: rgba(102, 126, 234, 0.5); + } + .metric-value { + font-size: 1.8rem; + font-weight: 700; + color: #667eea; + } + .metric-label { + font-size: 0.85rem; + color: rgba(255, 255, 255, 0.6); + margin-top: 0.3rem; + } + .metric-interpretation { + font-size: 0.75rem; + color: rgba(255, 255, 255, 0.45); + margin-top: 0.2rem; + font-style: italic; + } + + /* Score detail */ + .score-row { + background: rgba(255, 255, 255, 0.03); + border: 1px solid rgba(255, 255, 255, 0.06); + padding: 0.8rem 1rem; + border-radius: 8px; + margin-bottom: 0.5rem; + } + + /* Footer */ + .footer { + text-align: center; + padding: 2rem 0 1rem 0; + color: rgba(255, 255, 255, 0.3); + font-size: 0.8rem; + border-top: 1px solid rgba(255, 255, 255, 0.05); + margin-top: 3rem; + } + + /* Warning box */ + .warning-box { + background: rgba(255, 170, 0, 0.1); + border: 1px solid rgba(255, 170, 0, 0.3); + border-radius: 8px; + padding: 0.8rem 1rem; + margin: 0.5rem 0; + font-size: 0.9rem; + } + + /* Sidebar styling */ + .sidebar .sidebar-content { + padding-top: 1rem; + } + + /* Loading info box */ + .loading-info { + background: rgba(102, 126, 234, 0.1); + border: 1px solid rgba(102, 126, 234, 0.3); + border-radius: 10px; + padding: 1rem; + text-align: center; + } + + /* Hide Streamlit branding */ + #MainMenu {visibility: hidden;} + footer {visibility: hidden;} + header {visibility: hidden;} +""" + + +def inject_css(extra_css: str = "") -> None: + """Inject the shared CSS (plus any app-specific ``extra_css``).""" + st.markdown(f"", unsafe_allow_html=True) diff --git a/src/utils/AGENTS.md b/src/utils/AGENTS.md new file mode 100644 index 0000000..3e31654 --- /dev/null +++ b/src/utils/AGENTS.md @@ -0,0 +1,53 @@ +# src/utils — Shared Utilities + +## Purpose +Cross-cutting helpers consumed by analyzers, Streamlit apps, and tests. +Four distinct responsibilities: text processing, chart generation, UI copy contracts, +and logging setup. Does **not** own analysis logic or data models. + +## Entry Points +- `text_processing.py` — `TextProcessor`: text cleaning, tokenization, `TextMetrics` computation, NLTK bootstrap +- `visualization.py` — `ChartGenerator`: Plotly (primary) + Matplotlib Agg (fallback) chart factories +- `ui_contract.py` — Shared markdown strings for sidebar limitations, result reminders, mode guidance +- `logging_config.py` — `setup_logging()`, `get_logger()`, third-party log quieting + +## Contracts & Invariants + +### TextProcessor +- `TextProcessor.clean_text(text)` and `TextProcessor.compute_metrics(text)` are **classmethods** — call them without instantiation +- `compute_metrics` returns a `TextMetrics` dataclass; it always populates `word_frequencies`, `sentence_lengths`, and all scalar fields — never assume optional fields are absent +- NLTK data bootstrap is guarded by `_nltk_initialized` class-level flag — it runs once per process. Do not call `nltk.download()` elsewhere in the codebase +- `TextProcessor` uses class-level `_stopwords` cache; do not pass stopwords around manually + +### ChartGenerator +- All chart methods return a Plotly `Figure` (or Matplotlib `Figure` for Agg methods) — callers pass it to `st.plotly_chart()` or `st.pyplot()` +- `ChartGenerator` is configured from `VisualizationConfig` — color constants and default sizes live there, not in the chart methods +- `create_metrics_gauge(result)` expects a full `AnalysisResult`; `create_word_frequency_chart_plotly(freq_dict, top_n)` expects a pre-filtered dict + +### ui_contract.py +- All Streamlit entry points **must** use `build_limitations_markdown()`, `build_result_reminder_markdown()`, and `build_mode_guidance_markdown()` — do not inline the copy +- `LIMITATIONS_BULLETS` and `RESULT_LEVEL_REMINDER` are the single source of truth for ethical framing copy; update here, not in individual apps + +### logging_config.py +- Always call `get_logger(__name__)` at module level — never pass logger instances around +- `setup_logging()` suppresses `transformers`, `torch`, `urllib3`, `filelock` loggers — call it once at app startup (already done in each Streamlit entrypoint) + +## Patterns +To add a new chart type: +1. Add a method to `ChartGenerator` in `visualization.py` +2. Accept an `AnalysisResult` or specific data dict as input +3. Use color constants from `self.config` (`VisualizationConfig`) +4. Return a Plotly `Figure`; use Matplotlib only if Plotly can't handle the chart type + +To add new UI copy (e.g. a new sidebar section): +1. Add the string constant or builder function to `ui_contract.py` +2. Import and call it in the relevant Streamlit app — do not inline strings + +## Anti-patterns +- Do not import `streamlit` inside `text_processing.py`, `visualization.py`, or `logging_config.py` — they must remain importable without Streamlit +- Do not call `setup_logging()` more than once per process; it's already called in each app's setup block +- Do not store per-request state on `TextProcessor` class attributes — they are shared across all calls + +## Related Context +- Data models used by TextProcessor output: `src/models/AGENTS.md` +- Callers of ChartGenerator and ui_contract: `app.py`, `ensemble.py`, `test.py` at repo root diff --git a/src/utils/__init__.py b/src/utils/__init__.py index d45baca..be6a11b 100644 --- a/src/utils/__init__.py +++ b/src/utils/__init__.py @@ -1,12 +1,12 @@ """Utility modules.""" +from src.utils.logging_config import get_logger, setup_logging from src.utils.text_processing import TextProcessor from src.utils.visualization import ChartGenerator -from src.utils.logging_config import setup_logging, get_logger __all__ = [ "TextProcessor", "ChartGenerator", "setup_logging", "get_logger", -] \ No newline at end of file +] diff --git a/src/utils/logging_config.py b/src/utils/logging_config.py index 4fee309..3597e7f 100644 --- a/src/utils/logging_config.py +++ b/src/utils/logging_config.py @@ -55,4 +55,4 @@ def get_logger(name: str) -> logging.Logger: Returns: Configured logger instance. """ - return logging.getLogger(name) \ No newline at end of file + return logging.getLogger(name) diff --git a/src/utils/text_processing.py b/src/utils/text_processing.py index 428fd53..9a7b258 100644 --- a/src/utils/text_processing.py +++ b/src/utils/text_processing.py @@ -42,17 +42,28 @@ def ensure_nltk_data(cls) -> None: "averaged_perceptron_tagger", ] + all_available = True for package in required_packages: + resource = f"tokenizers/{package}" if "punkt" in package else package try: - nltk.data.find(f"tokenizers/{package}" if "punkt" in package else package) + nltk.data.find(resource) + continue except LookupError: - try: - logger.info(f"Downloading NLTK package: {package}") - nltk.download(package, quiet=True) - except Exception as e: - logger.warning(f"Failed to download {package}: {e}") + pass - cls._nltk_initialized = True + try: + logger.info(f"Downloading NLTK package: {package}") + nltk.download(package, quiet=True) + # Verify the resource is actually present after download. + nltk.data.find(resource) + except Exception as e: + logger.warning(f"Failed to obtain NLTK package {package}: {e}") + all_available = False + + # Only mark initialized when every required resource is present, so a + # failed/blocked download is retried on the next call instead of being + # silently treated as success. + cls._nltk_initialized = all_available @classmethod def get_stop_words(cls) -> set: @@ -64,19 +75,108 @@ def get_stop_words(cls) -> set: except Exception: logger.warning("Could not load stopwords, using default set") cls._stop_words = { - "the", "a", "an", "is", "are", "was", "were", "be", "been", - "being", "have", "has", "had", "do", "does", "did", "will", - "would", "could", "should", "may", "might", "can", "shall", - "to", "of", "in", "for", "on", "with", "at", "by", "from", - "as", "into", "through", "during", "before", "after", "above", - "below", "between", "and", "but", "or", "nor", "not", "so", - "yet", "both", "either", "neither", "each", "every", "all", - "any", "few", "more", "most", "other", "some", "such", "no", - "only", "own", "same", "than", "too", "very", "just", "because", - "this", "that", "these", "those", "i", "me", "my", "myself", - "we", "our", "ours", "you", "your", "he", "him", "his", "she", - "her", "it", "its", "they", "them", "their", "what", "which", - "who", "whom", "when", "where", "why", "how", + "the", + "a", + "an", + "is", + "are", + "was", + "were", + "be", + "been", + "being", + "have", + "has", + "had", + "do", + "does", + "did", + "will", + "would", + "could", + "should", + "may", + "might", + "can", + "shall", + "to", + "of", + "in", + "for", + "on", + "with", + "at", + "by", + "from", + "as", + "into", + "through", + "during", + "before", + "after", + "above", + "below", + "between", + "and", + "but", + "or", + "nor", + "not", + "so", + "yet", + "both", + "either", + "neither", + "each", + "every", + "all", + "any", + "few", + "more", + "most", + "other", + "some", + "such", + "no", + "only", + "own", + "same", + "than", + "too", + "very", + "just", + "because", + "this", + "that", + "these", + "those", + "i", + "me", + "my", + "myself", + "we", + "our", + "ours", + "you", + "your", + "he", + "him", + "his", + "she", + "her", + "it", + "its", + "they", + "them", + "their", + "what", + "which", + "who", + "whom", + "when", + "where", + "why", + "how", } return cls._stop_words @@ -132,8 +232,13 @@ def tokenize_sentences(cls, text: str) -> List[str]: return [s.strip() for s in re.split(r"[.!?]+", text) if s.strip()] @classmethod - def tokenize_words(cls, text: str, remove_stopwords: bool = False, - remove_punctuation: bool = True, lowercase: bool = True) -> List[str]: + def tokenize_words( + cls, + text: str, + remove_stopwords: bool = False, + remove_punctuation: bool = True, + lowercase: bool = True, + ) -> List[str]: """ Tokenize text into words with configurable preprocessing. @@ -184,9 +289,7 @@ def compute_metrics(cls, text: str) -> TextMetrics: sentences = cls.tokenize_sentences(cleaned) all_words = cls.tokenize_words(cleaned, remove_punctuation=True) - content_words = cls.tokenize_words( - cleaned, remove_stopwords=True, remove_punctuation=True - ) + content_words = cls.tokenize_words(cleaned, remove_stopwords=True, remove_punctuation=True) total_words = len(all_words) unique_words = len(set(all_words)) @@ -198,9 +301,7 @@ def compute_metrics(cls, text: str) -> TextMetrics: sentence_lengths = [len(cls.tokenize_words(s, remove_punctuation=True)) for s in sentences] # Average word length - avg_word_length = ( - sum(len(w) for w in all_words) / total_words if total_words > 0 else 0.0 - ) + avg_word_length = sum(len(w) for w in all_words) / total_words if total_words > 0 else 0.0 # Average sentence length avg_sentence_length = ( @@ -287,7 +388,7 @@ def compute_burstiness(cls, text: str) -> Tuple[float, Dict[str, float]]: # Variance of word frequencies variance = sum((f - mean_freq) ** 2 for f in frequencies) / len(frequencies) - std_dev = variance ** 0.5 + std_dev = variance**0.5 # Burstiness: (std - mean) / (std + mean) # Range: [-1, 1], higher = more bursty = more human-like @@ -327,7 +428,7 @@ def compute_sentence_variance(cls, text: str) -> float: return 0.0 lengths = [len(cls.tokenize_words(s, remove_punctuation=True)) for s in sentences] - lengths = [l for l in lengths if l > 0] + lengths = [ln for ln in lengths if ln > 0] if not lengths: return 0.0 @@ -337,10 +438,10 @@ def compute_sentence_variance(cls, text: str) -> float: if mean_length == 0: return 0.0 - variance = sum((l - mean_length) ** 2 for l in lengths) / len(lengths) - std_dev = variance ** 0.5 + variance = sum((ln - mean_length) ** 2 for ln in lengths) / len(lengths) + std_dev = variance**0.5 # Coefficient of variation cv = std_dev / mean_length - return round(cv, 4) \ No newline at end of file + return round(cv, 4) diff --git a/src/utils/ui_contract.py b/src/utils/ui_contract.py index 10ea689..2978fa2 100644 --- a/src/utils/ui_contract.py +++ b/src/utils/ui_contract.py @@ -8,9 +8,7 @@ "Do not use this result as sole evidence in academic or legal decisions.", ] -RESULT_LEVEL_REMINDER = ( - "Interpret this score alongside context, writing history, and human review." -) +RESULT_LEVEL_REMINDER = "Interpret this score alongside context, writing history, and human review." def build_limitations_markdown() -> str: diff --git a/src/utils/visualization.py b/src/utils/visualization.py index 4b989f3..b1df4db 100644 --- a/src/utils/visualization.py +++ b/src/utils/visualization.py @@ -7,14 +7,13 @@ from __future__ import annotations -from typing import Dict, List, Optional, Tuple +from typing import Dict, List -import plotly.graph_objects as go +import matplotlib +import matplotlib.pyplot as plt import plotly.express as px +import plotly.graph_objects as go from plotly.subplots import make_subplots -import matplotlib.pyplot as plt -import matplotlib -import numpy as np from src.config.settings import get_settings from src.models.result import AnalysisResult @@ -51,8 +50,11 @@ def create_word_frequency_chart_plotly( fig = go.Figure() fig.add_annotation( text="No word frequency data available", - xref="paper", yref="paper", - x=0.5, y=0.5, showarrow=False, + xref="paper", + yref="paper", + x=0.5, + y=0.5, + showarrow=False, font=dict(size=16, color="gray"), ) fig.update_layout( @@ -131,9 +133,14 @@ def create_word_frequency_chart_matplotlib( if not word_frequencies: ax.text( - 0.5, 0.5, "No word frequency data available", - transform=ax.transAxes, ha="center", va="center", - fontsize=14, color="gray", + 0.5, + 0.5, + "No word frequency data available", + transform=ax.transAxes, + ha="center", + va="center", + fontsize=14, + color="gray", ) ax.set_title(title) return fig @@ -153,8 +160,10 @@ def create_word_frequency_chart_matplotlib( bar.get_x() + bar.get_width() / 2.0, bar.get_height() + 0.3, str(count), - ha="center", va="bottom", - fontsize=9, fontweight="bold", + ha="center", + va="bottom", + fontsize=9, + fontweight="bold", ) ax.set_title(title, fontsize=14, fontweight="bold", pad=15) @@ -307,8 +316,11 @@ def create_sentence_length_chart(self, sentence_lengths: List[int]) -> go.Figure fig = go.Figure() fig.add_annotation( text="No sentence data available", - xref="paper", yref="paper", - x=0.5, y=0.5, showarrow=False, + xref="paper", + yref="paper", + x=0.5, + y=0.5, + showarrow=False, ) return fig @@ -326,7 +338,8 @@ def create_sentence_length_chart(self, sentence_lengths: List[int]) -> go.Figure name="Distribution", hovertemplate="Length: %{x}
Count: %{y}", ), - row=1, col=1, + row=1, + col=1, ) # Line chart showing trend @@ -340,14 +353,18 @@ def create_sentence_length_chart(self, sentence_lengths: List[int]) -> go.Figure name="Sentence Length", hovertemplate="Sentence %{x}: %{y} words", ), - row=1, col=2, + row=1, + col=2, ) # Add mean line mean_length = sum(sentence_lengths) / len(sentence_lengths) fig.add_hline( - y=mean_length, row=1, col=2, - line_dash="dash", line_color="rgba(255,170,0,0.7)", + y=mean_length, + row=1, + col=2, + line_dash="dash", + line_color="rgba(255,170,0,0.7)", annotation_text=f"Mean: {mean_length:.1f}", annotation_font_color="rgba(255,170,0,0.9)", ) @@ -363,4 +380,4 @@ def create_sentence_length_chart(self, sentence_lengths: List[int]) -> go.Figure fig.update_xaxes(gridcolor="rgba(128,128,128,0.2)", tickfont=dict(color="lightgray")) fig.update_yaxes(gridcolor="rgba(128,128,128,0.2)", tickfont=dict(color="lightgray")) - return fig \ No newline at end of file + return fig diff --git a/tests/__init__.py b/tests/__init__.py index b9473f8..4b7a4f7 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1 @@ -"""Test suite for AI Text Detector.""" \ No newline at end of file +"""Test suite for AI Text Detector.""" diff --git a/tests/conftest.py b/tests/conftest.py index 2261dc5..672e35b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,9 +2,10 @@ Shared test fixtures and configuration. """ -import pytest -import sys import os +import sys + +import pytest # Ensure src is on path sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) @@ -73,6 +74,7 @@ def repetitive_text(): def nltk_analyzer(): """Create an NLTK analyzer instance.""" from src.analyzers.nltk_analyzer import NLTKAnalyzer + return NLTKAnalyzer(ngram_size=3) @@ -80,4 +82,5 @@ def nltk_analyzer(): def text_processor(): """Create a TextProcessor instance.""" from src.utils.text_processing import TextProcessor - return TextProcessor() \ No newline at end of file + + return TextProcessor() diff --git a/tests/test_analyzer_internals.py b/tests/test_analyzer_internals.py new file mode 100644 index 0000000..a04d176 --- /dev/null +++ b/tests/test_analyzer_internals.py @@ -0,0 +1,196 @@ +"""Pure-method tests for analyzer interpretation/explanation/verdict logic. + +None of these load transformer or Brown-corpus models — they exercise the +branchy string/verdict helpers directly. +""" + +import pytest + +from src.analyzers.nltk_analyzer import NLTKAnalyzer +from src.config.settings import ConfidenceLevel, Verdict +from src.models.result import AnalysisResult, TextMetrics + + +@pytest.fixture +def nltk(): + # Construction does NOT build the Brown model (that happens lazily on first + # perplexity computation), so this is fast. + return NLTKAnalyzer(ngram_size=3) + + +# --------------------------------------------------------------------------- # +# BaseAnalyzer._determine_verdict (inherited, exercised via NLTKAnalyzer) +# --------------------------------------------------------------------------- # +class TestVerdictThresholds: + def _result(self, perplexity, burstiness, ld, sv, text_length=600): + r = AnalysisResult( + perplexity=perplexity, + burstiness=burstiness, + lexical_diversity=ld, + sentence_variance=sv, + text_length=text_length, + ) + return r + + def test_strong_ai_signals(self, nltk): + r = self._result(perplexity=10, burstiness=0.05, ld=0.2, sv=0.05) + nltk._determine_verdict(r) + assert r.verdict in (Verdict.AI_GENERATED, Verdict.LIKELY_AI) + assert r.confidence_level in list(ConfidenceLevel) + + def test_strong_human_signals(self, nltk): + r = self._result(perplexity=500, burstiness=0.6, ld=0.9, sv=0.9) + nltk._determine_verdict(r) + assert r.verdict in (Verdict.HUMAN_WRITTEN, Verdict.LIKELY_HUMAN) + + def test_mixed_is_uncertain_ish(self, nltk): + r = self._result(perplexity=150, burstiness=0.28, ld=0.5, sv=0.3) + nltk._determine_verdict(r) + assert r.verdict in list(Verdict) + assert 0.0 <= r.confidence <= 100.0 + + def test_short_text_lowers_reliability(self, nltk): + long_r = self._result(perplexity=10, burstiness=0.05, ld=0.2, sv=0.05, text_length=600) + short_r = self._result(perplexity=10, burstiness=0.05, ld=0.2, sv=0.05, text_length=60) + nltk._determine_verdict(long_r) + nltk._determine_verdict(short_r) + assert short_r.confidence <= long_r.confidence + + +# --------------------------------------------------------------------------- # +# BaseAnalyzer._generate_explanation branches +# --------------------------------------------------------------------------- # +class TestExplanationBranches: + def test_ai_leaning_explanation(self, nltk): + r = AnalysisResult( + perplexity=20, burstiness=0.05, lexical_diversity=0.2, sentence_variance=0.05 + ) + text = nltk._generate_explanation(r) + assert "perplexity" in text.lower() + assert "AI" in text or "ai" in text + + def test_human_leaning_explanation(self, nltk): + r = AnalysisResult( + perplexity=500, burstiness=0.6, lexical_diversity=0.9, sentence_variance=0.9 + ) + text = nltk._generate_explanation(r) + assert "human" in text.lower() + + def test_warnings_included(self, nltk): + r = AnalysisResult( + perplexity=100, burstiness=0.3, lexical_diversity=0.5, sentence_variance=0.3 + ) + r.add_warning("Text is very short.") + text = nltk._generate_explanation(r) + assert "short" in text.lower() + + +# --------------------------------------------------------------------------- # +# analyze() error path +# --------------------------------------------------------------------------- # +class TestInputCap: + def test_long_input_truncated_with_warning(self, nltk): + result = AnalysisResult() + long_text = "x" * (nltk.thresholds.max_input_chars + 500) + capped = nltk._apply_input_cap(long_text, result) + assert len(capped) == nltk.thresholds.max_input_chars + assert any("truncated" in w.lower() for w in result.warnings) + + def test_short_input_untouched(self, nltk): + result = AnalysisResult() + text = "a normal sentence" + assert nltk._apply_input_cap(text, result) == text + assert result.warnings == [] + + +class TestAnalyzeErrorPath: + def test_perform_analysis_failure_is_contained(self, nltk): + def boom(text, result): + raise RuntimeError("synthetic failure") + + nltk._perform_analysis = boom + result = nltk.analyze("This is a sufficiently long piece of text to analyze fully.") + assert result.verdict == Verdict.UNCERTAIN + assert result.confidence == 0.0 + assert any("error" in w.lower() for w in result.warnings) + + +# --------------------------------------------------------------------------- # +# NLTK interpretation helpers (pure) +# --------------------------------------------------------------------------- # +class TestNLTKInterpretations: + def test_perplexity_interpretations_cover_ranges(self, nltk): + outputs = [nltk._interpret_perplexity(v) for v in (10, 45, 100, 200, 5000)] + assert len(set(outputs)) >= 4 # distinct messages across ranges + + def test_burstiness_interpretations(self, nltk): + outputs = [nltk._interpret_burstiness(v) for v in (0.05, 0.15, 0.25, 0.4, 0.6)] + assert len(set(outputs)) >= 4 + + def test_lexical_and_sentence_interpretations(self, nltk): + assert nltk._interpret_lexical_diversity(0.2) + assert nltk._interpret_lexical_diversity(0.9) + assert nltk._interpret_sentence_variance(0.05) + assert nltk._interpret_sentence_variance(0.6) + + +# --------------------------------------------------------------------------- # +# GPT-2 / RoBERTa interpretation helpers (construction loads no model) +# --------------------------------------------------------------------------- # +class TestTransformerInterpretations: + def test_gpt2_interpretations(self): + from src.analyzers.gpt2_analyzer import GPT2Analyzer + + a = GPT2Analyzer() + assert a._interpret_gpt2_perplexity(10) != a._interpret_gpt2_perplexity(1000) + assert a._interpret_entropy(2.0) != a._interpret_entropy(9.0) + assert a._interpret_burstiness(0.05) != a._interpret_burstiness(0.5) + + def test_roberta_interpretations(self): + from src.analyzers.roberta_analyzer import RoBERTaAnalyzer + + a = RoBERTaAnalyzer() + assert a._interpret_roberta_score(0.95) != a._interpret_roberta_score(0.1) + + +# --------------------------------------------------------------------------- # +# Ensemble explanation / interpretation branches (no model load) +# --------------------------------------------------------------------------- # +class TestEnsembleNarrative: + def _ensemble(self): + from src.analyzers.ensemble_analyzer import EnsembleAnalyzer + + return EnsembleAnalyzer() + + def test_interpret_ensemble_score_ranges(self): + a = self._ensemble() + outs = [a._interpret_ensemble_score(v) for v in (0.9, 0.75, 0.62, 0.5, 0.32, 0.1)] + assert len(set(outs)) >= 5 + + def _combined(self, a, gpt2_ai, nltk_ai): + base = AnalysisResult(metrics=TextMetrics(total_words=40, unique_words=30)) + roberta = a._disabled_roberta_result() + gpt2 = AnalysisResult(verdict=Verdict.LIKELY_AI, perplexity=15.0 if gpt2_ai else 80.0) + nltk = AnalysisResult(verdict=Verdict.LIKELY_AI, perplexity=3000.0 if nltk_ai else 800.0) + return a._combine_results(base, roberta, gpt2, nltk), roberta, gpt2, nltk + + def test_all_agree_ai(self): + a = self._ensemble() + combined, rob, g, n = self._combined(a, True, True) + a._determine_verdict(combined) + text = a._generate_ensemble_explanation(combined, rob, g, n) + assert "agree" in text.lower() + + def test_all_agree_human(self): + a = self._ensemble() + combined, rob, g, n = self._combined(a, False, False) + a._determine_verdict(combined) + text = a._generate_ensemble_explanation(combined, rob, g, n) + assert "human" in text.lower() + + def test_mixed_signals(self): + a = self._ensemble() + combined, rob, g, n = self._combined(a, True, False) + a._determine_verdict(combined) + text = a._generate_ensemble_explanation(combined, rob, g, n) + assert "mixed" in text.lower() or "agree" in text.lower() diff --git a/tests/test_benchmark_runner.py b/tests/test_benchmark_runner.py new file mode 100644 index 0000000..ee866d0 --- /dev/null +++ b/tests/test_benchmark_runner.py @@ -0,0 +1,99 @@ +"""Tests for the benchmark runner internals (no model loading).""" + +import json + +import pytest + +from src.config.settings import Verdict +from src.evaluation import benchmark as bench +from src.evaluation.dataset import Sample +from src.models.result import AnalysisResult + + +class _FakeAnalyzer: + """Returns AI for even-indexed samples, human otherwise (deterministic).""" + + def __init__(self, samples): + self._labels = {s.text: s.label for s in samples} + + def analyze(self, text): + label = self._labels.get(text, 0) + verdict = Verdict.AI_GENERATED if label == 1 else Verdict.HUMAN_WRITTEN + return AnalysisResult(verdict=verdict, confidence=90.0) + + +def _samples(): + return [ + Sample(id="h1", label=0, source="t", text="human one"), + Sample(id="a1", label=1, source="t", text="ai one"), + Sample(id="h2", label=0, source="t", text="human two"), + Sample(id="a2", label=1, source="t", text="ai two"), + ] + + +class TestBuildAnalyzer: + def test_unknown_analyzer_raises(self): + with pytest.raises(ValueError): + bench._build_analyzer("does-not-exist") + + def test_known_names_construct_without_model_load(self): + # NLTK constructs lazily (no Brown build until analyze()). + analyzer = bench._build_analyzer("nltk") + assert analyzer.__class__.__name__ == "NLTKAnalyzer" + + +class TestSummaryAndSerialization: + def test_format_summary_contains_metrics(self): + samples = _samples() + result = bench.run_benchmark(_FakeAnalyzer(samples), samples, analyzer_name="fake") + text = bench._format_summary(result) + assert "fake" in text + assert "Accuracy" in text + assert "AUROC" in text + assert "FPR" in text + + def test_result_to_dict_roundtrips(self): + samples = _samples() + result = bench.run_benchmark(_FakeAnalyzer(samples), samples, analyzer_name="fake") + payload = result.to_dict() + assert payload["analyzer"] == "fake" + assert payload["n_samples"] == 4 + assert len(payload["predictions"]) == 4 + # Must be JSON-serialisable. + json.dumps(payload) + + def test_sample_prediction_rounding(self): + p = bench.SamplePrediction( + id="x", label=1, source="s", ai_probability=0.123456, verdict="AI", confidence=88.888 + ) + d = p.to_dict() + assert d["ai_probability"] == 0.1235 + assert d["confidence"] == 88.89 + + +class TestPlots: + def test_save_plots_writes_files(self, tmp_path): + samples = _samples() + result = bench.run_benchmark(_FakeAnalyzer(samples), samples, analyzer_name="fake") + written = bench.save_plots(result, tmp_path) + # matplotlib is a project dependency; expect ROC + calibration PNGs. + assert len(written) == 2 + for path in written: + assert path.exists() + assert path.suffix == ".png" + + +class TestMainCLI: + def test_main_writes_report_and_plots(self, tmp_path, monkeypatch): + samples = _samples() + monkeypatch.setattr(bench, "load_dataset", lambda path=None: samples) + monkeypatch.setattr(bench, "_build_analyzer", lambda name: _FakeAnalyzer(samples)) + + out = tmp_path / "nested" / "report.json" + plots = tmp_path / "plots" + rc = bench.main(["--analyzer", "nltk", "--output", str(out), "--plots", str(plots)]) + assert rc == 0 + assert out.exists() # parent dir auto-created + data = json.loads(out.read_text(encoding="utf-8")) + assert data["n_samples"] == 4 + assert list(plots.glob("*.png")) diff --git a/tests/test_binoculars_analyzer.py b/tests/test_binoculars_analyzer.py new file mode 100644 index 0000000..869adda --- /dev/null +++ b/tests/test_binoculars_analyzer.py @@ -0,0 +1,90 @@ +"""Tests for the Binoculars cross-perplexity analyzer. + +These mock the two-model computation so no weights are loaded. +""" + +import pytest + +from src.analyzers.binoculars_analyzer import BinocularsAnalyzer +from src.config.settings import Verdict + + +@pytest.fixture +def analyzer(): + return BinocularsAnalyzer() + + +def _patch_score(analyzer, score, observer_ppl=50.0): + analyzer._compute_binoculars = lambda text: (score, observer_ppl) + + +class TestBinocularsContract: + def test_method_name(self, analyzer): + assert "Binoculars" in analyzer.method_name + + def test_low_ratio_is_ai(self, analyzer): + # Ratio well below the midpoint => AI-leaning verdict. + _patch_score(analyzer, analyzer.config.score_midpoint - 0.1) + result = analyzer.analyze("Some sufficiently long text " * 10) + assert result.scores[0].name == "Binoculars AI Score" + assert result.scores[0].value > 0.5 + assert result.verdict in (Verdict.AI_GENERATED, Verdict.LIKELY_AI) + + def test_high_ratio_is_human(self, analyzer): + _patch_score(analyzer, analyzer.config.score_midpoint + 0.1) + result = analyzer.analyze("Some sufficiently long text " * 10) + assert result.scores[0].value < 0.5 + assert result.verdict in (Verdict.HUMAN_WRITTEN, Verdict.LIKELY_HUMAN) + + def test_midpoint_is_uncertain(self, analyzer): + _patch_score(analyzer, analyzer.config.score_midpoint) + result = analyzer.analyze("Some sufficiently long text " * 10) + assert result.scores[0].value == pytest.approx(0.5, abs=1e-6) + assert result.verdict == Verdict.UNCERTAIN + + def test_ratio_score_row_present(self, analyzer): + _patch_score(analyzer, 0.8) + result = analyzer.analyze("Some sufficiently long text " * 10) + names = [s.name for s in result.scores] + assert "Binoculars Ratio" in names + + def test_confidence_bounded(self, analyzer): + _patch_score(analyzer, 0.5) + result = analyzer.analyze("Some sufficiently long text " * 10) + assert 0.0 <= result.confidence <= 100.0 + + def test_empty_text_uncertain(self, analyzer): + result = analyzer.analyze("") + assert result.verdict == Verdict.UNCERTAIN + assert result.confidence == 0.0 + + def test_serialization(self, analyzer): + _patch_score(analyzer, 0.85) + result = analyzer.analyze("Some sufficiently long text " * 10) + payload = result.to_dict() + assert payload["method"] == analyzer.method_name + assert "scores" in payload + + +@pytest.mark.slow +class TestBinocularsRealModels: + """Exercises the real two-model computation (downloads gpt2 + distilgpt2).""" + + def test_compute_binoculars_returns_ratio(self, analyzer): + score, observer_ppl = analyzer._compute_binoculars( + "The mitochondria is the powerhouse of the cell, and cellular " + "respiration produces the energy currency of the organism." + ) + assert score > 0 + assert observer_ppl > 0 + + def test_end_to_end_analyze(self, analyzer): + from src.config.settings import Verdict + + result = analyzer.analyze( + "So anyway I went to the store and forgot my wallet, classic me, " + "had to walk all the way back home in the rain. Great day." + ) + assert result.verdict in list(Verdict) + assert result.scores[0].name == "Binoculars AI Score" + assert 0.0 <= result.scores[0].value <= 1.0 diff --git a/tests/test_calibration.py b/tests/test_calibration.py new file mode 100644 index 0000000..45e8a32 --- /dev/null +++ b/tests/test_calibration.py @@ -0,0 +1,47 @@ +"""Tests for the perplexity -> AI-probability calibration.""" + +import pytest + +from src.analyzers.calibration import logistic_ai_probability + + +class TestLogisticCalibration: + def test_midpoint_maps_to_half(self): + assert logistic_ai_probability(30, midpoint=30, slope=0.15) == pytest.approx(0.5) + + def test_lower_is_ai_direction(self): + # Below the midpoint => more AI-like (higher probability). + low = logistic_ai_probability(15, midpoint=30, slope=0.15, direction="lower_is_ai") + high = logistic_ai_probability(60, midpoint=30, slope=0.15, direction="lower_is_ai") + assert low > 0.5 > high + + def test_higher_is_ai_direction(self): + low = logistic_ai_probability(600, midpoint=1550, slope=0.0015, direction="higher_is_ai") + high = logistic_ai_probability(3000, midpoint=1550, slope=0.0015, direction="higher_is_ai") + assert high > 0.5 > low + + def test_output_bounded(self): + for ppl in (0, 1, 50, 500, 10000): + p = logistic_ai_probability(ppl, midpoint=30, slope=0.5) + assert 0.0 <= p <= 1.0 + + def test_human_typical_gpt2_perplexity_below_half(self): + # The C2 regression: human GPT-2 perplexity (~58) must NOT read as AI. + assert logistic_ai_probability(58, midpoint=30, slope=0.15) < 0.5 + + def test_ai_typical_gpt2_perplexity_above_half(self): + assert logistic_ai_probability(17, midpoint=30, slope=0.15) > 0.5 + + def test_invalid_slope(self): + with pytest.raises(ValueError): + logistic_ai_probability(30, midpoint=30, slope=0.0) + + def test_invalid_direction(self): + with pytest.raises(ValueError): + logistic_ai_probability(30, midpoint=30, slope=0.1, direction="sideways") + + def test_numerical_stability_extremes(self): + # lower_is_ai: perplexity far above midpoint => ~0 (human). + assert logistic_ai_probability(1e6, midpoint=30, slope=1.0) == pytest.approx(0.0) + # lower_is_ai: perplexity far below midpoint => ~1 (AI). + assert logistic_ai_probability(0, midpoint=1e6, slope=1.0) == pytest.approx(1.0) diff --git a/tests/test_dataset_loader.py b/tests/test_dataset_loader.py new file mode 100644 index 0000000..9654eec --- /dev/null +++ b/tests/test_dataset_loader.py @@ -0,0 +1,64 @@ +"""Tests for the benchmark dataset loader error handling.""" + +import pytest + +from src.evaluation.dataset import Sample, load_dataset + + +def _write(tmp_path, text): + p = tmp_path / "data.jsonl" + p.write_text(text, encoding="utf-8") + return p + + +class TestDatasetLoader: + def test_missing_file_raises(self, tmp_path): + with pytest.raises(FileNotFoundError): + load_dataset(tmp_path / "nope.jsonl") + + def test_valid_records(self, tmp_path): + p = _write( + tmp_path, + '{"id": "h1", "label": "human", "source": "s", "text": "hello world"}\n' + '{"id": "a1", "label": "ai", "source": "s", "text": "generated text"}\n', + ) + samples = load_dataset(p) + assert [s.label for s in samples] == [0, 1] + assert samples[1].is_ai is True + assert samples[0].is_ai is False + + def test_blank_lines_skipped(self, tmp_path): + p = _write( + tmp_path, + '\n{"id": "h1", "label": "human", "text": "hi there"}\n\n', + ) + assert len(load_dataset(p)) == 1 + + def test_invalid_json_raises(self, tmp_path): + p = _write(tmp_path, "{not valid json}\n") + with pytest.raises(ValueError): + load_dataset(p) + + def test_unknown_label_raises(self, tmp_path): + p = _write(tmp_path, '{"id": "x", "label": "robot", "text": "hi"}\n') + with pytest.raises(ValueError): + load_dataset(p) + + def test_empty_text_raises(self, tmp_path): + p = _write(tmp_path, '{"id": "x", "label": "ai", "text": " "}\n') + with pytest.raises(ValueError): + load_dataset(p) + + def test_all_blank_raises(self, tmp_path): + p = _write(tmp_path, "\n\n\n") + with pytest.raises(ValueError): + load_dataset(p) + + def test_default_id_when_missing(self, tmp_path): + p = _write(tmp_path, '{"label": "human", "text": "no id here"}\n') + samples = load_dataset(p) + assert samples[0].id == "sample-1" + + def test_sample_dataclass(self): + s = Sample(id="i", label=1, source="s", text="t") + assert s.is_ai diff --git a/tests/test_ensemble_analyzer.py b/tests/test_ensemble_analyzer.py index a637cc8..bfff047 100644 --- a/tests/test_ensemble_analyzer.py +++ b/tests/test_ensemble_analyzer.py @@ -40,23 +40,25 @@ def _mocked_sub_results(): ) ) + # AI-typical perplexities under the calibrated mapping (GPT-2 ~15, Brown + # ~3000) so the fused verdict reflects real detector behaviour. gpt2 = AnalysisResult( verdict=Verdict.LIKELY_AI, confidence=76.0, - perplexity=80.0, + perplexity=15.0, burstiness=0.15, sentence_variance=0.22, ) gpt2.add_score(DetectionScore(name="GPT-2 Perplexity", value=0.84, indicates_ai=True)) nltk = AnalysisResult( - verdict=Verdict.LIKELY_HUMAN, + verdict=Verdict.LIKELY_AI, confidence=64.0, - perplexity=210.0, + perplexity=3000.0, burstiness=0.42, sentence_variance=0.37, ) - nltk.add_score(DetectionScore(name="Perplexity", value=0.58, indicates_ai=False)) + nltk.add_score(DetectionScore(name="Perplexity", value=0.58, indicates_ai=True)) return roberta, gpt2, nltk @@ -76,8 +78,8 @@ def test_initialization(self, ensemble_analyzer): assert ensemble_analyzer is not None assert ensemble_analyzer.method_name == "Ensemble (GPT2+NLTK)" assert ensemble_analyzer.weights["roberta"] == 0.0 - assert ensemble_analyzer.weights["gpt2"] == 0.65 - assert ensemble_analyzer.weights["nltk"] == 0.35 + assert ensemble_analyzer.weights["gpt2"] == 0.75 + assert ensemble_analyzer.weights["nltk"] == 0.25 def test_analyzer_lazy_loading(self, ensemble_analyzer): """Test that individual analyzers are lazy-loaded.""" @@ -104,7 +106,9 @@ def test_short_text(self, ensemble_analyzer, short_text): assert result is not None assert len(result.warnings) > 0 assert any( - "short" in warning.lower() or "accuracy" in warning.lower() or "characters" in warning.lower() + "short" in warning.lower() + or "accuracy" in warning.lower() + or "characters" in warning.lower() for warning in result.warnings ) diff --git a/tests/test_ensemble_streamlit_contract.py b/tests/test_ensemble_streamlit_contract.py index 5446219..789f75d 100644 --- a/tests/test_ensemble_streamlit_contract.py +++ b/tests/test_ensemble_streamlit_contract.py @@ -2,7 +2,6 @@ from pathlib import Path - PROJECT_ROOT = Path(__file__).resolve().parent.parent @@ -26,7 +25,12 @@ def test_ensemble_uses_shared_ui_contract_helpers() -> None: def test_ensemble_contains_launch_hint_and_probabilistic_warning() -> None: content = _read("ensemble.py") assert "streamlit run ensemble.py" in content - assert "probabilistic" in content.lower() + # The probabilistic-results disclaimer is rendered via the shared footer + # component (src.ui.components.render_footer), so assert the app uses it and + # that the shared component carries the warning. + assert "render_footer(" in content + shared_footer = _read("src/ui/components.py") + assert "probabilistic" in shared_footer.lower() def test_ensemble_keeps_analyzer_comparison_section() -> None: diff --git a/tests/test_ensemble_weighted_fusion.py b/tests/test_ensemble_weighted_fusion.py index 9aacef0..dbbdcb3 100644 --- a/tests/test_ensemble_weighted_fusion.py +++ b/tests/test_ensemble_weighted_fusion.py @@ -1,12 +1,14 @@ -"""Deterministic weighted-fusion tests for EnsembleAnalyzer.""" +"""Deterministic calibrated-fusion tests for EnsembleAnalyzer.""" +from src.analyzers.calibration import logistic_ai_probability from src.analyzers.ensemble_analyzer import EnsembleAnalyzer -from src.config.settings import Verdict +from src.config.settings import Verdict, get_settings from src.models.result import AnalysisResult, DetectionScore, TextMetrics -def test_combine_results_uses_documented_default_weights(): +def test_combine_results_uses_calibrated_logistic_fusion(): analyzer = EnsembleAnalyzer() + cfg = get_settings().ensemble base_result = AnalysisResult(metrics=TextMetrics(total_words=10, unique_words=7)) @@ -26,17 +28,93 @@ def test_combine_results_uses_documented_default_weights(): nltk_result = AnalysisResult( verdict=Verdict.LIKELY_HUMAN, confidence=60.0, - perplexity=200.0, + perplexity=2000.0, burstiness=0.4, sentence_variance=0.5, ) combined = analyzer._combine_results(base_result, roberta_result, gpt2_result, nltk_result) - gpt2_ai_score = max(0, min(1, 1 - (100.0 / 500))) - nltk_ai_score = max(0, min(1, 1 - (200.0 / 500))) - expected = (0.0 * 0.8) + (0.65 * gpt2_ai_score) + (0.35 * nltk_ai_score) + gpt2_ai = logistic_ai_probability( + 100.0, midpoint=cfg.gpt2_ppl_midpoint, slope=cfg.gpt2_ppl_slope, direction="lower_is_ai" + ) + nltk_ai = logistic_ai_probability( + 2000.0, midpoint=cfg.nltk_ppl_midpoint, slope=cfg.nltk_ppl_slope, direction="higher_is_ai" + ) + # RoBERTa weight is 0, so it drops out of the blend. + expected = (cfg.weight_gpt2 * gpt2_ai) + (cfg.weight_nltk * nltk_ai) assert combined.scores assert combined.scores[0].name == "Ensemble AI Score" assert abs(combined.scores[0].value - expected) < 1e-6 + + +def test_human_scale_perplexity_is_not_flagged_ai(): + """Regression for the C2 bias: human-typical GPT-2 perplexity must map below 0.5.""" + analyzer = EnsembleAnalyzer() + + base_result = AnalysisResult(metrics=TextMetrics(total_words=40, unique_words=30)) + roberta_result = analyzer._disabled_roberta_result() + # Human-typical perplexities from the benchmark (GPT-2 ~58, Brown ~1200). + gpt2_result = AnalysisResult(verdict=Verdict.LIKELY_HUMAN, perplexity=58.0) + nltk_result = AnalysisResult(verdict=Verdict.LIKELY_HUMAN, perplexity=1200.0) + + combined = analyzer._combine_results(base_result, roberta_result, gpt2_result, nltk_result) + ensemble_score = combined.scores[0].value + + # Under the old `1 - ppl/500` map this was ~0.88 (flagged AI). It must now + # sit on the human side of the 0.5 boundary. + assert ensemble_score < 0.5 + + analyzer._determine_verdict(combined) + assert combined.verdict in (Verdict.HUMAN_WRITTEN, Verdict.LIKELY_HUMAN, Verdict.UNCERTAIN) + + +def test_binoculars_off_by_default_no_row_no_effect(): + analyzer = EnsembleAnalyzer() + assert analyzer.weights["binoculars"] == 0.0 + + base_result = AnalysisResult(metrics=TextMetrics(total_words=40, unique_words=30)) + roberta_result = analyzer._disabled_roberta_result() + gpt2_result = AnalysisResult(verdict=Verdict.LIKELY_AI, perplexity=58.0) + nltk_result = AnalysisResult(verdict=Verdict.LIKELY_HUMAN, perplexity=1200.0) + + # binoculars_ai defaults to None -> no term, no row. + combined = analyzer._combine_results(base_result, roberta_result, gpt2_result, nltk_result) + names = [s.name for s in combined.scores] + assert "Binoculars Score" not in names + + +def test_binoculars_contributes_when_weighted(): + analyzer = EnsembleAnalyzer() + # Enable Binoculars and rebalance so weights still sum to 1. + analyzer.weights = {"roberta": 0.0, "gpt2": 0.5, "nltk": 0.2, "binoculars": 0.3} + + base_result = AnalysisResult(metrics=TextMetrics(total_words=40, unique_words=30)) + roberta_result = analyzer._disabled_roberta_result() + gpt2_result = AnalysisResult(verdict=Verdict.LIKELY_HUMAN, perplexity=58.0) + nltk_result = AnalysisResult(verdict=Verdict.LIKELY_HUMAN, perplexity=1200.0) + + combined = analyzer._combine_results( + base_result, roberta_result, gpt2_result, nltk_result, binoculars_ai=0.9 + ) + names = [s.name for s in combined.scores] + assert "Binoculars Score" in names + # The strong AI binoculars signal (0.9) at 0.3 weight lifts the ensemble + # score above what GPT-2+NLTK (both human-leaning here) would give alone. + ensemble_score = combined.scores[0].value + assert ensemble_score >= 0.3 * 0.9 + + +def test_disabled_roberta_excluded_from_agreement(): + analyzer = EnsembleAnalyzer() + base_result = AnalysisResult(metrics=TextMetrics(total_words=40, unique_words=30)) + roberta_result = analyzer._disabled_roberta_result() + gpt2_result = AnalysisResult(verdict=Verdict.LIKELY_AI, perplexity=15.0) + nltk_result = AnalysisResult(verdict=Verdict.LIKELY_AI, perplexity=3000.0) + + combined = analyzer._combine_results(base_result, roberta_result, gpt2_result, nltk_result) + # RoBERTa row carries weight 0 so it is not a voter. + voters = [s for s in combined.scores[1:] if s.weight > 0] + assert all("RoBERTa" not in v.name for v in voters) + assert len(voters) == 2 # GPT-2 + NLTK diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py new file mode 100644 index 0000000..75abc9d --- /dev/null +++ b/tests/test_evaluation.py @@ -0,0 +1,154 @@ +"""Tests for the evaluation/benchmark layer (fast, no model loading).""" + +import pytest + +from src.config.settings import Verdict +from src.evaluation import metrics +from src.evaluation.benchmark import result_to_ai_probability, run_benchmark +from src.evaluation.dataset import load_dataset +from src.models.result import AnalysisResult, DetectionScore + + +# --------------------------------------------------------------------------- # +# Metrics +# --------------------------------------------------------------------------- # +class TestMetrics: + def test_perfect_classifier(self): + labels = [0, 0, 1, 1] + scores = [0.1, 0.2, 0.8, 0.9] + rep = metrics.binary_report(labels, scores) + assert rep.accuracy == 1.0 + assert rep.precision == 1.0 + assert rep.recall == 1.0 + assert rep.f1 == 1.0 + assert rep.roc_auc == 1.0 + assert rep.false_positive_rate == 0.0 + assert rep.false_negative_rate == 0.0 + + def test_inverted_classifier_auroc(self): + labels = [0, 0, 1, 1] + scores = [0.9, 0.8, 0.2, 0.1] # perfectly wrong + assert metrics.roc_auc(labels, scores) == 0.0 + + def test_chance_auroc_with_ties(self): + labels = [0, 1, 0, 1] + scores = [0.5, 0.5, 0.5, 0.5] + assert metrics.roc_auc(labels, scores) == pytest.approx(0.5) + + def test_confusion_counts(self): + labels = [1, 1, 0, 0] + scores = [0.9, 0.4, 0.6, 0.1] # threshold 0.5 + tp, fp, tn, fn = metrics.confusion_counts(labels, scores, threshold=0.5) + assert (tp, fp, tn, fn) == (1, 1, 1, 1) + + def test_false_positive_rate_definition(self): + # All human, half wrongly flagged as AI -> FPR 0.5 + labels = [0, 0, 0, 0] + scores = [0.9, 0.9, 0.1, 0.1] + rep = metrics.binary_report(labels, scores) + assert rep.false_positive_rate == 0.5 + assert rep.n_positive == 0 + + def test_expected_calibration_error_perfect(self): + # Predicted probs exactly match observed frequencies within bins. + labels = [0, 0, 1, 1] + scores = [0.0, 0.0, 1.0, 1.0] + assert metrics.expected_calibration_error(labels, scores) == pytest.approx(0.0) + + def test_ece_detects_miscalibration(self): + labels = [0, 0, 0, 0] + scores = [0.9, 0.9, 0.9, 0.9] # confident but always wrong + assert metrics.expected_calibration_error(labels, scores) == pytest.approx(0.9) + + def test_best_threshold_by_f1(self): + labels = [0, 0, 1, 1] + scores = [0.2, 0.3, 0.6, 0.7] + t, f1 = metrics.best_threshold_by_f1(labels, scores) + assert f1 == 1.0 + assert 0.3 < t <= 0.6 + + def test_roc_curve_monotone(self): + labels = [0, 1, 0, 1, 0, 1] + scores = [0.2, 0.8, 0.4, 0.6, 0.1, 0.9] + fpr, tpr, _ = metrics.roc_curve(labels, scores) + assert fpr[0] == 0.0 and tpr[0] == 0.0 + assert fpr[-1] == pytest.approx(1.0) + assert tpr[-1] == pytest.approx(1.0) + assert all(fpr[i] <= fpr[i + 1] + 1e-9 for i in range(len(fpr) - 1)) + + def test_rejects_out_of_range_scores(self): + with pytest.raises(ValueError): + metrics.binary_report([0, 1], [0.5, 1.5]) + + def test_rejects_bad_labels(self): + with pytest.raises(ValueError): + metrics.binary_report([0, 2], [0.5, 0.5]) + + def test_rejects_empty(self): + with pytest.raises(ValueError): + metrics.binary_report([], []) + + +# --------------------------------------------------------------------------- # +# Probability extraction +# --------------------------------------------------------------------------- # +class TestAiProbability: + def test_prefers_ensemble_score(self): + result = AnalysisResult(verdict=Verdict.LIKELY_HUMAN, confidence=90) + result.add_score(DetectionScore(name="Ensemble AI Score", value=0.73)) + assert result_to_ai_probability(result) == pytest.approx(0.73) + + def test_ai_verdict_maps_above_half(self): + result = AnalysisResult(verdict=Verdict.AI_GENERATED, confidence=80) + assert result_to_ai_probability(result) == pytest.approx(0.9) + + def test_human_verdict_maps_below_half(self): + result = AnalysisResult(verdict=Verdict.HUMAN_WRITTEN, confidence=80) + assert result_to_ai_probability(result) == pytest.approx(0.1) + + def test_uncertain_maps_to_half(self): + result = AnalysisResult(verdict=Verdict.UNCERTAIN, confidence=0) + assert result_to_ai_probability(result) == pytest.approx(0.5) + + def test_probability_bounds(self): + result = AnalysisResult(verdict=Verdict.AI_GENERATED, confidence=100) + p = result_to_ai_probability(result) + assert 0.0 <= p <= 1.0 + + +# --------------------------------------------------------------------------- # +# Dataset + runner +# --------------------------------------------------------------------------- # +class TestDatasetAndRunner: + def test_bundled_dataset_loads(self): + samples = load_dataset() + assert len(samples) >= 20 + assert {s.label for s in samples} == {0, 1} + assert all(s.text.strip() for s in samples) + + def test_dataset_balanced_enough(self): + samples = load_dataset() + n_ai = sum(s.label for s in samples) + n_human = len(samples) - n_ai + assert n_ai > 0 and n_human > 0 + + def test_run_benchmark_with_fake_analyzer(self): + class _FakeAnalyzer: + """Oracle: returns the ground truth so the harness math is testable.""" + + def __init__(self, samples): + self._by_text = {s.text: s.label for s in samples} + + def analyze(self, text): + label = self._by_text[text] + verdict = Verdict.AI_GENERATED if label == 1 else Verdict.HUMAN_WRITTEN + return AnalysisResult(verdict=verdict, confidence=90.0) + + samples = load_dataset() + result = run_benchmark(_FakeAnalyzer(samples), samples, analyzer_name="oracle") + assert result.n_samples == len(samples) + assert result.report_default.accuracy == 1.0 + assert result.report_default.roc_auc == 1.0 + d = result.to_dict() + assert d["analyzer"] == "oracle" + assert len(d["predictions"]) == len(samples) diff --git a/tests/test_gpt2_analyzer.py b/tests/test_gpt2_analyzer.py index de83949..8b34be4 100644 --- a/tests/test_gpt2_analyzer.py +++ b/tests/test_gpt2_analyzer.py @@ -6,21 +6,19 @@ """ import pytest + from src.analyzers.gpt2_analyzer import GPT2Analyzer from src.config.settings import ConfidenceLevel, Verdict - # Skip all tests in this module if torch is not available or in CI try: - import torch + import torch # noqa: F401 + HAS_TORCH = True except ImportError: HAS_TORCH = False -pytestmark = pytest.mark.skipif( - not HAS_TORCH, - reason="PyTorch not available" -) +pytestmark = pytest.mark.skipif(not HAS_TORCH, reason="PyTorch not available") class TestGPT2AnalyzerInit: @@ -103,4 +101,4 @@ def test_analyze_to_dict_contract(self, sample_ai_text): "scores", } - assert expected_keys.issubset(set(payload.keys())) \ No newline at end of file + assert expected_keys.issubset(set(payload.keys())) diff --git a/tests/test_infra.py b/tests/test_infra.py new file mode 100644 index 0000000..ea579d3 --- /dev/null +++ b/tests/test_infra.py @@ -0,0 +1,46 @@ +"""Tests for logging setup and the lazy analyzer-import machinery (no models).""" + +import logging + +import pytest + +import src.analyzers as analyzers +from src.utils.logging_config import get_logger, setup_logging + + +class TestLoggingConfig: + def test_setup_logging_console(self): + setup_logging("INFO") + logger = get_logger("test.logger") + assert isinstance(logger, logging.Logger) + logger.info("hello") # should not raise + + def test_setup_logging_to_file(self, tmp_path): + log_file = tmp_path / "app.log" + setup_logging("DEBUG", log_file=str(log_file)) + get_logger("test.file").debug("written") + assert log_file.exists() + + def test_get_logger_returns_named_logger(self): + assert get_logger("abc").name == "abc" + + +class TestLazyAnalyzerImports: + def test_lazy_class_is_importable(self): + # Accessing a heavy analyzer name triggers __getattr__ and returns the + # class object without instantiating (no torch model load). + cls = analyzers.GPT2Analyzer + assert cls.__name__ == "GPT2Analyzer" + + def test_all_lazy_names_resolve(self): + for name in ("GPT2Analyzer", "RoBERTaAnalyzer", "BinocularsAnalyzer", "EnsembleAnalyzer"): + assert getattr(analyzers, name).__name__ == name + + def test_unknown_attribute_raises(self): + with pytest.raises(AttributeError): + _ = analyzers.NotARealAnalyzer + + def test_dir_lists_public_api(self): + names = dir(analyzers) + assert "NLTKAnalyzer" in names + assert "EnsembleAnalyzer" in names diff --git a/tests/test_nltk_analyzer.py b/tests/test_nltk_analyzer.py index 6dc30de..1516711 100644 --- a/tests/test_nltk_analyzer.py +++ b/tests/test_nltk_analyzer.py @@ -3,8 +3,9 @@ """ import pytest + from src.analyzers.nltk_analyzer import NLTKAnalyzer -from src.config.settings import Verdict, ConfidenceLevel +from src.config.settings import ConfidenceLevel, Verdict class TestNLTKAnalyzerInit: @@ -159,4 +160,4 @@ def test_analyze_to_dict_contract(self, sample_ai_text): } assert expected_keys.issubset(set(payload.keys())) - assert result.analysis_time > 0 \ No newline at end of file + assert result.analysis_time > 0 diff --git a/tests/test_nltk_gpt2_streamlit_contract.py b/tests/test_nltk_gpt2_streamlit_contract.py index 8465d1e..646bf9b 100644 --- a/tests/test_nltk_gpt2_streamlit_contract.py +++ b/tests/test_nltk_gpt2_streamlit_contract.py @@ -2,7 +2,6 @@ from pathlib import Path - PROJECT_ROOT = Path(__file__).resolve().parent.parent @@ -17,14 +16,14 @@ def test_app_py_contains_mode_guidance_and_launch_hint() -> None: def test_test_py_contains_mode_guidance_and_launch_hint() -> None: - content = _read("test.py") - assert "streamlit run test.py" in content + content = _read("gpt2_app.py") + assert "streamlit run gpt2_app.py" in content assert "build_mode_guidance_markdown(" in content def test_both_files_contain_shared_limitations_and_reminder_calls() -> None: app_content = _read("app.py") - gpt_content = _read("test.py") + gpt_content = _read("gpt2_app.py") assert "build_limitations_markdown()" in app_content assert "build_limitations_markdown()" in gpt_content assert "build_result_reminder_markdown()" in app_content @@ -33,7 +32,7 @@ def test_both_files_contain_shared_limitations_and_reminder_calls() -> None: def test_entrypoints_keep_page_config_and_analyze_buttons() -> None: app_content = _read("app.py") - gpt_content = _read("test.py") + gpt_content = _read("gpt2_app.py") assert "st.set_page_config(" in app_content assert "st.set_page_config(" in gpt_content assert "Analyze Text" in app_content diff --git a/tests/test_result_model.py b/tests/test_result_model.py index 7837020..1186646 100644 --- a/tests/test_result_model.py +++ b/tests/test_result_model.py @@ -3,9 +3,9 @@ """ import json -import pytest -from src.models.result import AnalysisResult, TextMetrics, DetectionScore -from src.config.settings import Verdict, ConfidenceLevel + +from src.config.settings import ConfidenceLevel, Verdict +from src.models.result import AnalysisResult, DetectionScore, TextMetrics class TestTextMetrics: @@ -190,4 +190,4 @@ def test_to_dict_canonical_keys(self): } ) - assert frozenset(payload.keys()) == expected_keys \ No newline at end of file + assert frozenset(payload.keys()) == expected_keys diff --git a/tests/test_roberta_analyzer.py b/tests/test_roberta_analyzer.py index 52dac1e..dfdc8f3 100644 --- a/tests/test_roberta_analyzer.py +++ b/tests/test_roberta_analyzer.py @@ -3,11 +3,13 @@ """ import pytest + from src.analyzers.roberta_analyzer import RoBERTaAnalyzer from src.config.settings import Verdict try: - import torch + import torch # noqa: F401 + TORCH_AVAILABLE = True except ImportError: TORCH_AVAILABLE = False @@ -41,7 +43,7 @@ def test_model_lazy_loading(self, roberta_analyzer): # Model should be None before first use assert roberta_analyzer._model is None assert roberta_analyzer._tokenizer is None - + # Access model property to trigger loading model = roberta_analyzer.model assert model is not None @@ -51,16 +53,16 @@ def test_tokenizer_lazy_loading(self, roberta_analyzer): """Test that tokenizer is lazy-loaded.""" tokenizer = roberta_analyzer.tokenizer assert tokenizer is not None - assert hasattr(tokenizer, 'encode') + assert hasattr(tokenizer, "encode") def test_analyze_returns_result(self, roberta_analyzer, medium_text): """Test that analyze returns a valid result.""" result = roberta_analyzer.analyze(medium_text) - + assert result is not None - assert hasattr(result, 'verdict') - assert hasattr(result, 'confidence') - assert hasattr(result, 'scores') + assert hasattr(result, "verdict") + assert hasattr(result, "confidence") + assert hasattr(result, "scores") assert result.method == "RoBERTa Transformer" def test_empty_text_handling(self, roberta_analyzer): @@ -77,47 +79,50 @@ def test_short_text_handling(self, roberta_analyzer, short_text): def test_ai_text_analysis(self, roberta_analyzer, sample_ai_text): """Test analysis of AI-generated text.""" result = roberta_analyzer.analyze(sample_ai_text) - + assert result is not None assert result.verdict in [Verdict.AI_GENERATED, Verdict.LIKELY_AI, Verdict.UNCERTAIN] def test_human_text_analysis(self, roberta_analyzer, sample_human_text): """Test analysis of human-written text.""" result = roberta_analyzer.analyze(sample_human_text) - + assert result is not None # Note: Untrained RoBERTa gives random predictions (~50% accuracy) # Accept any verdict since model requires fine-tuning for accurate results assert result.verdict in [ - Verdict.AI_GENERATED, Verdict.LIKELY_AI, Verdict.UNCERTAIN, - Verdict.LIKELY_HUMAN, Verdict.HUMAN_WRITTEN + Verdict.AI_GENERATED, + Verdict.LIKELY_AI, + Verdict.UNCERTAIN, + Verdict.LIKELY_HUMAN, + Verdict.HUMAN_WRITTEN, ] def test_scores_structure(self, roberta_analyzer, medium_text): """Test that scores have proper structure.""" result = roberta_analyzer.analyze(medium_text) - + assert len(result.scores) > 0 - + # Check that scores have required attributes for score in result.scores: - assert hasattr(score, 'name') - assert hasattr(score, 'value') - assert hasattr(score, 'weight') - assert hasattr(score, 'interpretation') - assert hasattr(score, 'indicates_ai') + assert hasattr(score, "name") + assert hasattr(score, "value") + assert hasattr(score, "weight") + assert hasattr(score, "interpretation") + assert hasattr(score, "indicates_ai") def test_roberta_score_present(self, roberta_analyzer, medium_text): """Test that RoBERTa AI score is present in results.""" result = roberta_analyzer.analyze(medium_text) - + score_names = [s.name for s in result.scores] assert any("RoBERTa" in name for name in score_names) def test_metrics_computed(self, roberta_analyzer, medium_text): """Test that all metrics are computed.""" result = roberta_analyzer.analyze(medium_text) - + assert result.burstiness >= 0 assert result.lexical_diversity >= 0 assert result.sentence_variance >= 0 @@ -126,14 +131,14 @@ def test_metrics_computed(self, roberta_analyzer, medium_text): def test_confidence_in_range(self, roberta_analyzer, medium_text): """Test that confidence is in valid range.""" result = roberta_analyzer.analyze(medium_text) - + assert result.confidence >= 0.0 assert result.confidence <= 100.0 def test_analysis_time_recorded(self, roberta_analyzer, medium_text): """Test that analysis time is recorded.""" result = roberta_analyzer.analyze(medium_text) - + assert result.analysis_time > 0 assert result.analysis_time < 60 # Should complete in under 60 seconds @@ -149,7 +154,7 @@ def test_long_text_handling(self, roberta_analyzer): """Test handling of text longer than max_length.""" long_text = "This is a sentence. " * 100 # Create text longer than 512 tokens result = roberta_analyzer.analyze(long_text) - + assert result is not None assert result.verdict is not None @@ -162,7 +167,7 @@ def test_result_serialization(self, roberta_analyzer, medium_text): """Test that result can be serialized to dict.""" result = roberta_analyzer.analyze(medium_text) result_dict = result.to_dict() - + assert isinstance(result_dict, dict) assert "verdict" in result_dict assert "confidence" in result_dict @@ -172,10 +177,10 @@ def test_result_serialization(self, roberta_analyzer, medium_text): def test_consistent_results(self, roberta_analyzer): """Test that same text produces consistent results.""" text = "This is a test for consistency." - + result1 = roberta_analyzer.analyze(text) result2 = roberta_analyzer.analyze(text) - + # Should produce same verdict assert result1.verdict == result2.verdict # Confidence should be very close diff --git a/tests/test_streamlit_apps.py b/tests/test_streamlit_apps.py new file mode 100644 index 0000000..72d6767 --- /dev/null +++ b/tests/test_streamlit_apps.py @@ -0,0 +1,41 @@ +"""Smoke tests for the Streamlit entry points. + +These render each app headlessly with Streamlit's AppTest harness and assert the +script runs without raising. Model loading is gated behind the Analyze button, +so a bare run does not download or build any model (fast). + +They exist primarily as a safety net for the shared-UI refactor: if extracting +common components breaks an app's layout or imports, these fail immediately. +""" + +from pathlib import Path + +import pytest +from streamlit.testing.v1 import AppTest + +_ROOT = Path(__file__).resolve().parents[1] +_APPS = ["app.py", "gpt2_app.py", "ensemble.py"] + + +@pytest.mark.parametrize("app_file", _APPS) +def test_app_renders_without_exception(app_file): + at = AppTest.from_file(str(_ROOT / app_file), default_timeout=60) + at.run() + assert not at.exception, f"{app_file} raised: {at.exception}" + + +@pytest.mark.parametrize("app_file", _APPS) +def test_app_has_sidebar_and_body(app_file): + at = AppTest.from_file(str(_ROOT / app_file), default_timeout=60) + at.run() + # Every app renders markdown (headers/CSS) in the main body and sidebar. + assert len(at.markdown) > 0 + assert len(at.sidebar.markdown) > 0 + + +@pytest.mark.parametrize("app_file", _APPS) +def test_app_has_analyze_button(app_file): + at = AppTest.from_file(str(_ROOT / app_file), default_timeout=60) + at.run() + labels = " ".join(b.label for b in at.button).lower() + assert "analyze" in labels diff --git a/tests/test_text_processing.py b/tests/test_text_processing.py index de5e0dc..9829805 100644 --- a/tests/test_text_processing.py +++ b/tests/test_text_processing.py @@ -2,7 +2,6 @@ Tests for text processing utilities. """ -import pytest from src.utils.text_processing import TextProcessor @@ -112,7 +111,7 @@ def test_word_frequencies(self, sample_ai_text): def test_sentence_lengths(self, sample_ai_text): metrics = TextProcessor.compute_metrics(sample_ai_text) assert isinstance(metrics.sentence_lengths, list) - assert all(isinstance(l, int) and l >= 0 for l in metrics.sentence_lengths) + assert all(isinstance(n, int) and n >= 0 for n in metrics.sentence_lengths) class TestBurstiness: @@ -157,7 +156,8 @@ def test_variance_mixed_sentences(self): "Short. " "This is a much longer sentence with many more words in it that goes on and on. " "Medium length here. " - "Another very very very long sentence that contains a large number of different words and phrases." + "Another very very very long sentence that contains a large " + "number of different words and phrases." ) variance = TextProcessor.compute_sentence_variance(text) - assert variance > 0.3 # Should be higher for mixed lengths \ No newline at end of file + assert variance > 0.3 # Should be higher for mixed lengths diff --git a/tests/test_ui_components.py b/tests/test_ui_components.py new file mode 100644 index 0000000..e74c3b5 --- /dev/null +++ b/tests/test_ui_components.py @@ -0,0 +1,38 @@ +"""Tests for the shared UI component helpers (pure mapping functions).""" + +from src.config.settings import Verdict +from src.ui.components import render_error, verdict_css_class, verdict_emoji +from src.ui.styles import BASE_CSS + + +class TestVerdictMappings: + def test_every_verdict_has_a_css_class(self): + for verdict in Verdict: + css = verdict_css_class(verdict) + assert css.startswith("verdict-") + + def test_every_verdict_has_an_emoji(self): + for verdict in Verdict: + assert verdict_emoji(verdict) + + def test_ai_and_human_map_distinctly(self): + assert verdict_css_class(Verdict.AI_GENERATED) == "verdict-ai" + assert verdict_css_class(Verdict.HUMAN_WRITTEN) == "verdict-human" + assert verdict_css_class(Verdict.LIKELY_AI) == "verdict-likely-ai" + assert verdict_css_class(Verdict.LIKELY_HUMAN) == "verdict-likely-human" + assert verdict_css_class(Verdict.UNCERTAIN) == "verdict-uncertain" + + +class TestBaseCss: + def test_base_css_defines_shared_classes(self): + for cls in (".verdict-card", ".metric-card", ".warning-box", ".footer"): + assert cls in BASE_CSS + + +class TestRenderError: + def test_render_error_does_not_raise_or_leak(self, caplog): + # Should log the exception (server-side) and not propagate it. + with caplog.at_level("ERROR"): + render_error(ValueError("secret internal detail")) + # The exception text is logged, not returned/raised. + assert any("secret internal detail" in r.message or r.exc_info for r in caplog.records) diff --git a/tests/test_ui_contract.py b/tests/test_ui_contract.py index 563257e..4e58c16 100644 --- a/tests/test_ui_contract.py +++ b/tests/test_ui_contract.py @@ -30,9 +30,7 @@ def test_build_result_reminder_markdown_contract() -> None: def test_build_mode_guidance_markdown_includes_all_inputs() -> None: - content = build_mode_guidance_markdown( - "NLTK", "streamlit run app.py", "<1s", "<1 GB" - ) + content = build_mode_guidance_markdown("NLTK", "streamlit run app.py", "<1s", "<1 GB") assert "### 🧭 Mode Guidance" in content assert "NLTK" in content assert "streamlit run app.py" in content diff --git a/tests/test_visualization.py b/tests/test_visualization.py new file mode 100644 index 0000000..6167cf1 --- /dev/null +++ b/tests/test_visualization.py @@ -0,0 +1,71 @@ +"""Tests for the ChartGenerator visualization layer (no models loaded).""" + +import plotly.graph_objects as go +import pytest + +from src.models.result import AnalysisResult, DetectionScore, TextMetrics +from src.utils.visualization import ChartGenerator # sets matplotlib Agg backend + + +@pytest.fixture +def chart_gen(): + return ChartGenerator() + + +@pytest.fixture +def sample_result(): + result = AnalysisResult( + perplexity=45.0, + burstiness=0.3, + lexical_diversity=0.6, + sentence_variance=0.4, + confidence=72.0, + ) + result.metrics = TextMetrics( + total_words=120, + unique_words=80, + word_frequencies={"model": 5, "text": 4, "data": 3, "code": 2, "test": 1}, + sentence_lengths=[8, 12, 6, 15, 9], + ) + result.add_score(DetectionScore(name="Perplexity", value=0.7, weight=0.4, indicates_ai=True)) + result.add_score(DetectionScore(name="Burstiness", value=0.3, weight=0.25, indicates_ai=False)) + return result + + +class TestWordFrequencyPlotly: + def test_populated(self, chart_gen): + fig = chart_gen.create_word_frequency_chart_plotly({"a": 5, "b": 3, "c": 1}, top_n=2) + assert isinstance(fig, go.Figure) + + def test_empty(self, chart_gen): + fig = chart_gen.create_word_frequency_chart_plotly({}) + assert isinstance(fig, go.Figure) + + +class TestWordFrequencyMatplotlib: + def test_populated(self, chart_gen): + fig = chart_gen.create_word_frequency_chart_matplotlib({"a": 5, "b": 3}, top_n=2) + assert fig is not None + + def test_empty(self, chart_gen): + fig = chart_gen.create_word_frequency_chart_matplotlib({}) + assert fig is not None + + +class TestResultCharts: + def test_metrics_gauge(self, chart_gen, sample_result): + assert isinstance(chart_gen.create_metrics_gauge(sample_result), go.Figure) + + def test_score_breakdown(self, chart_gen, sample_result): + assert isinstance(chart_gen.create_score_breakdown_chart(sample_result), go.Figure) + + def test_score_breakdown_no_scores(self, chart_gen): + assert isinstance(chart_gen.create_score_breakdown_chart(AnalysisResult()), go.Figure) + + +class TestSentenceLengthChart: + def test_populated(self, chart_gen): + assert isinstance(chart_gen.create_sentence_length_chart([5, 9, 12, 7]), go.Figure) + + def test_empty(self, chart_gen): + assert isinstance(chart_gen.create_sentence_length_chart([]), go.Figure)