diff --git a/.gitignore b/.gitignore index 51f061e5..5103ad94 100644 --- a/.gitignore +++ b/.gitignore @@ -143,6 +143,7 @@ local.env .claude/ experiments/TR118/models/ PRODUCT_SPEC.md +VISION.md experiments/TR118/results/**/*.jsonl experiments/TR116/all_metrics_extracted.json experiments/TR116/results/**/metrics.json diff --git a/CHANGELOG.md b/CHANGELOG.md index b0a0e276..a3068fe8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,86 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.6.0] - 2026-06-25 + +State-of-the-art serving model: the planner now reflects how LLM inference +actually behaves (per the literature - PagedAttention, continuous batching, +prefill/decode disaggregation, goodput/Pareto), not replicas-of-single-stream. + +### Added +- **Continuous-batching throughput.** vLLM/TGI are modelled with per-GPU + continuous batching instead of single-stream replicas: aggregate decode + throughput rises with batch size up to the KV-cache cap, anchored to the + measured/roofline single-stream rate so it stays quant-correct. One GPU can now + replace several Ollama replicas (e.g. a 7B at 3 req/s on a 4090: Ollama 5 GPUs + vs vLLM 1 GPU at batch 8). `Candidate.effective_batch`. +- **Prefill/decode split.** Separate **TTFT** (prefill, compute-bound, from GPU + FP16 TFLOPS) and **TPOT** (decode, bandwidth-bound); end-to-end p95 now includes + prefill. `GPUSpec.fp16_tflops` for all GPUs; `plan --prompt-tokens`. +- **KV-cache-bound max concurrency** per GPU (`max_concurrent_seqs`), the real + concurrency limiter for batched backends. +- **Pareto frontier** (`plan --pareto`): the non-dominated cost/latency/quality + trade-off menu (tags cheapest / fastest / best-quality), not a single pick. +- **Variance-aware queueing** (`plan --workload steady|chatbot|bursty|agent`): + two-moment wait so high-variance/agent workloads inflate the tail and carry a + "validate with a load test" warning - analytical queueing otherwise silently + approves fleets that miss SLOs for heavy-tailed traffic. +- Numerical accuracy tests pinning throughput, the roofline calibration anchor, + the VRAM formula, TTFT, and batching invariants to ground truth (falsifiability). + +### Changed +- **Throughput scales linearly across GPU replicas** (replaced the Amdahl + serial-fraction model, which capped total throughput at ~1.8x regardless of + instance count and rejected models >=7B). Per-GPU batching is modelled + separately (above). +- **`cost_per_1m_tok` no longer understated by the instance count** (uses N-GPU + cost with N-GPU throughput; $/token is invariant in replica count). +- Broader quant support (legacy + i-quants: `Q4_0`, `Q5_1`, `IQ4_XS`, ...) with + effective bits-per-weight, so a model's native quant is costed correctly. +- Docs realigned to the planner product (research guides moved to an archive + section); ASCII-only source. + +### Fixed +- **Activation memory is now O(context), not O(context^2).** The quadratic term + diverged unphysically at long context (~130 GB at 32k for a 3B model), which + spuriously failed the VRAM gate and zeroed `max_concurrent_seqs` (killing + batching) at >=8k context. Flash/paged attention never materialises the + attention matrix, so it scales linearly; coefficient re-pinned to preserve the + calibrated 2k value. (Found by a blind code audit.) +- **`--json` is now valid when piped** for `bench`, `refit`, `compare`, and + `report` (added `highlight=False, soft_wrap=True`, matching the other six + commands). Rich previously reflowed long string values at width 79 and produced + invalid JSON for `... --json | jq`. +- **`refit --validate` is a real gate**: validation runs *before* the write, so + invalid coefficients are no longer persisted ahead of the failing exit. +- **Quality tier is family-aware** for off-registry models (consistent with the + reported quality), so the "concerning drop" advisory can fire instead of the + tier silently collapsing to `unknown`. +- **Per-key confidence weighting in `refit`**: each entry is blended by its own + successful-run count, not the global run total (which over-trusted + lightly-measured configs in a multi-config refit). +- Measured-corpus staleness warning: `plan`/`suggest` now warn (instead of + silently shadowing) when the cached corpus predates the installed version. +- Robust error handling: `measure` surfaces an unknown `--backend` cleanly; + backend `check_model` handles timeouts/HTTP errors (not just connect); the + resolver/discovery raise `ResolverError` (not a raw traceback) on a non-JSON + 200 response; a degenerate `...0b` identifier no longer raises ZeroDivisionError. +- `bench --context ... --quant Q` no longer drops the quant label; non-Ollama + context sweeps warn that the per-request context override was not applied. +- Roofline throughput is bandwidth-correct above FP16 (e.g. FP32 ~ 0.5x FP16); + Ollama `F16`/`F32` native-quant strings are normalised to `FP16`/`FP32`. +- `eval --fp16-baseline` exposes tier classification (previously always + `unknown` from the CLI). Build floor corrected to `setuptools>=77` (PEP 639). + +### Notes +- Per-backend MFU/MBU *calibration* is deferred: the `measure` loop already + supersedes the roofline estimate with real measurements for any benchmarked + model, which is stronger than tuning a global constant. +- Known minor limitations (low impact, deferred): VRAM mixes decimal-GB weight + with binary-GiB KV (~7.4%, conservative); ambiguous partial GPU names (e.g. + "RTX 4080") resolve to the first DB match by VRAM; a genuine 0.0 BERTScore is + treated as "unavailable". + ## [0.5.0] - 2026-06-24 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 99d54447..376e7c24 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ ChimeraForge is an LLM inference benchmarking and deployment planning platform, broken out from the Banterhearts program. It provides quantified, reproducible answers to LLM deployment decisions, backed by ~204,000 real measurements on consumer GPUs. Ships both research artifacts (32 technical reports, TR108-TR137 + TR142/TR146) and production CLI tools (`chimeraforge plan` and `chimeraforge bench`). -**Version:** 0.5.0 | **License:** MIT | **Python:** >=3.10 | **Rust:** >=1.70 +**Version:** 0.6.0 | **License:** MIT | **Python:** >=3.10 | **Rust:** >=1.70 ## Quick Reference @@ -36,7 +36,7 @@ chimeraforge plan --model qwen3:14b --measure # bench live first, then plan (p # Run benchmarks (requires live Ollama) chimeraforge bench --model llama3.2-3b --runs 5 -# Run tests (431 total; model-agnostic adds resolver/discovery/catalog/measure/diagnostics + specs) +# Run tests (476 total; 0.6.0 adds KV-batch/prefill-decode/continuous-batching/variance/pareto/accuracy) pytest tests/ -v # Lint @@ -53,7 +53,7 @@ cd src/rust/demo_multiagent && cargo build --release ``` src/ chimeraforge/ # CLI tool + capacity planner (pip-installable) - __init__.py # Exports __version__ = "0.5.0" + __init__.py # Exports __version__ = "0.6.0" cli.py # Typer entry point, registers plan/suggest/safety/... (lazy imports) commands/ # One module per CLI command (plan.py, suggest.py, safety.py, ...) planner/ @@ -154,6 +154,16 @@ resources/prompts/ # Legacy banter_prompts.txt (not used in b ## Planner Architecture (src/chimeraforge/planner/) +### Serving model (0.6.0) +The planner models LLM serving as the literature describes it, not replicas-of-single-stream: +- **Prefill vs decode:** TTFT = prefill (compute-bound, `2*params*prompt_tokens / (fp16_tflops*MFU)`); TPOT = decode (bandwidth-bound). End-to-end p95 = TTFT + decode + queueing. `GPUSpec.fp16_tflops` drives prefill; `--prompt-tokens` sets input length. +- **Continuous batching:** vLLM/TGI serve B concurrent sequences per GPU; `ThroughputModel.batched_decode_tps()` = `B*bw*MBU / (weight_eff + B*kv_per_seq)`, `weight_eff = bw*MBU/n1_tps` (anchored to measured/roofline single-stream, quant-correct), capped by `max_concurrent_seqs` (KV-bound) and the decode compute ceiling. Ollama = B=1. The engine searches **(N replicas x B batch/GPU)** for the cheapest SLO-feasible config. `BACKEND_CONTINUOUS_BATCHING`, `Candidate.effective_batch`. +- **Replicas scale linearly** (eta=1); the old Amdahl serial-fraction model was wrong for replica fan-out (capped throughput at ~1.8x; rejected >=7B) and is no longer applied. +- **Variance-aware queueing:** two-moment wait `(1+Cs^2)/2 * M/M/1` (`Cs^2=0` reproduces M/D/1). `--workload {steady,chatbot,bursty,agent}` -> `WORKLOAD_CV2`; high variance inflates the tail + warns (analytical queueing silently approves broken fleets for agent traffic otherwise). +- **Pareto output:** `plan --pareto` -> `pareto_frontier()` (non-dominated on cost/p95/quality), the trade-off menu instead of one cost-sorted pick. +- **Cost:** `cost_per_1m_tok` uses N-GPU cost with N-GPU throughput (invariant in replica count). +- Numerical accuracy gates in `tests/test_accuracy.py` pin predictions to ground truth. + The `chimeraforge plan` CLI runs a 4-gate exhaustive search (plus an opt-in 5th safety gate) over (model x quant x backend x N_agents): **Gate 1 — VRAM:** `weight_gb + kv_cache_gb + activations_gb <= hw_vram` diff --git a/README.md b/README.md index a59ff4a7..452756e1 100644 --- a/README.md +++ b/README.md @@ -645,8 +645,8 @@ This research was conducted as part of the Banterhearts LLM Performance Research --- -**Last Updated:** June 25, 2026 (v0.5.0) +**Last Updated:** June 25, 2026 (v0.6.0) **Repository:** https://github.com/Sahil170595/Chimeraforge **PyPI:** https://pypi.org/project/chimeraforge/ -**Status:** Phase 1 + Phase 2 + Phase 3 Complete | v0.5.0 +**Status:** Phase 1 + Phase 2 + Phase 3 Complete | v0.6.0 diff --git a/docs/API.md b/docs/API.md index ef4eb27c..00d81252 100644 --- a/docs/API.md +++ b/docs/API.md @@ -2,7 +2,7 @@ ## Overview -This document covers the public Python API for the `chimeraforge` package (v0.2.1). +This document covers the public Python API for the `chimeraforge` package (v0.6.0). For CLI usage, see the [README](../README.md). Install: `pip install chimeraforge[all]` @@ -164,7 +164,8 @@ scores = evaluate_quality( references=["The capital of France is Paris."], ) print(f"Composite: {scores.composite:.3f}") -print(f"Tier: {classify_tier(scores.composite)}") +# Tier needs the FP16 baseline composite to measure the drop against: +print(f"Tier: {classify_tier(scores.composite, fp16_composite=0.85)}") ``` ### Metrics @@ -174,7 +175,7 @@ print(f"Tier: {classify_tier(scores.composite)}") - **`compute_bert_score(preds, refs)`** — BERTScore F1 (requires `evaluate` + `bert-score`) - **`compute_coherence(preds, refs)`** — length-ratio heuristic - **`compute_composite(scores)`** — weighted: 0.2*EM + 0.3*ROUGE + 0.3*BERT + 0.2*coherence -- **`classify_tier(score)`** — negligible (>=-3pp), acceptable (>=-10pp), concerning, unacceptable +- **`classify_tier(composite, fp16_composite)`** — negligible (>=-3pp), acceptable (>=-10pp), concerning, unacceptable ### Built-in tasks diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b66474eb..9728a446 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,5 +1,11 @@ # Architecture Documentation +> **Scope:** this document covers the **research / agent-benchmarking subsystem** +> (`src/python/banterhearts/`, `src/rust/`) that produced the TR-series data. For +> the **planner CLI** (`src/chimeraforge/` -- the shipping product), see the +> "Planner Architecture" section of the top-level `CLAUDE.md` and +> [Using the Planner](planning.md). + ## Overview Chimeraforge is a benchmarking and research repository focused on LLM performance optimization. The architecture is designed for reproducibility, extensibility, and comprehensive performance analysis. diff --git a/docs/README.md b/docs/README.md index 7ccb0445..13a05bf5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,144 +1,71 @@ # Documentation Index -Comprehensive documentation for Chimeraforge, a benchmarking and research repository for LLM performance optimization. - -## Quick Start - -- **[Quick Start Guide](quick_start.md)** - Get up and running in minutes with your first benchmark -- **[Installation Guide](installation.md)** - OS-specific setup instructions and prerequisites - -## Core Documentation - -### Setup & Configuration -- **[Installation](installation.md)** - Complete installation guide for all platforms -- **[Dual Ollama Setup](dual_ollama_setup.md)** - Setting up dual Ollama instances for true concurrency -- **[Architecture](ARCHITECTURE.md)** - System architecture and design principles - -### Running Benchmarks -- **[Benchmarking Guide](benchmarking.md)** - Comprehensive guide to running all benchmark types -- **[Multi-Agent Guide](multi_agent.md)** - Multi-agent concurrent execution scenarios -- **[Python Agents](python_agents.md)** - Python agent implementation details -- **[Rust Agents](rust_agents.md)** - Rust agent implementation details - -### Optimization & Tuning -- **[Chimera Optimization](chimera_optimization.md)** - Configuration optimization strategies -- **[Performance Tuning](performance_tuning.md)** - Performance tuning techniques and best practices - -### Analysis & Reporting -- **[Statistical Analysis](statistical_analysis.md)** - Statistical rigor, sample sizes, and analysis methods -- **[Rust vs Python](rust_vs_python.md)** - Cross-language performance comparison -- **[Technical Reports](technical_reports.md)** - Complete index of all technical reports (TR108+) -- **[Methodology](methodology.md)** - Research methodology and experimental design - -### Reference -- **[FAQ](faq.md)** - Frequently asked questions and troubleshooting -- **[Benchmarks README](../benchmarks/README.md)** - Directory map for stored benchmark artifacts - -## Project Documentation - -### Contributing -- **[Contributing Guide](../CONTRIBUTING.md)** - How to contribute to the project -- **[Code of Conduct](../CODE_OF_CONDUCT.md)** - Community standards and expectations - -### Project Information -- **[Changelog](../CHANGELOG.md)** - Version history and changes -- **[Security Policy](../SECURITY.md)** - Security reporting and best practices -- **[Architecture](ARCHITECTURE.md)** - System architecture documentation - -## Documentation Structure - -### By User Type - -**New Users**: -1. Start with [Quick Start](quick_start.md) -2. Follow [Installation](installation.md) -3. Run your first benchmark -4. Read [Benchmarking Guide](benchmarking.md) for details - -**Researchers**: -1. Review [Methodology](methodology.md) -2. Study [Technical Reports](technical_reports.md) -3. Understand [Statistical Analysis](statistical_analysis.md) -4. Review [Architecture](ARCHITECTURE.md) - -**Developers**: -1. Read [Architecture](ARCHITECTURE.md) -2. Review [Contributing Guide](../CONTRIBUTING.md) -3. Study agent implementations ([Python](python_agents.md) / [Rust](rust_agents.md)) -4. Understand benchmarking framework - -**Operators**: -1. Follow [Installation](installation.md) -2. Set up [Dual Ollama](dual_ollama_setup.md) -3. Run benchmarks using [Benchmarking Guide](benchmarking.md) -4. Troubleshoot using [FAQ](faq.md) - -### By Topic - -**Performance Optimization**: -- [Chimera Optimization](chimera_optimization.md) -- [Performance Tuning](performance_tuning.md) -- [Rust vs Python](rust_vs_python.md) - -**Multi-Agent Systems**: -- [Multi-Agent Guide](multi_agent.md) -- [Dual Ollama Setup](dual_ollama_setup.md) -- Technical Reports: TR110, TR113, TR114 +ChimeraForge is a model-agnostic **LLM deployment / capacity-planning CLI**: given +your hardware, workload, latency SLO, quality target, and budget, it recommends +the best (model x quantization x backend x GPU-instance-count) configuration -- +backed by real benchmarks, with honest per-prediction provenance. It also bundles +the research harness (agent benchmarking, Rust vs Python, the TR series) that the +planner's data came from; those guides are archived below. + +## The Planner CLI (start here) + +- **[Installation](installation.md)** - install from PyPI, optional extras +- **[Quick Start](quick_start.md)** - first run in minutes +- **[Using the Planner](planning.md)** - `plan` / `suggest` / `measure` / `catalog`, end to end +- **[API Reference](API.md)** - the public Python API (`chimeraforge.planner`, resolver, discovery, measure) + +```bash +pip install chimeraforge +chimeraforge plan --model Qwen/Qwen2.5-7B-Instruct --hardware "RTX 4090 24GB" --request-rate 2 +chimeraforge suggest --source ollama --hardware "RTX 4080 12GB" --budget 500 +chimeraforge measure --model qwen3:14b --ollama-url http://localhost:11434 +``` -**Language-Specific**: -- [Python Agents](python_agents.md) -- [Rust Agents](rust_agents.md) -- [Rust vs Python](rust_vs_python.md) +Ten commands: `plan`, `suggest`, `measure`, `catalog`, `safety`, `bench`, `eval`, +`refit`, `compare`, `report`. Run `chimeraforge --help` or `chimeraforge --help`. -**Research & Analysis**: -- [Methodology](methodology.md) -- [Statistical Analysis](statistical_analysis.md) -- [Technical Reports](technical_reports.md) +## Project Information -## Technical Reports +- **[Changelog](../CHANGELOG.md)** - version history +- **[Contributing](../CONTRIBUTING.md)** | **[Security](../SECURITY.md)** | **[Code of Conduct](../CODE_OF_CONDUCT.md)** -The canonical technical report archive is `outputs/publish_ready/reports/`. +--- -Current coverage: +## Research Archive -- Phase 1: `TR108-TR122` -- Phase 2: `TR123-TR133` -- Phase 3: `TR134-TR137` -- Conclusive synthesis sets: `108-116`, `117-122`, `123-133`, `134-137` +ChimeraForge began as the public breakout of the Banterhearts performance-research +program (~204,000 measurements across TR108-TR137). These guides document that +research -- the agent-benchmarking harness, the language/runtime studies, and the +methodology behind the bundled data. They describe the code under +`src/python/banterhearts/` and `src/rust/`, **not** the planner CLI. -See [Technical Reports](technical_reports.md) for the current linked index. +### Benchmarking & methodology +- **[Benchmarking Guide](benchmarking.md)** - running the benchmark harness +- **[Methodology](methodology.md)** - experimental design, isolation, cold starts +- **[Statistical Analysis](statistical_analysis.md)** - sample sizes, CIs, rigor +- **[Technical Reports](technical_reports.md)** - index of the TR series (TR108+) -## Repository Structure -``` -Chimeraforge/ -├── src/ # Source code (Python & Rust) -├── experiments/ # Research experiments (TR series) -├── data/ # Data files (baselines, CSV, research) -├── outputs/ # Generated outputs (artifacts, reports, runs) -├── benchmarks/ # Benchmark results -├── scripts/ # Utility scripts -├── docs/ # Documentation (this directory) -└── logs/ # Log files -``` +### Agent research (Rust vs Python, multi-agent) +- **[Rust vs Python](rust_vs_python.md)** - cross-language comparison (TR111-TR116) +- **[Rust Agents](rust_agents.md)** / **[Python Agents](python_agents.md)** - implementations +- **[Multi-Agent Guide](multi_agent.md)** - concurrent execution scenarios +- **[Dual Ollama Setup](dual_ollama_setup.md)** - required for the multi-agent experiments -See [Architecture](ARCHITECTURE.md) for detailed structure documentation. +### Optimization & structure +- **[Chimera Optimization](chimera_optimization.md)** - config optimization (TR108) +- **[Performance Tuning](performance_tuning.md)** - tuning techniques +- **[Architecture](ARCHITECTURE.md)** - the agent/research subsystem architecture +- **[Repository Structure](repo_structure.md)** - folder layout & data governance +- **[FAQ](faq.md)** - common questions -## Getting Help +The canonical technical-report archive lives in `outputs/publish_ready/reports/`. -- **FAQ**: Check [FAQ](faq.md) for common questions -- **Issues**: Open an issue on GitHub -- **Discussions**: Use GitHub Discussions -- **Contributing**: See [Contributing Guide](../CONTRIBUTING.md) +--- ## Documentation Standards -- All documentation is ASCII-only for universal compatibility -- Commands assume execution from repository root unless noted -- Code examples are tested and verified -- Links are relative to documentation structure -- Last updated dates are maintained - ---- - -**Last Updated**: January 2025 +- ASCII-only for universal compatibility +- Commands assume execution from the repository root unless noted +- Links are relative to this `docs/` directory +**Last Updated:** June 2026 (v0.5.0) diff --git a/docs/faq.md b/docs/faq.md index e1ce7593..dedb8306 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -217,7 +217,7 @@ See individual technical reports for citation information. ### Can I contribute benchmarks? -Yes! See [Contributing Guide](contributing.md) for details. +Yes! See [Contributing Guide](../CONTRIBUTING.md) for details. ## Still Have Questions? diff --git a/docs/installation.md b/docs/installation.md index c23add4b..b80d4ed1 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -234,7 +234,7 @@ ollama run gemma3:latest "test" # Should use GPU - [Quick Start Guide](quick_start.md) - [Benchmarking Guide](benchmarking.md) -- [Configuration Reference](configuration.md) +- [Using the Planner](planning.md) --- diff --git a/docs/performance_tuning.md b/docs/performance_tuning.md index 34a62365..1ffaafa3 100644 --- a/docs/performance_tuning.md +++ b/docs/performance_tuning.md @@ -305,7 +305,6 @@ async def batch_generate(prompts: List[str]): - [Chimera Optimization Guide](chimera_optimization.md) - [Benchmarking Guide](benchmarking.md) -- [Configuration Reference](configuration.md) - [TR108: Single-Inference Optimization](../outputs/publish_ready/reports/Technical_Report_108.md) --- diff --git a/docs/planning.md b/docs/planning.md new file mode 100644 index 00000000..240231a4 --- /dev/null +++ b/docs/planning.md @@ -0,0 +1,99 @@ +# Using the Planner + +ChimeraForge answers: **"For my hardware, workload, and budget, what should I +actually deploy?"** It searches (model x quantization x backend x GPU-instance +count) and filters through gates -- VRAM, quality, latency, cost, and an opt-in +safety screen -- then ranks the survivors. Every number carries provenance +(`measured` / `estimated` / `unknown`) so you know which to trust. + +## plan -- recommend a configuration + +By registry size class (uses the bundled, measured corpus): + +```bash +chimeraforge plan --model-size 8b --hardware "RTX 4090 24GB" --request-rate 1.0 --budget 300 +``` + +For **any** model (the model-agnostic path) -- a Hugging Face repo, an Ollama +tag, or manual overrides: + +```bash +# HuggingFace repo: pulls real params + attention geometry from config.json +chimeraforge plan --model Qwen/Qwen2.5-7B-Instruct --hardware "RTX 4090 24GB" + +# Ollama tag: pulls GGUF metadata from a live /api/show +chimeraforge plan --model qwen3:14b --ollama-url http://localhost:11434 + +# Air-gapped / unreleased: supply the geometry yourself +chimeraforge plan --model my/model-7b --params-b 7 --n-layers 32 --n-kv-heads 8 --d-head 128 --no-network +``` + +Key inputs (all have explicit units and defaults; see `plan --help`): + +| Flag | Meaning | Default | +|------|---------|---------| +| `--request-rate` | requests/sec | 1.0 | +| `--prompt-tokens` | input length (drives prefill / TTFT) | 512 | +| `--avg-tokens` | output length (drives decode / TPOT) | 128 | +| `--latency-slo` | max p95 end-to-end, ms | 5000 | +| `--quality-target` | min composite quality 0-1 | 0.5 | +| `--budget` | max USD/month | 100 | +| `--safety-target` | min refusal rate 0-1 (opt-in gate) | off | +| `--hardware` | GPU name (`plan --list-hardware`) | RTX 4080 12GB | + +Reading the output: the **Performance** panel reports N=1 throughput, **TTFT** +(prefill, compute-bound), **TPOT** (per output token, bandwidth-bound), end-to-end +p95, and **max concurrent sequences per GPU** (KV-cache bound). A `~` marks an +estimated number. If nothing fits, the planner names the **binding gate** ("Why +nothing fit"). Add `--json` for machine-readable output (clean for CI / `jq`). + +## measure -- plan on real numbers, not estimates + +For an off-registry model the throughput starts as a roofline *estimate*. Benchmark +it once and the planner switches to *measured*: + +```bash +chimeraforge measure --model qwen3:14b --ollama-url http://localhost:11434 +chimeraforge plan --model qwen3:14b --measure # measure, then plan, in one step +``` + +`measure` runs the real `bench` machinery (N=1 throughput, service time, and +concurrency scaling) and folds it into a local corpus +(`~/.cache/chimeraforge/fitted_models.json`); `plan`/`suggest` prefer it +automatically afterward. Quality is deliberately not auto-measured -- it stays +labeled `estimated`/`unknown` rather than faking a benchmark composite. + +## suggest -- discover and rank models + +```bash +chimeraforge suggest --source ollama --hardware "RTX 4090 24GB" --budget 500 +chimeraforge suggest --source hf --hf-limit 8 --hardware "RTX 4080 12GB" +chimeraforge suggest --source catalog --hardware "RTX 4080 12GB" # offline, after `catalog --build` +``` + +Pulls candidate models from your installed Ollama (`/api/tags`), the HF Hub +(top text-generation), and/or the local catalog; resolves each to real specs; +and shows the best config per model. + +## catalog -- a local, offline model set + +```bash +chimeraforge catalog --build # resolve a curated seed (+ --with-ollama), cache specs +chimeraforge catalog # list what is cached +``` + +Once built, `suggest --source catalog` ranks the set with no network. + +## How predictions are made (and their limits) + +- **VRAM / KV-cache:** first-principles from real architecture -- exact for any model. +- **Throughput:** measured lookup when available; otherwise a memory-bandwidth + roofline for decode. Roofline is a best-case estimate; use `measure` for real numbers. +- **Quality:** measured composite (registry) or a family prior (`estimated`) or a + neutral `unknown` -- never a fabricated benchmark. +- **Safety:** lookup-only (TR134/TR142); `unknown` for unscreened models, by design. +- **Cost:** GPU $/hr x instances; `$/1M tokens` is invariant in replica count. + +Network resolution (HF / Ollama metadata) needs the `resolve` extra +(`pip install "chimeraforge[resolve]"`). The `--no-network` flag and manual +overrides keep it fully offline. diff --git a/docs/rust_vs_python.md b/docs/rust_vs_python.md index 2fb4eeb1..6b9f4149 100644 --- a/docs/rust_vs_python.md +++ b/docs/rust_vs_python.md @@ -289,7 +289,6 @@ Comprehensive cross-language performance analysis based on Technical Reports 112 - [TR110: Python Multi-Agent](../outputs/publish_ready/reports/Technical_Report_110.md) - [TR114_v2: Rust Multi-Agent](../outputs/publish_ready/reports/Technical_Report_114_v2.md) - [TR115_v2: Rust Runtime Optimization](../outputs/publish_ready/reports/Technical_Report_115_v2.md) -- [Production Deployment Guide](production_deployment.md) - [Chimera Optimization Guide](chimera_optimization.md) --- diff --git a/pyproject.toml b/pyproject.toml index 583a2773..33a4cbcc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,10 @@ [build-system] -requires = ["setuptools>=68", "wheel"] +requires = ["setuptools>=77", "wheel"] build-backend = "setuptools.build_meta" [project] name = "chimeraforge" -version = "0.5.0" +version = "0.6.0" description = "LLM deployment optimizer (performance, cost, and safety) — backed by ~204,000 real measurements on consumer GPUs" readme = "README.md" requires-python = ">=3.10" diff --git a/src/chimeraforge/__init__.py b/src/chimeraforge/__init__.py index a57075fc..722b970c 100644 --- a/src/chimeraforge/__init__.py +++ b/src/chimeraforge/__init__.py @@ -1,8 +1,8 @@ -"""ChimeraForge — LLM deployment optimizer. +"""ChimeraForge - LLM deployment optimizer. Backed by ~204,000 real measurements across 32 technical reports on consumer GPUs. Plan, benchmark, evaluate, and optimize LLM inference deployments. """ -__version__ = "0.5.0" +__version__ = "0.6.0" diff --git a/src/chimeraforge/bench/__init__.py b/src/chimeraforge/bench/__init__.py index d3e53886..20e5fe71 100644 --- a/src/chimeraforge/bench/__init__.py +++ b/src/chimeraforge/bench/__init__.py @@ -1,4 +1,4 @@ -"""ChimeraForge Benchmarking Engine — run real LLM inference benchmarks. +"""ChimeraForge Benchmarking Engine - run real LLM inference benchmarks. Public API: run_benchmark Run a single benchmark configuration diff --git a/src/chimeraforge/bench/backends/__init__.py b/src/chimeraforge/bench/backends/__init__.py index 18a4ecbf..afa04ff1 100644 --- a/src/chimeraforge/bench/backends/__init__.py +++ b/src/chimeraforge/bench/backends/__init__.py @@ -1,4 +1,4 @@ -"""Backend registry — maps backend names to adapter classes.""" +"""Backend registry - maps backend names to adapter classes.""" from __future__ import annotations diff --git a/src/chimeraforge/bench/backends/ollama.py b/src/chimeraforge/bench/backends/ollama.py index 66667039..49645212 100644 --- a/src/chimeraforge/bench/backends/ollama.py +++ b/src/chimeraforge/bench/backends/ollama.py @@ -59,6 +59,10 @@ async def check_model(self, model: str) -> tuple[bool, str]: return False, f"Model not found. Run: ollama pull {model}" except httpx.ConnectError: return False, f"Ollama not running at {self.base_url}" + except httpx.TimeoutException: + return False, f"Ollama timed out at {self.base_url}" + except httpx.HTTPError as exc: + return False, f"Ollama model check failed at {self.base_url}: {exc}" async def generate( self, diff --git a/src/chimeraforge/bench/backends/tgi.py b/src/chimeraforge/bench/backends/tgi.py index a744bbb4..e0c6ede0 100644 --- a/src/chimeraforge/bench/backends/tgi.py +++ b/src/chimeraforge/bench/backends/tgi.py @@ -73,6 +73,10 @@ async def check_model(self, model: str) -> tuple[bool, str]: ) except httpx.ConnectError: return False, f"TGI not running at {self.base_url}" + except httpx.TimeoutException: + return False, f"TGI timed out at {self.base_url}" + except httpx.HTTPError as exc: + return False, f"TGI model check failed at {self.base_url}: {exc}" async def generate( self, diff --git a/src/chimeraforge/bench/backends/vllm.py b/src/chimeraforge/bench/backends/vllm.py index 4f3a191c..28e012bd 100644 --- a/src/chimeraforge/bench/backends/vllm.py +++ b/src/chimeraforge/bench/backends/vllm.py @@ -66,6 +66,10 @@ async def check_model(self, model: str) -> tuple[bool, str]: ) except httpx.ConnectError: return False, f"vLLM not running at {self.base_url}" + except httpx.TimeoutException: + return False, f"vLLM timed out at {self.base_url}" + except httpx.HTTPError as exc: + return False, f"vLLM model check failed at {self.base_url}: {exc}" async def generate( self, diff --git a/src/chimeraforge/bench/metrics.py b/src/chimeraforge/bench/metrics.py index aa4494e2..801a92ba 100644 --- a/src/chimeraforge/bench/metrics.py +++ b/src/chimeraforge/bench/metrics.py @@ -1,4 +1,4 @@ -"""Benchmark metrics — dataclasses and statistical helpers. +"""Benchmark metrics - dataclasses and statistical helpers. Defines the core data model for benchmark results: individual run metrics, statistical summaries, environment metadata, and the diff --git a/src/chimeraforge/bench/runner.py b/src/chimeraforge/bench/runner.py index b57d0812..f107757a 100644 --- a/src/chimeraforge/bench/runner.py +++ b/src/chimeraforge/bench/runner.py @@ -323,6 +323,14 @@ async def run_context_sweep( context_length=ctx, **kwargs, ) + # Only Ollama accepts a per-request context override (num_ctx); vLLM/TGI + # serve at their startup-configured context, so the labelled ctx was NOT + # applied. Flag it so the differentiated rows aren't read as a ctx effect. + if backend_name != "ollama": + result.warnings.append( + f"context_length={ctx} recorded but NOT applied: the {backend_name} " + "completion API has no per-request context override (set it at server startup)" + ) results.append(result) return results diff --git a/src/chimeraforge/cli.py b/src/chimeraforge/cli.py index 7a107553..df35086e 100644 --- a/src/chimeraforge/cli.py +++ b/src/chimeraforge/cli.py @@ -1,4 +1,4 @@ -"""ChimeraForge CLI — Typer application. +"""ChimeraForge CLI - Typer application. The command implementations live in ``chimeraforge.commands.*``; this module wires them onto the Typer ``app``. Heavy modules are imported lazily inside each @@ -26,7 +26,7 @@ app = typer.Typer( name="chimeraforge", - help="LLM deployment optimizer — backed by ~204,000 real measurements.", + help="LLM deployment optimizer - backed by ~204,000 real measurements.", no_args_is_help=True, add_completion=False, ) @@ -49,7 +49,7 @@ def main( is_eager=True, ), ) -> None: - """ChimeraForge — LLM deployment optimizer.""" + """ChimeraForge - LLM deployment optimizer.""" # Register commands (implementations in chimeraforge.commands.*). diff --git a/src/chimeraforge/commands/bench.py b/src/chimeraforge/commands/bench.py index ad7a9177..2a361ad8 100644 --- a/src/chimeraforge/commands/bench.py +++ b/src/chimeraforge/commands/bench.py @@ -1,4 +1,4 @@ -"""`bench` command — live LLM inference benchmarking.""" +"""`bench` command - live LLM inference benchmarking.""" from __future__ import annotations @@ -160,6 +160,7 @@ async def _run() -> None: model=model, backend_name=backend, context_lengths=ctx_lengths, + quant=quant, runs=runs, workload=workload, rate=rate, @@ -188,7 +189,9 @@ async def _run() -> None: if output_json: data = [result_to_dict(r) for r in results] - console.print(json_mod.dumps(data, indent=2)) + # highlight=False + soft_wrap: emit valid JSON for `--json | jq`; Rich + # otherwise reflows long string values (width 79 when piped) and corrupts them. + console.print(json_mod.dumps(data, indent=2), highlight=False, soft_wrap=True) else: for r in results: agg = r.aggregate diff --git a/src/chimeraforge/commands/catalog.py b/src/chimeraforge/commands/catalog.py index 4e518bc5..39c4276f 100644 --- a/src/chimeraforge/commands/catalog.py +++ b/src/chimeraforge/commands/catalog.py @@ -1,4 +1,4 @@ -"""`catalog` command — build and inspect the local model catalog. +"""`catalog` command - build and inspect the local model catalog. The catalog is a persisted set of resolved ModelSpecs (params + architecture) covering a curated seed of popular models plus, optionally, the models installed @@ -28,12 +28,12 @@ def catalog( output_json: bool = typer.Option(False, "--json", help="Output as JSON."), verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable debug logging."), ) -> None: - """Show the local model catalog, or rebuild it with ``--build``. + """Show the local model catalog, or rebuild it with --build. - Without ``--build`` this lists what is already cached. ``--build`` resolves - the bundled curated seed (and, with ``--with-ollama``, your installed models) - against the live HF/Ollama metadata APIs and persists the result so - ``suggest --source catalog`` works offline afterwards. + Without --build this lists what is already cached. --build resolves the + bundled curated seed (and, with --with-ollama, your installed models) against + the live HF/Ollama metadata APIs and persists the result so + 'suggest --source catalog' works offline afterwards. """ import json as json_mod import logging diff --git a/src/chimeraforge/commands/compare.py b/src/chimeraforge/commands/compare.py index fee7693b..122ee493 100644 --- a/src/chimeraforge/commands/compare.py +++ b/src/chimeraforge/commands/compare.py @@ -1,4 +1,4 @@ -"""`compare` command — diff benchmark results between runs.""" +"""`compare` command - diff benchmark results between runs.""" from __future__ import annotations @@ -69,7 +69,9 @@ def compare( raise typer.Exit(code=1) if output_json: - console.print(format_comparison_json(rows)) + # highlight=False + soft_wrap: valid JSON for `--json | jq` (Rich otherwise + # reflows long values at width 79 when piped and corrupts them). + console.print(format_comparison_json(rows), highlight=False, soft_wrap=True) else: format_comparison_table(rows, console) format_comparison_summary(rows, console) diff --git a/src/chimeraforge/commands/eval.py b/src/chimeraforge/commands/eval.py index 463ca4b4..789f331b 100644 --- a/src/chimeraforge/commands/eval.py +++ b/src/chimeraforge/commands/eval.py @@ -1,4 +1,4 @@ -"""`eval` command — quality evaluation with text-similarity metrics.""" +"""`eval` command - quality evaluation with text-similarity metrics.""" from __future__ import annotations @@ -39,6 +39,12 @@ def eval_cmd( "-q", help="Quantization level (e.g., Q4_K_M).", ), + fp16_baseline: float = typer.Option( + None, + "--fp16-baseline", + help="FP16 composite score to classify the quality drop tier against " + "(e.g. from a prior FP16 eval run). Without it the tier stays 'unknown'.", + ), list_tasks_flag: bool = typer.Option( False, "--list-tasks", @@ -119,6 +125,7 @@ def eval_cmd( model=model, quant=quant, task=task_name, + fp16_composite=fp16_baseline, ) if output_json: diff --git a/src/chimeraforge/commands/measure.py b/src/chimeraforge/commands/measure.py index 3e70b4d9..9761fd9b 100644 --- a/src/chimeraforge/commands/measure.py +++ b/src/chimeraforge/commands/measure.py @@ -1,4 +1,4 @@ -"""`measure` command — benchmark a live model and fold it into the planner corpus.""" +"""`measure` command - benchmark a live model and fold it into the planner corpus.""" from __future__ import annotations @@ -83,7 +83,8 @@ def _on_progress(done: int, total: int) -> None: on_progress=_on_progress, ) ) - except RuntimeError as exc: + except (RuntimeError, ValueError) as exc: + # RuntimeError: backend/model pre-flight failure; ValueError: unknown --backend. console.print(f"[red]Error:[/] {exc}") console.print( "[dim]The model must be served by the backend " diff --git a/src/chimeraforge/commands/plan.py b/src/chimeraforge/commands/plan.py index c98c3b9a..2facf5eb 100644 --- a/src/chimeraforge/commands/plan.py +++ b/src/chimeraforge/commands/plan.py @@ -1,4 +1,4 @@ -"""`plan` command — predictive capacity planner.""" +"""`plan` command - predictive capacity planner.""" from __future__ import annotations @@ -66,7 +66,18 @@ def plan( avg_tokens: int = typer.Option( 128, "--avg-tokens", - help="Average output tokens per request.", + help="Average output tokens per request (decode length).", + ), + prompt_tokens: int = typer.Option( + 512, + "--prompt-tokens", + help="Average input prompt length in tokens (drives prefill / TTFT).", + ), + workload: str = typer.Option( + "steady", + "--workload", + help="Service-time variance preset: steady, chatbot, bursty, agent. " + "High-variance (agent) inflates the tail estimate and warns.", ), models_path: str = typer.Option( None, @@ -119,6 +130,12 @@ def plan( "--json", help="Output as JSON instead of Rich tables.", ), + pareto: bool = typer.Option( + False, + "--pareto", + help="Show the cost/latency/quality trade-off frontier (the menu of " + "non-dominated configs), not just the single cheapest pick.", + ), list_hardware: bool = typer.Option( False, "--list-hardware", @@ -138,14 +155,20 @@ def plan( ) -> None: """Recommend optimal LLM deployment configuration. - Searches model × quantization × backend × instance-count space, + Searches model x quantization x backend x instance-count space, filtering through VRAM, quality, latency, and budget gates. """ import logging - from chimeraforge.planner.engine import enumerate_candidates, find_models_for_size + from chimeraforge.planner.engine import ( + enumerate_candidates, + find_models_for_size, + pareto_frontier, + ) from chimeraforge.planner.formatter import ( format_json, + format_pareto, + format_pareto_json, format_recommendation, print_hardware_table, print_models_table, @@ -188,6 +211,13 @@ def plan( if safety_target is not None and not 0.0 <= safety_target <= 1.0: console.print("[red]Error:[/] --safety-target must be between 0.0 and 1.0.") raise typer.Exit(code=1) + + from chimeraforge.planner.constants import WORKLOAD_CV2 + + if workload not in WORKLOAD_CV2: + console.print(f"[red]Error:[/] --workload must be one of: {', '.join(WORKLOAD_CV2)}.") + raise typer.Exit(code=1) + workload_cv2 = WORKLOAD_CV2[workload] if measure_first and not model: console.print("[red]Error:[/] --measure requires --model.") raise typer.Exit(code=1) @@ -286,13 +316,20 @@ def plan( safety_target=safety_target, specs=specs, trace=trace, + prompt_tokens=prompt_tokens, + workload_cv2=workload_cv2, ) + frontier = pareto_frontier(candidates) if pareto else None + if output_json: # highlight=False + soft_wrap: emit plain JSON so it stays valid (Rich # would otherwise reflow long string values and corrupt them) and pipes # cleanly to `jq`. - console.print(format_json(candidates), highlight=False, soft_wrap=True) + payload = format_pareto_json(frontier) if pareto else format_json(candidates) + console.print(payload, highlight=False, soft_wrap=True) + elif pareto: + format_pareto(frontier, hardware) else: format_recommendation( candidates, @@ -303,9 +340,10 @@ def plan( budget=budget, safety_target=safety_target, ) - if not candidates and trace: - from chimeraforge.planner.engine import summarize_trace - console.print("\n[bold]Why nothing fit:[/]") - for line in summarize_trace(trace): - console.print(f" [yellow]-[/] {line}") + if not candidates and trace and not output_json: + from chimeraforge.planner.engine import summarize_trace + + console.print("\n[bold]Why nothing fit:[/]") + for line in summarize_trace(trace): + console.print(f" [yellow]-[/] {line}") diff --git a/src/chimeraforge/commands/refit.py b/src/chimeraforge/commands/refit.py index 21c91e6d..b358ad52 100644 --- a/src/chimeraforge/commands/refit.py +++ b/src/chimeraforge/commands/refit.py @@ -1,4 +1,4 @@ -"""`refit` command — re-fit planner coefficients from benchmark results.""" +"""`refit` command - re-fit planner coefficients from benchmark results.""" from __future__ import annotations @@ -101,10 +101,31 @@ def refit( except ImportError: out = Path.home() / ".chimeraforge" / "fitted_models.json" + # Validate BEFORE writing so --validate is a real gate, not advisory: invalid + # coefficients (e.g. a quant multiplier < FP16, non-positive throughput) must + # never be persisted with a misleading non-zero exit implying nothing was saved. + vresult = None + if validate: + from chimeraforge.refit.validator import ( + format_validation_json, + format_validation_table, + validate_fitted_models, + ) + + vresult = validate_fitted_models(merged) + if not vresult.passed: + if output_json: + # highlight=False + soft_wrap: valid JSON for `--json | jq`. + console.print(format_validation_json(vresult), highlight=False, soft_wrap=True) + else: + format_validation_table(vresult, console) + console.print("[red]Validation failed -- refit NOT saved.[/]") + raise typer.Exit(code=1) + saved = save_fitted_models(merged, out) if output_json: - console.print(json_mod.dumps(summary, indent=2)) + console.print(json_mod.dumps(summary, indent=2), highlight=False, soft_wrap=True) else: lines = [ f"Bench results loaded: {summary['bench_results_loaded']}", @@ -120,17 +141,10 @@ def refit( console.print(Panel("\n".join(lines), title="Refit Summary", border_style="green")) console.print(f"[green]Saved to:[/] {saved}") - if validate: - from chimeraforge.refit.validator import ( - format_validation_json, - format_validation_table, - validate_fitted_models, - ) + if validate and vresult is not None: + from chimeraforge.refit.validator import format_validation_json, format_validation_table - vresult = validate_fitted_models(merged) if output_json: - console.print(format_validation_json(vresult)) + console.print(format_validation_json(vresult), highlight=False, soft_wrap=True) else: format_validation_table(vresult, console) - if not vresult.passed: - raise typer.Exit(code=1) diff --git a/src/chimeraforge/commands/report.py b/src/chimeraforge/commands/report.py index d8de2e37..fe9a5abc 100644 --- a/src/chimeraforge/commands/report.py +++ b/src/chimeraforge/commands/report.py @@ -1,4 +1,4 @@ -"""`report` command — generate benchmark reports from result files.""" +"""`report` command - generate benchmark reports from result files.""" from __future__ import annotations @@ -119,6 +119,7 @@ def report( "n_results": rpt.n_results, "timestamp": rpt.timestamp, } - console.print(json_mod.dumps(meta, indent=2)) + # highlight=False + soft_wrap: valid JSON for `--json | jq`. + console.print(json_mod.dumps(meta, indent=2), highlight=False, soft_wrap=True) else: format_report_rich(results, console) diff --git a/src/chimeraforge/commands/safety.py b/src/chimeraforge/commands/safety.py index 3f20c6ba..e2d29546 100644 --- a/src/chimeraforge/commands/safety.py +++ b/src/chimeraforge/commands/safety.py @@ -1,4 +1,4 @@ -"""`safety` command — measure a model's refusal rate against a live backend.""" +"""`safety` command - measure a model's refusal rate against a live backend.""" from __future__ import annotations diff --git a/src/chimeraforge/commands/suggest.py b/src/chimeraforge/commands/suggest.py index 89318ee7..2f5189be 100644 --- a/src/chimeraforge/commands/suggest.py +++ b/src/chimeraforge/commands/suggest.py @@ -1,4 +1,4 @@ -"""`suggest` command — discover and rank deployable models for your hardware.""" +"""`suggest` command - discover and rank deployable models for your hardware.""" from __future__ import annotations diff --git a/src/chimeraforge/compare/__init__.py b/src/chimeraforge/compare/__init__.py index 8fcf718d..a3d9da9c 100644 --- a/src/chimeraforge/compare/__init__.py +++ b/src/chimeraforge/compare/__init__.py @@ -1,4 +1,4 @@ -"""Compare — diff benchmark result files side by side.""" +"""Compare - diff benchmark result files side by side.""" from chimeraforge.compare.comparator import ( ComparisonRow, diff --git a/src/chimeraforge/compare/comparator.py b/src/chimeraforge/compare/comparator.py index 2cd76d56..b01ca0fa 100644 --- a/src/chimeraforge/compare/comparator.py +++ b/src/chimeraforge/compare/comparator.py @@ -1,4 +1,4 @@ -"""Compare benchmark result files — load, match, delta, format. +"""Compare benchmark result files - load, match, delta, format. Loads two or more bench result JSON files, matches configurations by (model, backend, quant, workload, context_length), computes deltas, diff --git a/src/chimeraforge/eval/__init__.py b/src/chimeraforge/eval/__init__.py index 1e3a4e52..f76fb345 100644 --- a/src/chimeraforge/eval/__init__.py +++ b/src/chimeraforge/eval/__init__.py @@ -1,4 +1,4 @@ -"""ChimeraForge Quality Evaluation — text-similarity metrics for LLM outputs. +"""ChimeraForge Quality Evaluation - text-similarity metrics for LLM outputs. Public API: evaluate_quality Compute all quality metrics for predictions vs references diff --git a/src/chimeraforge/eval/runner.py b/src/chimeraforge/eval/runner.py index 40d5764c..54a480b3 100644 --- a/src/chimeraforge/eval/runner.py +++ b/src/chimeraforge/eval/runner.py @@ -1,4 +1,4 @@ -"""Eval orchestrator — run quality evaluation on model outputs. +"""Eval orchestrator - run quality evaluation on model outputs. Provides functions to evaluate predictions against references, load from files, and format results as Rich tables or JSON. diff --git a/src/chimeraforge/measure.py b/src/chimeraforge/measure.py index f01a5bef..6465218a 100644 --- a/src/chimeraforge/measure.py +++ b/src/chimeraforge/measure.py @@ -81,6 +81,12 @@ def fold_into_corpus( sf[serial_key] = serial_fraction scaling["fitted"] = True + # Stamp the package version so load_effective_models can warn when the corpus + # predates an upgrade (its embedded bundled-coefficient snapshot would + # otherwise silently shadow improved coefficients for un-measured models). + from chimeraforge import __version__ + + merged["_chimeraforge_version"] = __version__ save_fitted_models(merged, corpus_path) return merged diff --git a/src/chimeraforge/planner/__init__.py b/src/chimeraforge/planner/__init__.py index 556ea7e2..3c607d72 100644 --- a/src/chimeraforge/planner/__init__.py +++ b/src/chimeraforge/planner/__init__.py @@ -1,4 +1,4 @@ -"""ChimeraForge Capacity Planner — predict-only models and search engine.""" +"""ChimeraForge Capacity Planner - predict-only models and search engine.""" from chimeraforge.planner.engine import Candidate, enumerate_candidates, find_models_for_size from chimeraforge.planner.models import ( diff --git a/src/chimeraforge/planner/constants.py b/src/chimeraforge/planner/constants.py index 4311c025..24d41112 100644 --- a/src/chimeraforge/planner/constants.py +++ b/src/chimeraforge/planner/constants.py @@ -1,4 +1,4 @@ -"""Constants — quant levels, model registry, backends. +"""Constants - quant levels, model registry, backends. Extracted from TR133 research. No repo-specific paths or imports. """ @@ -42,6 +42,35 @@ # Supported serving backends BACKENDS = ["ollama", "vllm", "tgi"] +# Which backends do continuous (in-flight) batching -- one GPU serves many +# sequences concurrently, amortizing weight reads. Ollama (llama.cpp) effectively +# serves one stream per slot, so it is modelled at batch=1 (replicas, not batch). +BACKEND_CONTINUOUS_BATCHING: dict[str, bool] = { + "ollama": False, + "vllm": True, + "tgi": True, +} + +# Decode compute-utilisation ceiling for batched decode (when large batches turn +# the FC/MLP kernels compute-bound). Decode is mostly memory-bound, so this acts +# as a safety cap rarely reached on consumer GPUs. +DECODE_COMPUTE_MFU = 0.5 + +# Workload service-time variance (squared coefficient of variation, Cs^2) presets. +# Analytical queueing is conservative for low-variance traffic but under-estimates +# the tail for high-variance/agent workloads (heavy-tailed service: a few requests +# run 100x longer and hold a slot). Cs^2=0 is deterministic (reproduces M/D/1). +WORKLOAD_CV2: dict[str, float] = { + "steady": 0.0, # fixed-length, deterministic + "chatbot": 1.0, # variable output length (typical) + "bursty": 4.0, # mixed short/long + "agent": 8.0, # heavy-tailed (long tool calls / multi-turn) +} + +# At/above this Cs^2 the analytical p95 is not trustworthy on its own -- warn and +# advise a real load test / simulation (the head-of-line-blocking regime). +HIGH_VARIANCE_CV2 = 4.0 + # Roofline throughput estimate for off-registry models. Decode is memory-bound: # each token streams all weights once, so tok/s ~= MBU * bandwidth / weight_bytes. # MBU (memory-bandwidth utilisation) calibrated from the llama3.2-1b ollama FP16 @@ -52,6 +81,24 @@ DEFAULT_ARCH: dict[str, int] = {"n_layers": 32, "n_kv_heads": 8, "d_head": 128} DEFAULT_PARAMS_B = 3.0 +# Fraction of VRAM a batched server can devote to KV-cache after weights + +# activations + framework overhead. PagedAttention packs KV at block granularity, +# so realised utilisation is high but not 1.0. Used to bound concurrent sequences. +KV_CACHE_UTILISATION = 0.9 + +# KV-cache element size in bytes. Backends keep KV in FP16 even when weights are +# quantized (KV quantization is not yet modelled here). +KV_DTYPE_BYTES = 2 + +# Prefill is compute-bound: ~2 FLOPs per parameter per prompt token. MFU (model +# FLOPs utilisation) discounts peak TFLOPS to realised; 0.3-0.5 is typical for a +# single-stream forward pass. Calibratable later from measured TTFT. +FLOPS_PER_PARAM_PER_TOKEN = 2 +PREFILL_MFU = 0.4 + +# Default prompt (input) length in tokens for TTFT estimation when unspecified. +DEFAULT_PROMPT_TOKENS = 512 + # Model registry: params in billions MODEL_PARAMS_B: dict[str, float] = { "qwen2.5-0.5b": 0.49, diff --git a/src/chimeraforge/planner/data/fitted_models.json b/src/chimeraforge/planner/data/fitted_models.json index 0efe5e5e..907cedd2 100644 --- a/src/chimeraforge/planner/data/fitted_models.json +++ b/src/chimeraforge/planner/data/fitted_models.json @@ -1,7 +1,7 @@ { "vram": { "overhead_factor": 1.058047797687617, - "act_coeff": 0.00454768594328633, + "act_coeff": 0.00909537188657266, "fitted": true }, "throughput": { diff --git a/src/chimeraforge/planner/discovery.py b/src/chimeraforge/planner/discovery.py index 96098043..32c0bd1f 100644 --- a/src/chimeraforge/planner/discovery.py +++ b/src/chimeraforge/planner/discovery.py @@ -35,7 +35,7 @@ DEFAULT_HF_LIMIT = 8 -# ── Network discovery ───────────────────────────────────────────────── +# -- Network discovery ------------------------------------------------- def fetch_ollama_tags(base_url: str = DEFAULT_OLLAMA_URL) -> list[str]: @@ -47,6 +47,8 @@ def fetch_ollama_tags(base_url: str = DEFAULT_OLLAMA_URL) -> list[str]: return [m["name"] for m in resp.json().get("models", []) if m.get("name")] except httpx.HTTPError as exc: raise ResolverError(f"could not list Ollama models at {base_url}: {exc}") from exc + except ValueError as exc: # non-JSON 200 body + raise ResolverError(f"Ollama returned a non-JSON tag list at {base_url}: {exc}") from exc def fetch_hf_text_generation( @@ -65,6 +67,8 @@ def fetch_hf_text_generation( return [m["id"] for m in resp.json() if m.get("id")] except httpx.HTTPError as exc: raise ResolverError(f"could not list HF models: {exc}") from exc + except ValueError as exc: # malformed JSON in the model list + raise ResolverError(f"HF returned a non-JSON model list: {exc}") from exc def discover_identifiers( @@ -107,7 +111,7 @@ def resolve_many( return specs, errors -# ── Pure orchestration ──────────────────────────────────────────────── +# -- Pure orchestration ------------------------------------------------ def best_per_model(candidates: list[Candidate]) -> list[Candidate]: @@ -155,7 +159,7 @@ def suggest( return best_per_model(candidates) -# ── Live catalog ────────────────────────────────────────────────────── +# -- Live catalog ------------------------------------------------------ # # A persistent index of resolved ModelSpecs, so `suggest --source catalog` # ranks a curated set of known-good models offline (after one `catalog build`). diff --git a/src/chimeraforge/planner/engine.py b/src/chimeraforge/planner/engine.py index 9e90c1c0..a8c7b02d 100644 --- a/src/chimeraforge/planner/engine.py +++ b/src/chimeraforge/planner/engine.py @@ -10,14 +10,24 @@ from dataclasses import dataclass, field from chimeraforge.planner.constants import ( + BACKEND_CONTINUOUS_BATCHING, BACKENDS, + DEFAULT_ARCH, + DEFAULT_PROMPT_TOKENS, + HIGH_VARIANCE_CV2, + MODEL_ARCH, MODEL_PARAMS_B, QUANT_BPW, QUANT_LEVELS, ) from chimeraforge.planner.hardware import get_gpu from chimeraforge.planner.models import PlannerModels -from chimeraforge.planner.resolver import SOURCE_REGISTRY, SOURCE_REGISTRY_APPROX, ModelSpec +from chimeraforge.planner.resolver import ( + SOURCE_MANUAL, + SOURCE_REGISTRY, + SOURCE_REGISTRY_APPROX, + ModelSpec, +) @dataclass @@ -43,6 +53,13 @@ class Candidate: params_b: float = 0.0 model_source: str = SOURCE_REGISTRY provenance: dict[str, str] = field(default_factory=dict) + # KV-cache-bound max concurrent sequences a single GPU can hold (0.6.0). + max_concurrent_seqs: int = 0 + # Latency split (0.6.0): prefill time-to-first-token + decode time-per-output-token. + ttft_ms: float = 0.0 + tpot_ms: float = 0.0 + # Continuous-batching: requests served concurrently per GPU (B; 1 = single-stream). + effective_batch: int = 1 def find_models_for_size(target_size: str) -> list[str]: @@ -81,10 +98,12 @@ def enumerate_candidates( safety_target: float | None = None, specs: dict[str, ModelSpec] | None = None, trace: list[tuple[str, str, str, str]] | None = None, + prompt_tokens: int = DEFAULT_PROMPT_TOKENS, + workload_cv2: float = 0.0, ) -> list[Candidate]: """Search (model, quant, backend, N) space with gates. - Gates: VRAM, quality, latency, budget — plus an opt-in safety gate + Gates: VRAM, quality, latency, budget - plus an opt-in safety gate (rejects cells whose refusal rate < ``safety_target``). When ``safety_target`` is None the safety gate is inert but each candidate still carries its refusal rate and RTSI risk tier. @@ -114,10 +133,21 @@ def _reject(model: str, quant: str, gate: str, detail: str) -> None: spec = specs.get(model) if spec is None and model in MODEL_PARAMS_B: spec = ModelSpec.from_registry(model) + params_known = spec is not None or model in MODEL_PARAMS_B params_b = spec.params_b if spec else MODEL_PARAMS_B.get(model, 3.0) arch = spec.arch() if spec else None family = spec.family if spec else None - model_source = spec.source if spec else SOURCE_REGISTRY + # Don't mislabel an unknown model as "registry": only a real registry hit + # or a resolved spec has a trustworthy source. (CLI paths always populate + # specs; this guards direct enumerate_candidates() library callers.) + model_source = ( + spec.source if spec else (SOURCE_REGISTRY if model in MODEL_PARAMS_B else SOURCE_MANUAL) + ) + + # TTFT (prefill) is compute-bound: same for all quants/backends of a model + # on this GPU and prompt length, so compute it once. 0.0 when GPU compute + # is unknown -> latency falls back to decode-only. + ttft_ms = models.latency.predict_ttft_ms(params_b, prompt_tokens, hardware) # ``alias`` is the registry model whose measured data we may reuse: the # model itself for registry hits, the matched model for offline @@ -141,17 +171,25 @@ def _reject(model: str, quant: str, gate: str, detail: str) -> None: _reject(model, quant, "vram", f"{vram:.1f}GB > {hw_vram:.0f}GB capacity") continue + # KV-cache-bound concurrency a single GPU can hold + per-sequence KV + # size; both feed the batched-throughput model (0.6.0). + arch_eff = arch or MODEL_ARCH.get(model, DEFAULT_ARCH) + max_seqs = models.vram.max_concurrent_seqs( + params_b, quant, arch_eff, context_length, hw_vram + ) + kv_per_seq_gb = models.vram.kv_cache_gb(arch_eff, context_length, 1) + # Gate 2: Quality (with provenance: measured | estimated | unknown) quality, quality_source = models.quality.estimate(lookup_name, quant, family) if quality < quality_target: _reject(model, quant, "quality", f"{quality:.2f} < target {quality_target}") continue - quality_tier = models.quality.quality_tier(lookup_name, quant) + quality_tier = models.quality.quality_tier(lookup_name, quant, family) # Safety gate (Gate 5): safety data is per (model, quant) and - # backend-independent, so evaluate it here — before the backend/N - # loop — to skip known-unsafe cells early. Opt-in via safety_target. + # backend-independent, so evaluate it here - before the backend/N + # loop - to skip known-unsafe cells early. Opt-in via safety_target. safety_refusal = models.safety.predict_refusal(lookup_name, quant) rtsi_risk = models.safety.rtsi_risk(lookup_name, quant) if ( @@ -182,44 +220,59 @@ def _reject(model: str, quant: str, gate: str, detail: str) -> None: throughput_source = "estimated" used_roofline = True - # Find minimum N to meet request_rate. N counts INDEPENDENT GPU - # instances (VRAM is per-GPU, cost is per-GPU x N), so replicas - # behind a load balancer scale linearly (eta = 1). The Amdahl - # serial-fraction model is concurrency-on-one-backend physics, not - # replica fan-out; modelling per-GPU batching throughput is Phase 2 - # (KV-cache-bound). Each replica runs single-stream (conservative). + # Search (N replicas x B batch-per-GPU) for the cheapest config + # meeting the rate under the latency SLO. N replicas scale linearly + # (eta=1). For a continuous-batching backend (vLLM/TGI) one GPU + # serves B concurrent sequences -- aggregate throughput rises with + # B up to the KV-cache cap -- so a single GPU can replace several + # single-stream (Ollama) replicas. Higher B trades per-request + # latency (TPOT) for aggregate throughput; we pick the smallest + # feasible (N, then B) for lowest cost + lowest latency. eta = 1.0 required_tps = request_rate * avg_tokens + batched = BACKEND_CONTINUOUS_BATCHING.get(backend, False) + b_max = max_seqs if (batched and max_seqs > 1) else 1 + batch_grid = _batch_grid(b_max) - # Find minimum N that satisfies both throughput and latency - best_n = None + best = None # (n, b, per_gpu_tps, per_req_tps, lat) for n in range(1, 17): - total_tps = n * n1_tps # linear replica scaling - if total_tps < required_tps: - continue - - lat = models.latency.predict_p95( - lookup_name, - backend, - request_rate, - n_agents=n, - avg_tokens=avg_tokens, - quant=quant, - hardware=hardware, - n1_tps=n1_tps, - ) - if lat["p95_ms"] <= latency_slo: - best_n = n + for b in batch_grid: + per_gpu = models.throughput.batched_decode_tps( + n1_tps, kv_per_seq_gb, b, hardware, params_b + ) + if n * per_gpu < required_tps: + continue + per_req = per_gpu / b + lat = models.latency.predict_p95( + lookup_name, + backend, + request_rate, + n_agents=n, + avg_tokens=avg_tokens, + quant=quant, + hardware=hardware, + n1_tps=per_req, + ttft_ms=ttft_ms, + concurrent_per_agent=b, + service_cv2=workload_cv2, + ) + if lat["p95_ms"] <= latency_slo: + best = (n, b, per_gpu, per_req, lat) + break + if best: break - if best_n is None: - cap_tps = 16 * n1_tps + if best is None: + cap_tps = 16 * models.throughput.batched_decode_tps( + n1_tps, kv_per_seq_gb, b_max, hardware, params_b + ) if cap_tps < required_tps: _reject( model, quant, "throughput", - f"{backend}: max {cap_tps:.0f} tok/s at N=16 < {required_tps:.0f} needed", + f"{backend}: max {cap_tps:.0f} tok/s at N=16 B={b_max} " + f"< {required_tps:.0f} needed", ) else: _reject( @@ -227,17 +280,9 @@ def _reject(model: str, quant: str, gate: str, detail: str) -> None: ) continue - total_tps = best_n * n1_tps - lat = models.latency.predict_p95( - lookup_name, - backend, - request_rate, - n_agents=best_n, - avg_tokens=avg_tokens, - quant=quant, - hardware=hardware, - n1_tps=n1_tps, - ) + best_n, best_b, per_gpu_tps, per_req_tps, lat = best + total_tps = best_n * per_gpu_tps + tpot_ms = 1000.0 / per_req_tps if per_req_tps > 0 else 0.0 # Gate 4: Cost monthly = models.cost.predict_monthly(hw_cost_hr) * best_n @@ -267,6 +312,11 @@ def _reject(model: str, quant: str, gate: str, detail: str) -> None: } warnings = [] + if workload_cv2 >= HIGH_VARIANCE_CV2: + warnings.append( + "high service-time variance (agent/bursty): analytical p95 " + "under-estimates the tail -- validate with a load test" + ) if lat["saturated"]: warnings.append("utilisation > 70% safety cap") if quality_tier == "concerning": @@ -289,6 +339,11 @@ def _reject(model: str, quant: str, gate: str, detail: str) -> None: f"off-registry model ({model_source}): throughput is a roofline " "estimate, not measured" ) + if not params_known: + warnings.append( + f"params/architecture unknown; assumed {params_b:.1f}B -- " + "pass a resolvable --model or manual overrides" + ) if quality_source == "unknown": warnings.append("quality unscreened (neutral 0.5 prior, not measured)") elif quality_source == "estimated" and not use_measured: @@ -318,6 +373,10 @@ def _reject(model: str, quant: str, gate: str, detail: str) -> None: params_b=round(params_b, 4), model_source=model_source, provenance=provenance, + max_concurrent_seqs=max_seqs, + ttft_ms=round(ttft_ms, 1), + tpot_ms=round(tpot_ms, 1), + effective_batch=best_b, ) ) @@ -326,6 +385,47 @@ def _reject(model: str, quant: str, gate: str, detail: str) -> None: return candidates +def pareto_frontier(candidates: list[Candidate]) -> list[Candidate]: + """Non-dominated configs on (cost down, p95 latency down, quality up). + + For a fixed workload every candidate already meets the throughput + SLO gates, + so the remaining trade-offs are cost vs latency vs quality. A candidate is + dominated if another is no worse on all three and strictly better on one. The + frontier is the menu of real trade-offs (cheapest, lowest-latency, highest + quality, and the bends between) -- not a single cost-sorted point. Returned + sorted by monthly cost ascending. + """ + + def dominates(b: Candidate, a: Candidate) -> bool: + no_worse = ( + b.monthly_cost <= a.monthly_cost + and b.p95_latency_ms <= a.p95_latency_ms + and b.quality >= a.quality + ) + strictly_better = ( + b.monthly_cost < a.monthly_cost + or b.p95_latency_ms < a.p95_latency_ms + or b.quality > a.quality + ) + return no_worse and strictly_better + + front = [a for a in candidates if not any(dominates(b, a) for b in candidates if b is not a)] + front.sort(key=lambda c: (c.monthly_cost, c.p95_latency_ms)) + return front + + +def _batch_grid(b_max: int) -> list[int]: + """Batch sizes to try, 1..b_max on a log grid (cheap search, B can be large).""" + if b_max <= 1: + return [1] + grid, b = [], 1 + while b < b_max: + grid.append(b) + b *= 2 + grid.append(b_max) + return grid + + # Order in which a (model, quant) cell is tested; used to pick the *binding* # gate (the furthest one reached) when summarising a failed search. _GATE_ORDER = ["vram", "quality", "safety", "throughput", "latency", "budget"] diff --git a/src/chimeraforge/planner/formatter.py b/src/chimeraforge/planner/formatter.py index 87f0938b..9c26ca77 100644 --- a/src/chimeraforge/planner/formatter.py +++ b/src/chimeraforge/planner/formatter.py @@ -75,9 +75,16 @@ def format_recommendation( tp_color = "green" if tp_basis == "measured" else "yellow" perf.add_row("N=1 throughput", f"{best.throughput_tps} tok/s [{tp_color}]({tp_basis})[/]") perf.add_row("Total throughput", f"{best.total_throughput_tps} tok/s") - perf.add_row("Scaling eta(N)", str(best.eta)) - perf.add_row("p95 latency", f"{best.p95_latency_ms} ms") + if best.effective_batch > 1: + perf.add_row("Batch per GPU", f"{best.effective_batch} concurrent (continuous batching)") + if best.ttft_ms: + perf.add_row("TTFT (prefill)", f"{best.ttft_ms} ms") + if best.tpot_ms: + perf.add_row("TPOT (per token)", f"{best.tpot_ms} ms") + perf.add_row("p95 latency", f"{best.p95_latency_ms} ms (end-to-end)") perf.add_row("Utilisation", f"{best.utilisation:.1%}") + if best.max_concurrent_seqs: + perf.add_row("Max concurrent/GPU", f"{best.max_concurrent_seqs} seqs (KV-cache bound)") # Quality + cost table cost_table = Table(show_header=False, box=None, padding=(0, 2)) @@ -243,6 +250,66 @@ def format_suggestions_json( ) +def format_pareto(frontier: list[Candidate], hardware: str) -> None: + """Render the Pareto frontier (cost / latency / quality trade-off menu).""" + if not frontier: + console.print( + Panel( + "[bold red]No viable configuration found.[/]", + title="ChimeraForge Pareto Frontier", + border_style="red", + ) + ) + return + + cheapest = min(frontier, key=lambda c: c.monthly_cost) + fastest = min(frontier, key=lambda c: c.p95_latency_ms) + best_q = max(frontier, key=lambda c: c.quality) + + table = Table(title=f"Pareto frontier for {hardware} (non-dominated trade-offs)") + table.add_column("Model") + table.add_column("Quant") + table.add_column("Backend") + table.add_column("N", justify="right") + table.add_column("Batch", justify="right") + table.add_column("$/mo", justify="right") + table.add_column("p95 ms", justify="right") + table.add_column("Quality", justify="right") + table.add_column("Pick", style="dim") + + for c in frontier: + tags = [] + if c is cheapest: + tags.append("cheapest") + if c is fastest: + tags.append("fastest") + if c is best_q: + tags.append("best-quality") + q_mark = "" if c.provenance.get("quality") == "measured" else "~" + table.add_row( + c.model, + c.quant, + c.backend, + str(c.n_agents), + str(c.effective_batch), + f"${c.monthly_cost}", + f"{c.p95_latency_ms}", + f"{q_mark}{c.quality}", + ", ".join(tags), + ) + console.print() + console.print(table) + console.print( + f" [dim]{len(frontier)} non-dominated configs. Each is best at *something* " + f"(cost / latency / quality); points off the frontier are strictly worse.[/]\n" + ) + + +def format_pareto_json(frontier: list[Candidate]) -> str: + """Pareto frontier as JSON.""" + return json.dumps([asdict(c) for c in frontier], indent=2) + + def print_hardware_table() -> None: """Print GPU database as a Rich table.""" table = Table(title="Available GPUs") diff --git a/src/chimeraforge/planner/hardware.py b/src/chimeraforge/planner/hardware.py index 6f3a80bb..8e61b860 100644 --- a/src/chimeraforge/planner/hardware.py +++ b/src/chimeraforge/planner/hardware.py @@ -15,31 +15,38 @@ class GPUSpec: name: str vram_gb: float - bandwidth_gbps: float # Memory bandwidth in GB/s + bandwidth_gbps: float # Memory bandwidth in GB/s (decode/TPOT is bound by this) cost_per_hour: float # $/hr (cloud rental or amortised consumer) + fp16_tflops: float = 0.0 # Dense FP16 Tensor TFLOPS, FP32 accumulate (prefill/TTFT) -# Reference GPU — all TR measurements collected on this card +# Reference GPU - all TR measurements collected on this card REFERENCE_GPU = "RTX 4080 12GB" +# fp16_tflops: dense FP16 Tensor Core, FP32-accumulate, non-sparse. Datacenter +# values are official datasheets (A100 312, H100 SXM 989, L4 121, T4 65); +# consumer Ada are derived (dense = 2x FP32 shader, cross-checked vs RTX 4090's +# published 165.2); consumer Ampere use the FP32-accumulate dense rate (half the +# FP16-accumulate figure) to stay on one basis. Approximate; prefill MFU and the +# `measure` path absorb the slack. GPU_DB: dict[str, GPUSpec] = { - # Consumer — NVIDIA - "RTX 4060 8GB": GPUSpec("RTX 4060 8GB", 8.0, 272.0, 0.020), - "RTX 4060 Ti 8GB": GPUSpec("RTX 4060 Ti 8GB", 8.0, 288.0, 0.025), - "RTX 4060 Ti 16GB": GPUSpec("RTX 4060 Ti 16GB", 16.0, 288.0, 0.030), - "RTX 4070 12GB": GPUSpec("RTX 4070 12GB", 12.0, 504.0, 0.030), - "RTX 4070 Ti 12GB": GPUSpec("RTX 4070 Ti 12GB", 12.0, 504.0, 0.035), - "RTX 4080 12GB": GPUSpec("RTX 4080 12GB", 12.0, 556.0, 0.035), - "RTX 4080 16GB": GPUSpec("RTX 4080 16GB", 16.0, 717.0, 0.045), - "RTX 4090 24GB": GPUSpec("RTX 4090 24GB", 24.0, 1008.0, 0.060), - "RTX 3090 24GB": GPUSpec("RTX 3090 24GB", 24.0, 936.0, 0.040), - "RTX 3080 10GB": GPUSpec("RTX 3080 10GB", 10.0, 760.0, 0.025), - # Data-center — NVIDIA - "A100 40GB": GPUSpec("A100 40GB", 40.0, 1555.0, 1.10), - "A100 80GB": GPUSpec("A100 80GB", 80.0, 2039.0, 1.60), - "H100 80GB": GPUSpec("H100 80GB", 80.0, 3352.0, 2.50), - "L4 24GB": GPUSpec("L4 24GB", 24.0, 300.0, 0.50), - "T4 16GB": GPUSpec("T4 16GB", 16.0, 320.0, 0.35), + # Consumer - NVIDIA + "RTX 4060 8GB": GPUSpec("RTX 4060 8GB", 8.0, 272.0, 0.020, 30.2), + "RTX 4060 Ti 8GB": GPUSpec("RTX 4060 Ti 8GB", 8.0, 288.0, 0.025, 44.1), + "RTX 4060 Ti 16GB": GPUSpec("RTX 4060 Ti 16GB", 16.0, 288.0, 0.030, 44.1), + "RTX 4070 12GB": GPUSpec("RTX 4070 12GB", 12.0, 504.0, 0.030, 58.4), + "RTX 4070 Ti 12GB": GPUSpec("RTX 4070 Ti 12GB", 12.0, 504.0, 0.035, 80.2), + "RTX 4080 12GB": GPUSpec("RTX 4080 12GB", 12.0, 556.0, 0.035, 80.2), + "RTX 4080 16GB": GPUSpec("RTX 4080 16GB", 16.0, 717.0, 0.045, 97.5), + "RTX 4090 24GB": GPUSpec("RTX 4090 24GB", 24.0, 1008.0, 0.060, 165.2), + "RTX 3090 24GB": GPUSpec("RTX 3090 24GB", 24.0, 936.0, 0.040, 71.0), + "RTX 3080 10GB": GPUSpec("RTX 3080 10GB", 10.0, 760.0, 0.025, 59.5), + # Data-center - NVIDIA + "A100 40GB": GPUSpec("A100 40GB", 40.0, 1555.0, 1.10, 312.0), + "A100 80GB": GPUSpec("A100 80GB", 80.0, 2039.0, 1.60, 312.0), + "H100 80GB": GPUSpec("H100 80GB", 80.0, 3352.0, 2.50, 989.0), + "L4 24GB": GPUSpec("L4 24GB", 24.0, 300.0, 0.50, 121.0), + "T4 16GB": GPUSpec("T4 16GB", 16.0, 320.0, 0.35, 65.0), } diff --git a/src/chimeraforge/planner/identity.py b/src/chimeraforge/planner/identity.py index 42709e95..ae2cc232 100644 --- a/src/chimeraforge/planner/identity.py +++ b/src/chimeraforge/planner/identity.py @@ -117,7 +117,7 @@ def resolve_model(identifier: str) -> str | None: if len(candidates) == 1: return candidates[0] params = parse_params_b(identifier) - if params is None: + if params is None or params <= 0: # e.g. a degenerate "...0b" token -> no division return None best = min(candidates, key=lambda m: abs(MODEL_PARAMS_B[m] - params)) return best if abs(MODEL_PARAMS_B[best] - params) / params <= _PARAMS_TOLERANCE else None diff --git a/src/chimeraforge/planner/models.py b/src/chimeraforge/planner/models.py index 6f713f3e..baa5ccbf 100644 --- a/src/chimeraforge/planner/models.py +++ b/src/chimeraforge/planner/models.py @@ -20,10 +20,15 @@ from pathlib import Path from chimeraforge.planner.constants import ( + DEFAULT_ARCH, + FLOPS_PER_PARAM_PER_TOKEN, + KV_CACHE_UTILISATION, + KV_DTYPE_BYTES, MBU_DEFAULT, MODEL_ARCH, MODEL_FAMILY, MODEL_PARAMS_B, + PREFILL_MFU, QUANT_BPW, ) from chimeraforge.planner.hardware import bandwidth_ratio, get_gpu @@ -31,7 +36,7 @@ log = logging.getLogger("chimeraforge.planner.models") -# ── 1. VRAM Model ───────────────────────────────────────────────────── +# -- 1. VRAM Model ----------------------------------------------------- @dataclass @@ -60,8 +65,18 @@ def predict( bpw = QUANT_BPW.get(quant, 16.0) weight_gb = params * bpw / 8 - arch = arch or MODEL_ARCH.get(model, {"n_layers": 32, "n_kv_heads": 8, "d_head": 128}) + arch = arch or MODEL_ARCH.get(model, DEFAULT_ARCH) + kv_gb = self.kv_cache_gb(arch, context_length, batch_size) + # Linear in context: flash/paged attention never materialises the O(ctx^2) + # attention matrix, so activation memory is O(ctx). (A quadratic term + # diverges unphysically at long context -- 130 GB at 32k for a 3B model.) + act_gb = self.act_coeff * arch["n_layers"] * (context_length / 1024) + return weight_gb * self.overhead_factor + kv_gb + act_gb + + @staticmethod + def kv_cache_gb(arch: dict[str, int], context_length: int, batch_size: int = 1) -> float: + """KV-cache size in GB: ``2 (K+V) * layers * batch * ctx * kv_heads * d_head * dtype``.""" kv_bytes = ( 2 * arch["n_layers"] @@ -69,13 +84,36 @@ def predict( * context_length * arch["n_kv_heads"] * arch["d_head"] - * 2 + * KV_DTYPE_BYTES ) - kv_gb = kv_bytes / (1024**3) - - act_gb = self.act_coeff * arch["n_layers"] * (context_length / 1024) ** 2 + return kv_bytes / (1024**3) - return weight_gb * self.overhead_factor + kv_gb + act_gb + def max_concurrent_seqs( + self, + params_b: float, + quant: str, + arch: dict[str, int], + context_length: int, + hw_vram_gb: float, + utilisation: float = KV_CACHE_UTILISATION, + ) -> int: + """Max concurrent sequences a single GPU can hold, KV-cache bound. + + This is the real concurrency limiter for batched backends (vLLM/TGI): + after model weights + activations, the remaining VRAM divided by the + per-sequence KV-cache caps how many requests can be in flight at once. + First-principles memory arithmetic -- no fitting. Returns 0 if the weights + alone don't fit. + """ + weight_gb = params_b * QUANT_BPW.get(quant, 16.0) / 8 * self.overhead_factor + act_gb = ( + self.act_coeff * arch["n_layers"] * (context_length / 1024) + ) # O(ctx), see predict() + free_gb = hw_vram_gb * utilisation - weight_gb - act_gb + per_seq_gb = self.kv_cache_gb(arch, context_length, batch_size=1) + if per_seq_gb <= 0 or free_gb <= 0: + return 0 + return int(free_gb / per_seq_gb) def to_dict(self) -> dict: return { @@ -94,7 +132,7 @@ def from_dict(cls, d: dict) -> VRAMModel: return m -# ── 2. Throughput Model ─────────────────────────────────────────────── +# -- 2. Throughput Model ----------------------------------------------- @dataclass @@ -119,6 +157,11 @@ def quant_multiplier(self, quant: str) -> float: bpw = QUANT_BPW.get(quant) if bpw is None: return 1.0 + if bpw > 16.0: + # Above FP16 (e.g. FP32): no dequant speedup, pure bandwidth penalty + # (decode streams 2x the weight bytes). Nearest-bpw would wrongly pick + # FP16=1.0 and predict the full FP16 rate. + return 16.0 / bpw known = [ (QUANT_BPW[q], mult) for q, mult in self.quant_multipliers.items() if q in QUANT_BPW ] @@ -183,6 +226,42 @@ def roofline_tps( base_tps = mbu * bandwidth / fp16_weight_gb return max(base_tps * self.quant_multiplier(quant), 0.1) + def batched_decode_tps( + self, + n1_tps: float, + kv_per_seq_gb: float, + batch: int, + hardware: str | None = None, + params_b: float | None = None, + mbu: float = MBU_DEFAULT, + ) -> float: + """Aggregate decode tok/s for a continuous-batching backend at batch B. + + Anchored to the single-stream ``n1_tps`` (measured or roofline, so it stays + quant-correct), then adds KV-amortization physics: at batch B the weights + are read once per step but each of the B sequences reads its own KV, so + + aggregate(B) = B * bw*MBU / (weight_eff + B * kv_per_seq) + + where ``weight_eff = bw*MBU / n1_tps`` backs the effective weight bytes out + of the calibrated single-stream rate. Rises ~linearly with B while weights + dominate, then saturates at ``bw*MBU / kv_per_seq`` (KV-bandwidth bound). + Capped by the decode compute ceiling. Returns ``n1_tps`` for batch <= 1. + """ + if batch <= 1 or n1_tps <= 0: + return max(n1_tps, 0.1) + gpu = get_gpu(hardware) if hardware else None + bandwidth = gpu.bandwidth_gbps if gpu else 556.0 + denom = bandwidth * mbu # effective GB/s + weight_eff_gb = denom / n1_tps + agg = batch * denom / (weight_eff_gb + batch * kv_per_seq_gb) + if params_b and gpu and gpu.fp16_tflops > 0: + from chimeraforge.planner.constants import DECODE_COMPUTE_MFU + + compute_ceiling = gpu.fp16_tflops * 1e12 * DECODE_COMPUTE_MFU / (2 * params_b * 1e9) + agg = min(agg, compute_ceiling) + return max(agg, n1_tps) + def to_dict(self) -> dict: return { "lookup": self.lookup, @@ -204,7 +283,7 @@ def from_dict(cls, d: dict) -> ThroughputModel: return m -# ── 3. Scaling Model ────────────────────────────────────────────────── +# -- 3. Scaling Model -------------------------------------------------- @dataclass @@ -246,7 +325,7 @@ def from_dict(cls, d: dict) -> ScalingModel: return m -# ── 4. Quality Model ────────────────────────────────────────────────── +# -- 4. Quality Model -------------------------------------------------- @dataclass @@ -311,16 +390,34 @@ def estimate(self, model: str, quant: str, family: str | None = None) -> tuple[f delta = self.quant_deltas.get(quant, 0.0) return max(0.0, min(1.0, fp16 + delta)), "estimated" - def quality_tier(self, model: str, quant: str) -> str: - """Classify quality drop into a tier.""" + def quality_tier(self, model: str, quant: str, family: str | None = None) -> str: + """Classify quality drop into a tier. + + Family-aware, mirroring :meth:`estimate`: an off-registry model whose + family matches the registry derives its FP16 baseline (and predicted + quality) from the family mean, so the tier is consistent with the + reported quality instead of silently collapsing to ``unknown``. + """ fp16 = self.fp16_baselines.get(model) - if fp16 is None: - fp16_key = f"{model}|FP16" - if fp16_key in self.lookup: - fp16 = self.lookup[fp16_key] - predicted = self.predict(model, quant) + if fp16 is None and f"{model}|FP16" in self.lookup: + fp16 = self.lookup[f"{model}|FP16"] + if fp16 is None and family is not None: + same_family = [ + v for m, v in self.fp16_baselines.items() if MODEL_FAMILY.get(m) == family + ] + if same_family: + fp16 = sum(same_family) / len(same_family) if fp16 is None or fp16 <= 0: return "unknown" + # Predicted quality consistent with estimate(): direct lookup, else the + # FP16 baseline (+ quant delta) -- not predict(), which is not family-aware. + key = f"{model}|{quant}" + if key in self.lookup: + predicted = self.lookup[key] + elif quant == "FP16": + predicted = fp16 + else: + predicted = max(0.0, min(1.0, fp16 + self.quant_deltas.get(quant, 0.0))) drop_pp = (predicted - fp16) * 100 if drop_pp >= self.TIERS["negligible"]: return "negligible" @@ -349,7 +446,7 @@ def from_dict(cls, d: dict) -> QualityModel: return m -# ── 5. Cost Model ───────────────────────────────────────────────────── +# -- 5. Cost Model ----------------------------------------------------- @dataclass @@ -379,17 +476,38 @@ def from_dict(cls, d: dict) -> CostModel: return cls(hw_cost_per_hour=d.get("hw_cost_per_hour", 0.035)) -# ── 6. Latency Model ───────────────────────────────────────────────── +# -- 6. Latency Model ------------------------------------------------- @dataclass class LatencyModel: - """M/D/1 queueing approximation with 70% utilisation safety cap.""" + """Latency: prefill (TTFT) + decode (TPOT), with M/D/1 queueing on top.""" service_times: dict[str, float] = field(default_factory=dict) safety_factor: float = 0.70 fitted: bool = False + @staticmethod + def predict_ttft_ms( + params_b: float, + prompt_tokens: int, + hardware: str | None = None, + mfu: float = PREFILL_MFU, + ) -> float: + """Time-to-first-token = prefill compute time (compute-bound, ms). + + Prefill does ~2 FLOPs/param/token over the prompt; time = FLOPs / + (peak_TFLOPS * MFU). Returns 0.0 when the GPU's compute is unknown (so the + caller omits a prefill term rather than guessing). Decode/TPOT is modelled + separately via throughput (bandwidth-bound). + """ + gpu = get_gpu(hardware) if hardware else None + tflops = gpu.fp16_tflops if gpu else 0.0 + if tflops <= 0 or params_b <= 0 or prompt_tokens <= 0: + return 0.0 + flops = FLOPS_PER_PARAM_PER_TOKEN * params_b * 1e9 * prompt_tokens + return flops / (tflops * 1e12 * mfu) * 1000.0 + def predict_p95( self, model: str, @@ -402,22 +520,28 @@ def predict_p95( scaling_model: ScalingModel | None = None, hardware: str | None = None, n1_tps: float | None = None, + ttft_ms: float = 0.0, + concurrent_per_agent: int = 1, + service_cv2: float = 0.0, ) -> dict: """Predict p95 latency and utilisation. - ``n1_tps`` lets the caller supply an already-computed N=1 throughput (e.g. - a roofline estimate for an off-registry model) so the service time stays - consistent with the engine's throughput choice instead of being - recomputed from the lookup/power-law default. + ``n1_tps`` is the rate a *single request* decodes at (slower at high batch + under contention), driving the in-service latency. ``concurrent_per_agent`` + is how many requests one GPU serves at once (the batch size for a + continuous-batching backend; 1 for replicas), so system capacity is + ``n_agents * concurrent_per_agent / service_time`` -- this decouples + per-request latency from aggregate capacity. ``ttft_ms`` adds prefill, so + service time is the full request: TTFT + avg_tokens * decode-per-token. """ service_ms = None if n1_tps is not None and n1_tps > 0: - service_ms = avg_tokens / n1_tps * 1000 + service_ms = ttft_ms + avg_tokens / n1_tps * 1000 elif throughput_model is not None: tps = throughput_model.predict(model, backend, quant, hardware) if tps > 0: - service_ms = avg_tokens / tps * 1000 + service_ms = ttft_ms + avg_tokens / tps * 1000 if service_ms is None: key = f"{model}|{backend}" @@ -432,13 +556,17 @@ def predict_p95( eta = 1.0 if scaling_model and n_agents > 1: eta = scaling_model.predict_eta(model, backend, n_agents) - total_capacity = n_agents * mu * eta + total_capacity = n_agents * concurrent_per_agent * mu * eta rho = request_rate / total_capacity if total_capacity > 0 else 1.0 saturated = rho > self.safety_factor if rho < 1.0: - mean_wait_s = rho / (2 * total_capacity * (1 - rho)) + # Two-moment (Allen-Cunneen) wait: (Ca^2 + Cs^2)/2 x M/M/1 wait, with + # Poisson arrivals (Ca^2=1). Cs^2=0 -> (1+0)/2 = M/D/1 (the prior + # behaviour); higher Cs^2 (agent/bursty) inflates the tail. + mm1_wait_s = rho / (total_capacity * (1 - rho)) + mean_wait_s = (1.0 + service_cv2) / 2.0 * mm1_wait_s p95_ms = service_ms + mean_wait_s * 1000 * 3 else: p95_ms = float("inf") @@ -468,7 +596,7 @@ def from_dict(cls, d: dict) -> LatencyModel: return m -# ── 7. Safety Model ─────────────────────────────────────────────────── +# -- 7. Safety Model --------------------------------------------------- @dataclass @@ -526,7 +654,7 @@ def from_dict(cls, d: dict) -> SafetyModel: return m -# ── Aggregate model container ───────────────────────────────────────── +# -- Aggregate model container ----------------------------------------- @dataclass @@ -582,6 +710,23 @@ def load_effective_models(models_path: str | Path | None = None) -> PlannerModel corpus = measured_corpus_path() if corpus.is_file(): try: + with open(corpus, encoding="utf-8") as f: + raw = json.load(f) + # The corpus embeds a snapshot of the bundled coefficients it was + # built on. Warn (don't silently shadow) if it predates the installed + # package, so an upgrade's improved coefficients aren't masked for + # models the user never re-measured. Re-run `measure` to refresh. + from chimeraforge import __version__ + + stamp = raw.get("_chimeraforge_version") + if stamp and stamp != __version__: + log.warning( + "measured corpus %s was written by chimeraforge %s (installed: %s); " + "re-run `measure` to pick up updated bundled coefficients", + corpus, + stamp, + __version__, + ) return load_models(corpus) except (ValueError, OSError) as exc: log.warning("ignoring unreadable measured corpus %s: %s", corpus, exc) diff --git a/src/chimeraforge/planner/resolver.py b/src/chimeraforge/planner/resolver.py index 591316b5..49f939c6 100644 --- a/src/chimeraforge/planner/resolver.py +++ b/src/chimeraforge/planner/resolver.py @@ -35,7 +35,7 @@ MODEL_PARAMS_B, QUANT_BPW, ) -from chimeraforge.planner.identity import parse_identity, resolve_model +from chimeraforge.planner.identity import parse_identity, parse_quant, resolve_model log = logging.getLogger("chimeraforge.planner.resolver") @@ -132,7 +132,7 @@ def from_registry(cls, name: str) -> ModelSpec: ) -# ── Pure parsers (dict -> ModelSpec) ────────────────────────────────── +# -- Pure parsers (dict -> ModelSpec) ---------------------------------- def parse_param_size(label: str | None) -> float | None: @@ -200,7 +200,7 @@ def _info(suffix: str) -> int | None: n_kv_heads=n_kv_heads, d_head=d_head, hidden_size=hidden, - native_quant=ident.quant or details.get("quantization_level"), + native_quant=ident.quant or parse_quant(details.get("quantization_level") or ""), family=ident.family or (arch or None), variant=ident.variant, source=SOURCE_OLLAMA, @@ -277,7 +277,7 @@ def spec_from_overrides(name: str, overrides: dict) -> ModelSpec | None: ) -# ── Spec cache ──────────────────────────────────────────────────────── +# -- Spec cache -------------------------------------------------------- def _cache_dir() -> Path: @@ -325,7 +325,7 @@ def _cache_store(identifier: str, spec: ModelSpec) -> None: log.warning("could not write spec cache for %s: %s", identifier, exc) -# ── Network fetchers (network -> dict) ──────────────────────────────── +# -- Network fetchers (network -> dict) -------------------------------- def _httpx(): @@ -350,6 +350,8 @@ def fetch_ollama_show(tag: str, base_url: str = DEFAULT_OLLAMA_URL) -> dict: raise ResolverError(f"Ollama has no model '{tag}' (pull it first): {exc}") from exc except httpx.HTTPError as exc: raise ResolverError(f"could not reach Ollama at {base_url}: {exc}") from exc + except ValueError as exc: # non-JSON 200 body (captive portal / proxy) + raise ResolverError(f"Ollama returned a non-JSON response at {base_url}: {exc}") from exc def fetch_hf(repo: str, hf_token: str | None = None) -> tuple[dict, float | None]: @@ -389,9 +391,11 @@ def fetch_hf(repo: str, hf_token: str | None = None) -> tuple[dict, float | None return config, params_b except httpx.HTTPError as exc: raise ResolverError(f"could not fetch HF metadata for '{repo}': {exc}") from exc + except ValueError as exc: # malformed JSON in config.json / model_info + raise ResolverError(f"HF returned a non-JSON response for '{repo}': {exc}") from exc -# ── Top-level resolver ──────────────────────────────────────────────── +# -- Top-level resolver ------------------------------------------------ def resolve_spec( diff --git a/src/chimeraforge/refit/__init__.py b/src/chimeraforge/refit/__init__.py index f0c29c9a..f54122c6 100644 --- a/src/chimeraforge/refit/__init__.py +++ b/src/chimeraforge/refit/__init__.py @@ -1,4 +1,4 @@ -"""Refit — update planner coefficients from benchmark results.""" +"""Refit - update planner coefficients from benchmark results.""" from chimeraforge.refit.fitter import ( bayesian_blend_throughput, diff --git a/src/chimeraforge/refit/fitter.py b/src/chimeraforge/refit/fitter.py index cc5d92d3..979c9363 100644 --- a/src/chimeraforge/refit/fitter.py +++ b/src/chimeraforge/refit/fitter.py @@ -273,14 +273,53 @@ def count_total_runs(results: list[dict]) -> int: return total +def _successful_runs(r: dict) -> int: + """Count SUCCESSFUL runs for a result (recorded samples), not attempted. + + ``individual_runs`` holds only the runs that produced metrics; the ``runs`` + field is the attempted count, which over-credits confidence when some failed. + """ + runs = r.get("individual_runs") + if runs is not None: + return len(runs) + n = r.get("runs") + return n if n is not None else 0 + + +def runs_per_throughput_key(results: list[dict]) -> dict[str, int]: + """Successful-run count per ``'{model}|{backend}|{quant}'`` throughput key.""" + counts: dict[str, int] = defaultdict(int) + for r in results: + agg = r.get("aggregate", {}) + if (agg.get("throughput_tps", {}).get("mean") or 0) > 0: + quant = r.get("quant") or "FP16" + counts[f"{r.get('model', '')}|{r.get('backend', '')}|{quant}"] += _successful_runs(r) + return dict(counts) + + +def runs_per_service_key(results: list[dict]) -> dict[str, int]: + """Successful-run count per ``'{model}|{backend}'`` service-time key.""" + counts: dict[str, int] = defaultdict(int) + for r in results: + agg = r.get("aggregate", {}) + if (agg.get("total_duration_ms", {}).get("mean") or 0) > 0: + counts[f"{r.get('model', '')}|{r.get('backend', '')}"] += _successful_runs(r) + return dict(counts) + + def bayesian_blend_throughput( existing_lookup: dict[str, float], measured_lookup: dict[str, float], n_total_runs: int, + runs_per_key: dict[str, int] | None = None, ) -> dict[str, float]: - """Blend measured throughputs with existing (global prior) using confidence weighting. + """Blend measured throughputs with existing (prior) using confidence weighting. - Confidence weight: ``w = min(1.0, n_total_runs / CONFIDENCE_RUN_THRESHOLD)``. + Confidence weight per key: ``w = min(1.0, runs_for_key / CONFIDENCE_RUN_THRESHOLD)`` + when *runs_per_key* is supplied, so each entry is weighted by ITS OWN sample + size -- not the global run total, which would over-trust a lightly-measured + config merely because other configs were also benchmarked. Falls back to the + global ``n_total_runs`` weight when *runs_per_key* is None. For each key in *measured_lookup*: - If the key exists in *existing_lookup*: @@ -292,17 +331,22 @@ def bayesian_blend_throughput( Args: existing_lookup: The current throughput lookup from fitted_models. measured_lookup: Newly measured throughput entries. - n_total_runs: Total number of individual benchmark runs (drives confidence). + n_total_runs: Global run total (fallback confidence when no per-key map). + runs_per_key: Optional per-key successful-run counts (preferred). Returns: Blended throughput lookup dict. """ - w = min(1.0, n_total_runs / CONFIDENCE_RUN_THRESHOLD) + global_w = min(1.0, n_total_runs / CONFIDENCE_RUN_THRESHOLD) blended = dict(existing_lookup) # preserve all existing entries for key, measured_val in measured_lookup.items(): existing_val = existing_lookup.get(key) if existing_val is not None: + if runs_per_key is not None: + w = min(1.0, runs_per_key.get(key, 0) / CONFIDENCE_RUN_THRESHOLD) + else: + w = global_w blended[key] = (1 - w) * existing_val + w * measured_val else: blended[key] = measured_val @@ -313,25 +357,31 @@ def _bayesian_blend_service_times( existing_st: dict[str, float], measured_st: dict[str, float], n_total_runs: int, + runs_per_key: dict[str, int] | None = None, ) -> dict[str, float]: """Blend measured service times with existing using confidence weighting. - Same formula as :func:`bayesian_blend_throughput` but for service times. + Same per-key weighting as :func:`bayesian_blend_throughput`. Args: existing_st: Current service_times from fitted_models. measured_st: Newly measured service times. - n_total_runs: Total number of individual benchmark runs. + n_total_runs: Global run total (fallback when no per-key map). + runs_per_key: Optional per-key successful-run counts (preferred). Returns: Blended service times dict. """ - w = min(1.0, n_total_runs / CONFIDENCE_RUN_THRESHOLD) + global_w = min(1.0, n_total_runs / CONFIDENCE_RUN_THRESHOLD) blended = dict(existing_st) for key, measured_val in measured_st.items(): existing_val = existing_st.get(key) if existing_val is not None: + if runs_per_key is not None: + w = min(1.0, runs_per_key.get(key, 0) / CONFIDENCE_RUN_THRESHOLD) + else: + w = global_w blended[key] = (1 - w) * existing_val + w * measured_val else: blended[key] = measured_val @@ -432,12 +482,17 @@ def refit_from_bench( st = extract_service_times(results) pl = fit_power_law(tp_lookup) - # Bayesian-blend throughput and service times with existing + # Bayesian-blend throughput and service times with existing, weighting each + # entry by its OWN successful-run count (not the global total). existing_tp_lookup = existing.get("throughput", {}).get("lookup", {}) - blended_tp = bayesian_blend_throughput(existing_tp_lookup, tp_lookup, n_total_runs) + blended_tp = bayesian_blend_throughput( + existing_tp_lookup, tp_lookup, n_total_runs, runs_per_throughput_key(results) + ) existing_st = existing.get("latency", {}).get("service_times", {}) - blended_st = _bayesian_blend_service_times(existing_st, st, n_total_runs) + blended_st = _bayesian_blend_service_times( + existing_st, st, n_total_runs, runs_per_service_key(results) + ) # Compute hardware offsets (measured / predicted ratios) hw_offsets = compute_hardware_offsets(results, existing) diff --git a/src/chimeraforge/safety/__init__.py b/src/chimeraforge/safety/__init__.py index e420a531..08877ae7 100644 --- a/src/chimeraforge/safety/__init__.py +++ b/src/chimeraforge/safety/__init__.py @@ -1,4 +1,4 @@ -"""Live safety screening — refusal-rate measurement against a running model. +"""Live safety screening - refusal-rate measurement against a running model. The planner's safety gate (``chimeraforge.planner``) makes deployment *decisions* from bundled TR134/TR142 data. This package *measures*: it runs diff --git a/src/chimeraforge/safety/classifier.py b/src/chimeraforge/safety/classifier.py index 3228d505..f98572e9 100644 --- a/src/chimeraforge/safety/classifier.py +++ b/src/chimeraforge/safety/classifier.py @@ -56,7 +56,7 @@ def classify_refusal(response: str, head_chars: int = DEFAULT_HEAD_CHARS) -> boo """Return True if *response* reads as a refusal. Inspects the first ``head_chars`` characters (refusals are front-loaded). An - empty/whitespace response is a non-refusal here — it is a failure to answer, + empty/whitespace response is a non-refusal here - it is a failure to answer, not a refusal; the runner records empties separately as warnings. """ if not response or not response.strip(): diff --git a/src/chimeraforge/safety/runner.py b/src/chimeraforge/safety/runner.py index d8510cfc..11caea04 100644 --- a/src/chimeraforge/safety/runner.py +++ b/src/chimeraforge/safety/runner.py @@ -1,8 +1,8 @@ -"""Safety screen runner — runs refusal probes against a live backend. +"""Safety screen runner - runs refusal probes against a live backend. Where the planner's safety gate makes a *decision* from bundled TR134/TR142 data, this *measures*: it sends each prompt to a running model, classifies the -response as a refusal or not, and aggregates a refusal rate — so a (model, +response as a refusal or not, and aggregates a refusal rate - so a (model, quant) the bundled table does not cover can still be screened. """ diff --git a/tests/test_accuracy.py b/tests/test_accuracy.py new file mode 100644 index 00000000..1740134d --- /dev/null +++ b/tests/test_accuracy.py @@ -0,0 +1,105 @@ +"""Numerical accuracy gates -- make the predictive models falsifiable. + +These pin predictions to ground truth (the bundled measured corpus, the roofline +calibration anchor, and first-principles arithmetic) within stated tolerance +bands. If a model change silently shifts a number, one of these fails. The point +(per the cold review) is that without numerical assertions the models are +unfalsifiable; this file is the falsifiability gate. +""" + +from __future__ import annotations + +import pytest + +from chimeraforge.planner.models import LatencyModel, VRAMModel + +REF_GPU = "RTX 4080 12GB" # the rig the bundled corpus was measured on (ratio 1.0) + + +class TestThroughputReproducesMeasured: + """On the reference GPU, predict() reproduces the measured lookup exactly.""" + + @pytest.mark.parametrize( + "model,expected", + [("llama3.2-1b", 146.33), ("llama3.2-3b", 95.86), ("qwen2.5-1.5b", 139.61)], + ) + def test_ollama_fp16_lookup(self, bundled_models, model, expected): + tps = bundled_models.throughput.predict(model, "ollama", "FP16", REF_GPU) + assert tps == pytest.approx(expected, rel=0.01) + + +class TestRooflineCalibration: + """The MBU=0.65 anchor: roofline reproduces the llama3.2-1b ollama datapoint.""" + + def test_anchor_point_within_band(self, bundled_models): + # 0.65 * 556 GB/s / (1.24B * 2 bytes) = 145.7 tok/s vs measured 146.33. + tps = bundled_models.throughput.roofline_tps(1.24, "FP16", REF_GPU) + assert tps == pytest.approx(146.33, rel=0.03) + + def test_roofline_scales_with_bandwidth(self, bundled_models): + # 4090 (1008 GB/s) vs 4080 (556) -> ~1.81x for the same model. + t4080 = bundled_models.throughput.roofline_tps(7.0, "FP16", "RTX 4080 12GB") + t4090 = bundled_models.throughput.roofline_tps(7.0, "FP16", "RTX 4090 24GB") + assert t4090 / t4080 == pytest.approx(1008.0 / 556.0, rel=0.02) + + +class TestVRAMAccuracy: + def test_formula_composes_weight_kv_activation(self, bundled_models): + # predict() must equal weight*overhead + KV + activations using the model's + # own (fitted) coefficients -- validates the formula without hardcoding a + # value that legitimately drifts on refit. + m = bundled_models.vram + arch = {"n_layers": 28, "n_kv_heads": 8, "d_head": 128} # llama3.2-3b + weight = 3.21 * 16 / 8 * m.overhead_factor + kv = m.kv_cache_gb(arch, 2048, 1) + act = m.act_coeff * arch["n_layers"] * (2048 / 1024) # linear in ctx (flash attn) + assert m.predict("llama3.2-3b", "FP16", 2048) == pytest.approx(weight + kv + act, rel=1e-3) + + def test_long_context_activation_stays_physical(self, bundled_models): + # Regression: activation memory must be O(ctx), not O(ctx^2). A quadratic + # term blew up to ~130 GB at 32k for a 3B model and spuriously failed the + # VRAM gate. Linear keeps it bounded well under the weight footprint. + v = bundled_models.vram.predict("llama3.2-3b", "FP16", 32768) + assert v < 20.0, f"32k-ctx VRAM {v:.1f} GB is unphysically large -- activation not linear?" + + def test_absolute_band(self, bundled_models): + # A 3.21B FP16 model at 2048 ctx must land in a physically sane VRAM band. + assert 6.5 < bundled_models.vram.predict("llama3.2-3b", "FP16", 2048) < 8.5 + + def test_kv_cache_byte_exact(self): + # 2(KV) * 28L * 2048 ctx * 8 kv-heads * 128 d * 2 bytes = 234,881,024 B. + kv = VRAMModel.kv_cache_gb({"n_layers": 28, "n_kv_heads": 8, "d_head": 128}, 2048, 1) + assert kv == pytest.approx(234_881_024 / 1024**3, rel=1e-6) + + +class TestTTFTAccuracy: + def test_prefill_compute_formula(self): + # 2 * 7e9 * 512 / (165.2e12 * 0.4) * 1000 = 108.5 ms on a 4090. + ttft = LatencyModel.predict_ttft_ms(7.0, 512, "RTX 4090 24GB") + assert ttft == pytest.approx(108.5, rel=0.01) + + +class TestBatchedThroughputInvariants: + """Physical invariants of the continuous-batching curve.""" + + KV = 0.05 # GB per sequence (small relative to weights) + + def test_b1_equals_single_stream(self, bundled_models): + assert bundled_models.throughput.batched_decode_tps(100.0, self.KV, 1, REF_GPU) == 100.0 + + def test_monotonic_increasing_then_saturates(self, bundled_models): + tp = bundled_models.throughput + a = tp.batched_decode_tps(100.0, self.KV, 2, REF_GPU) + b = tp.batched_decode_tps(100.0, self.KV, 8, REF_GPU) + c = tp.batched_decode_tps(100.0, self.KV, 64, REF_GPU) + assert 100.0 < a < b < c # aggregate rises with batch + # saturates below the KV-bandwidth ceiling bw*MBU/kv. + ceiling = 556.0 * 0.65 / self.KV + assert c < ceiling + + def test_per_request_rate_drops_with_batch(self, bundled_models): + # The throughput<->latency tradeoff: per-sequence rate falls as batch rises. + tp = bundled_models.throughput + agg2 = tp.batched_decode_tps(100.0, self.KV, 2, REF_GPU) + agg16 = tp.batched_decode_tps(100.0, self.KV, 16, REF_GPU) + assert agg16 / 16 < agg2 / 2 <= 100.0 diff --git a/tests/test_compare.py b/tests/test_compare.py index 900c1817..d7574b0f 100644 --- a/tests/test_compare.py +++ b/tests/test_compare.py @@ -1,4 +1,4 @@ -"""ChimeraForge Compare — unit tests. +"""ChimeraForge Compare - unit tests. Tests the compare pipeline: key generation, grouping, delta calculation, Rich table formatting, JSON output, and CLI integration. diff --git a/tests/test_eval.py b/tests/test_eval.py index 0b2cd4d7..a4c2bdfc 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -1,4 +1,4 @@ -"""ChimeraForge Eval — unit tests. +"""ChimeraForge Eval - unit tests. Tests quality metrics (exact match, ROUGE-L, BERTScore, coherence, composite, tiers), built-in tasks, runner, and CLI integration. @@ -442,7 +442,7 @@ def test_eval_json_output(self): # progress bar. In a real shell that goes to stderr (stdout stays clean # JSON), but typer's CliRunner merges the streams into result.output, and # the JSON "[" lands mid-line after the bar's carriage return. Locate the - # array start ("[" followed by a newline — tqdm's "[00:00" is not) and + # array start ("[" followed by a newline - tqdm's "[00:00" is not) and # raw_decode so any trailing noise is ignored. import re diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py index 41c030b8..bdd1dbbe 100644 --- a/tests/test_monitoring.py +++ b/tests/test_monitoring.py @@ -63,7 +63,14 @@ def fake_capture(): pm.aggregator.add_point(MetricPoint("cpu_percent", 1.0, "%")) pm.capture_snapshot = fake_capture # type: ignore + # Lifecycle: the daemon loop must start and stop cleanly... pm.start() pm.stop() + # ...then drive capture deterministically so the assertion doesn't race the + # start/stop of the background thread (which may run fake_capture zero times). + for _ in range(3): + pm.capture_snapshot() digest = pm.build_digest() assert "summary" in digest and "digest" in digest and "suggestions" in digest + # The digest must actually reflect captured data, not merely have the keys. + assert digest["summary"]["cpu_percent"]["count"] >= 1 diff --git a/tests/test_planner_cli.py b/tests/test_planner_cli.py index 1c02c2bf..82948d72 100644 --- a/tests/test_planner_cli.py +++ b/tests/test_planner_cli.py @@ -138,6 +138,41 @@ def test_plan_offregistry_manual_override_json(self): assert c["provenance"]["throughput"] == "estimated" assert c["provenance"]["safety"] == "unknown" + def test_plan_pareto(self): + from typer.testing import CliRunner + + from chimeraforge.cli import app + + result = CliRunner().invoke(app, ["plan", "--pareto", "--quality-target", "0"]) + assert result.exit_code == 0 + assert "frontier" in self._strip_ansi(result.output).lower() + + def test_plan_pareto_json(self): + from typer.testing import CliRunner + + from chimeraforge.cli import app + + result = CliRunner().invoke(app, ["plan", "--pareto", "--json", "--quality-target", "0"]) + assert result.exit_code == 0 + data = self._extract_json(result.output) + assert isinstance(data, list) and data + + def test_plan_workload_agent_accepted(self): + from typer.testing import CliRunner + + from chimeraforge.cli import app + + result = CliRunner().invoke(app, ["plan", "--workload", "agent", "--quality-target", "0"]) + assert result.exit_code == 0 + + def test_plan_bad_workload_rejected(self): + from typer.testing import CliRunner + + from chimeraforge.cli import app + + result = CliRunner().invoke(app, ["plan", "--workload", "bogus"]) + assert result.exit_code == 1 + def test_plan_overrides_require_single_model(self): from typer.testing import CliRunner from chimeraforge.cli import app diff --git a/tests/test_planner_core.py b/tests/test_planner_core.py index bdb643fc..e9701acb 100644 --- a/tests/test_planner_core.py +++ b/tests/test_planner_core.py @@ -21,7 +21,7 @@ from chimeraforge.planner.models import load_models -# ── Constants & Registry ───────────────────────────────────────────── +# -- Constants & Registry --------------------------------------------- class TestConstants: @@ -46,7 +46,7 @@ def test_model_arch_keys(self): assert arch["d_head"] > 0 -# ── Hardware DB ────────────────────────────────────────────────────── +# -- Hardware DB ------------------------------------------------------ class TestHardwareDB: @@ -83,7 +83,7 @@ def test_bandwidth_ratio_unknown(self): assert bandwidth_ratio("Unknown GPU") == 1.0 -# ── Serialization Round-Trip ───────────────────────────────────────── +# -- Serialization Round-Trip ----------------------------------------- class TestSerialization: diff --git a/tests/test_planner_engine.py b/tests/test_planner_engine.py index 40fb569b..41362b0c 100644 --- a/tests/test_planner_engine.py +++ b/tests/test_planner_engine.py @@ -15,7 +15,7 @@ from chimeraforge.planner.resolver import ModelSpec -# ── Planner Engine ─────────────────────────────────────────────────── +# -- Planner Engine --------------------------------------------------- class TestPlanner: @@ -132,7 +132,7 @@ def test_json_output(self, bundled_models): assert "monthly_cost" in data[0] -# ── Model-Agnostic Specs ───────────────────────────────────────────── +# -- Model-Agnostic Specs --------------------------------------------- class TestOffRegistrySpecs: @@ -247,6 +247,47 @@ def test_registry_alias_reuses_measured_data(self, bundled_models): assert any("approximated" in w for w in cands[0].warnings) +class TestPrefillDecodeFields: + """Candidates carry TTFT (prefill) and TPOT (decode) on a known GPU.""" + + def test_candidate_has_ttft_and_tpot(self, bundled_models): + cands = enumerate_candidates( + models=bundled_models, + target_models=["llama3.2-3b"], + hardware="RTX 4090 24GB", + request_rate=0.5, + latency_slo=10000, + quality_target=0.3, + budget=200, + avg_tokens=128, + context_length=2048, + prompt_tokens=512, + ) + c = cands[0] + assert c.ttft_ms > 0 # known GPU -> prefill computed + assert c.tpot_ms > 0 # decode per-token latency + # TPOT should be ~ 1000 / N=1 throughput. + assert c.tpot_ms == pytest.approx(1000.0 / c.throughput_tps, rel=0.05) + + def test_longer_prompt_raises_ttft(self, bundled_models): + def ttft(pt): + cs = enumerate_candidates( + models=bundled_models, + target_models=["llama3.2-3b"], + hardware="RTX 4090 24GB", + request_rate=0.5, + latency_slo=10000, + quality_target=0.3, + budget=200, + avg_tokens=128, + context_length=2048, + prompt_tokens=pt, + ) + return cs[0].ttft_ms + + assert ttft(2048) > ttft(256) + + class TestRejectionTrace: """The optional trace explains why a search returned nothing.""" @@ -270,8 +311,8 @@ def test_vram_blocked_trace(self, bundled_models): assert "vram" in gates or "quality" in gates def test_summarize_trace_picks_binding_gate(self, bundled_models): - # 14B off-registry at an extreme rate even 16 linear replicas can't serve - # -> throughput-bound (the fastest/smallest quant fits VRAM but not rate). + # 14B at an extreme rate that even 16 GPUs x max batch can't serve + # -> throughput-bound (the smallest quant fits VRAM but not the rate). spec = ModelSpec( name="big/m", params_b=14.0, n_layers=40, n_kv_heads=8, d_head=128, source="hf" ) @@ -280,10 +321,10 @@ def test_summarize_trace_picks_binding_gate(self, bundled_models): models=bundled_models, target_models=["big/m"], hardware="RTX 4090 24GB", - request_rate=30.0, # 30*128=3840 tok/s; 16 replicas of a 14B can't reach it + request_rate=500.0, # 64000 tok/s; beyond 16 GPUs even with batching latency_slo=10000, quality_target=0.0, - budget=100000, + budget=1_000_000, avg_tokens=128, context_length=2048, specs={"big/m": spec}, @@ -294,9 +335,9 @@ def test_summarize_trace_picks_binding_gate(self, bundled_models): assert any("big/m" in ln and "throughput" in ln for ln in lines) def test_linear_replica_scaling_unblocks_large_models(self, bundled_models): - # C-1: N independent GPUs scale linearly, so a 7B that one GPU can't serve - # at the rate now plans with multiple replicas (used to be rejected when - # Amdahl capped total throughput at ~1.8x regardless of N). + # C-1: a 7B that one single-stream GPU can't serve at the rate now plans + # (was rejected when Amdahl capped total throughput at ~1.8x). On the + # non-batching Ollama path that means linear replicas (eta=1). spec = ModelSpec( name="mistral/7b", params_b=7.0, n_layers=32, n_kv_heads=8, d_head=128, source="hf" ) @@ -304,7 +345,7 @@ def test_linear_replica_scaling_unblocks_large_models(self, bundled_models): models=bundled_models, target_models=["mistral/7b"], hardware="RTX 4080 12GB", - request_rate=1.0, # 128 tok/s; one 7B replica can't, several can + request_rate=1.0, latency_slo=10000, quality_target=0.0, budget=1000, @@ -312,21 +353,22 @@ def test_linear_replica_scaling_unblocks_large_models(self, bundled_models): context_length=2048, specs={"mistral/7b": spec}, ) - assert cands, "linear replica scaling should let a 7B meet 1 req/s with N>1" - c = cands[0] - assert c.n_agents > 1 - assert c.eta == 1.0 # replicas, not Amdahl - # Linear: total throughput is exactly N * per-replica. - assert c.total_throughput_tps == pytest.approx(c.n_agents * c.throughput_tps, rel=1e-3) + assert cands, "a 7B should now plan at 1 req/s (not rejected)" + ollama = next(c for c in cands if c.backend == "ollama") + assert ollama.effective_batch == 1 # Ollama = single-stream replicas + assert ollama.n_agents > 1 # needs several replicas at this rate + assert ollama.total_throughput_tps == pytest.approx( + ollama.n_agents * ollama.throughput_tps, rel=1e-3 + ) def test_cost_per_1m_invariant_in_replica_count(self, bundled_models): - # C-2: adding identical replicas must not change $/token (cost and tokens - # both scale by N). Previously cost_per_1m was understated by N. + # C-2: adding identical (single-stream) replicas must not change $/token. + # Use the Ollama path, where higher rate genuinely adds replicas. spec = ModelSpec( name="mistral/7b", params_b=7.0, n_layers=32, n_kv_heads=8, d_head=128, source="hf" ) - def best(rate): + def ollama_q4(rate): cs = enumerate_candidates( models=bundled_models, target_models=["mistral/7b"], @@ -339,12 +381,151 @@ def best(rate): context_length=2048, specs={"mistral/7b": spec}, ) - return next(c for c in cs if c.quant == "Q4_K_M") + return next(c for c in cs if c.backend == "ollama" and c.quant == "Q4_K_M") - low, high = best(1.0), best(3.0) + low, high = ollama_q4(1.0), ollama_q4(3.0) assert high.n_agents > low.n_agents # more replicas at higher rate assert high.cost_per_1m_tok == pytest.approx(low.cost_per_1m_tok, rel=1e-3) + +class TestContinuousBatching: + """0.6.0: batched backends serve concurrent requests on one GPU (vLLM/TGI).""" + + def _spec(self): + return ModelSpec( + name="m/7b", params_b=7.0, n_layers=32, n_kv_heads=8, d_head=128, source="hf" + ) + + def _plan(self, models, rate): + return enumerate_candidates( + models=models, + target_models=["m/7b"], + hardware="RTX 4090 24GB", + request_rate=rate, + latency_slo=10000, + quality_target=0.0, + budget=10000, + avg_tokens=128, + context_length=2048, + specs={"m/7b": self._spec()}, + ) + + def test_vllm_batches_ollama_does_not(self, bundled_models): + cands = self._plan(bundled_models, rate=2.0) + vllm = next(c for c in cands if c.backend == "vllm") + ollama = next(c for c in cands if c.backend == "ollama") + assert vllm.effective_batch > 1 # continuous batching + assert ollama.effective_batch == 1 # single-stream + + def test_batching_needs_fewer_gpus_than_replicas(self, bundled_models): + # At a rate one single-stream GPU can't serve, vLLM should meet it with + # fewer GPUs than Ollama (batching replaces replicas). + cands = self._plan(bundled_models, rate=3.0) + vllm = next(c for c in cands if c.backend == "vllm") + ollama = next(c for c in cands if c.backend == "ollama") + assert vllm.n_agents <= ollama.n_agents + # vLLM aggregate per the batch is well above single-stream. + assert vllm.total_throughput_tps >= ollama.total_throughput_tps + + def test_batch_bounded_by_kv_cache(self, bundled_models): + cands = self._plan(bundled_models, rate=5.0) + vllm = next(c for c in cands if c.backend == "vllm") + assert vllm.effective_batch <= vllm.max_concurrent_seqs + + +class TestVarianceGuard: + """0.6.0: high-variance (agent) workloads inflate the tail and warn.""" + + def _plan(self, models, cv2): + return enumerate_candidates( + models=models, + target_models=["llama3.2-3b"], + hardware="RTX 4080 12GB", + request_rate=0.5, + latency_slo=10000, + quality_target=0.3, + budget=300, + avg_tokens=128, + context_length=2048, + workload_cv2=cv2, + ) + + def test_agent_workload_warns(self, bundled_models): + agent = self._plan(bundled_models, 8.0) + steady = self._plan(bundled_models, 0.0) + assert any("variance" in w for w in agent[0].warnings) + assert not any("variance" in w for w in steady[0].warnings) + + def test_steady_default_unchanged(self, bundled_models): + # cv2=0 must leave candidate p95 identical to the no-arg (default) behaviour. + default = enumerate_candidates( + models=bundled_models, + target_models=["llama3.2-3b"], + hardware="RTX 4080 12GB", + request_rate=0.5, + latency_slo=10000, + quality_target=0.3, + budget=300, + avg_tokens=128, + context_length=2048, + ) + explicit = self._plan(bundled_models, 0.0) + assert default and explicit + assert explicit[0].p95_latency_ms == pytest.approx(default[0].p95_latency_ms) + + +class TestParetoFrontier: + """0.6.0: non-dominated cost/latency/quality trade-off menu.""" + + def _cand(self, cost, p95, quality, model="m"): + from chimeraforge.planner.engine import Candidate + + return Candidate( + model=model, + quant="Q4_K_M", + backend="vllm", + n_agents=1, + vram_gb=4.0, + quality=quality, + quality_tier="negligible", + throughput_tps=100.0, + total_throughput_tps=100.0, + eta=1.0, + p95_latency_ms=p95, + utilisation=0.3, + monthly_cost=cost, + cost_per_1m_tok=0.1, + safety_refusal=None, + rtsi_risk="UNKNOWN", + warnings=[], + ) + + def test_excludes_dominated(self): + from chimeraforge.planner.engine import pareto_frontier + + a = self._cand(10, 1000, 0.5, "cheap") # cheapest + b = self._cand(20, 500, 0.5, "fast") # faster, pricier -> non-dominated + c = self._cand(30, 2000, 0.5, "dom") # dominated by a (cheaper+faster, == q) + d = self._cand(40, 300, 0.7, "premium") # fastest + best quality + front = pareto_frontier([a, b, c, d]) + models = {x.model for x in front} + assert models == {"cheap", "fast", "premium"} # 'dom' excluded + assert front[0].monthly_cost <= front[-1].monthly_cost # sorted by cost + + def test_frontier_has_the_three_extremes(self): + from chimeraforge.planner.engine import pareto_frontier + + cs = [self._cand(10, 1000, 0.5), self._cand(40, 300, 0.7), self._cand(20, 500, 0.6)] + front = pareto_frontier(cs) + assert min(c.monthly_cost for c in cs) in {c.monthly_cost for c in front} + assert min(c.p95_latency_ms for c in cs) in {c.p95_latency_ms for c in front} + assert max(c.quality for c in cs) in {c.quality for c in front} + + def test_empty(self): + from chimeraforge.planner.engine import pareto_frontier + + assert pareto_frontier([]) == [] + def test_native_legacy_quant_pinned_and_costed(self, bundled_models): # M-2: a q4_0 native tag must be evaluated at q4_0 (real bpw), not dropped. spec = ModelSpec( @@ -390,7 +571,7 @@ def test_no_trace_overhead_when_none(self, bundled_models): assert cands -# ── Safety Gate (Gate 5) ───────────────────────────────────────────── +# -- Safety Gate (Gate 5) --------------------------------------------- class TestSafetyGate: @@ -444,7 +625,7 @@ def test_non_monotonic_3b_is_data_faithful(self, bundled_models): def test_unknown_safety_passes_with_warning(self, bundled_models): # qwen2.5-0.5b is in the planner registry but has no safety data. - # The gate blocks only KNOWN-unsafe cells, so it passes — with a warning. + # The gate blocks only KNOWN-unsafe cells, so it passes - with a warning. cands = self._plan(bundled_models, "qwen2.5-0.5b", safety_target=0.8) assert cands for c in cands: @@ -460,7 +641,7 @@ def test_rtsi_high_warning_on_kept_cell(self, bundled_models): assert any("RTSI" in w and "HIGH" in w for w in q2k[0].warnings) -# ── Spot Checks (Real Data Validation) ─────────────────────────────── +# -- Spot Checks (Real Data Validation) ------------------------------- class TestSpotChecks: @@ -584,7 +765,7 @@ def test_empty_target_models(self, bundled_models): assert candidates == [] def test_n_search_tries_higher_n_for_latency(self, bundled_models): - """With tight latency SLO, engine should try N > min-throughput-N.""" + """With a tight latency SLO, every returned config must actually meet it.""" candidates = enumerate_candidates( models=bundled_models, target_models=["llama3.2-3b"], @@ -596,5 +777,9 @@ def test_n_search_tries_higher_n_for_latency(self, bundled_models): avg_tokens=128, context_length=2048, ) - # The tight-latency N-search path should run and return a candidate list. - assert isinstance(candidates, list) + # The N-search must escalate replicas until the SLO holds -- so any + # candidate it returns has to satisfy the tight latency bound, not just + # be a list. (Empty is acceptable only if nothing can meet it.) + assert all(c.p95_latency_ms <= 3000 for c in candidates) + # At this rate a feasible config exists, so the search must find one. + assert candidates diff --git a/tests/test_planner_models.py b/tests/test_planner_models.py index 6a53c773..5b97d7f0 100644 --- a/tests/test_planner_models.py +++ b/tests/test_planner_models.py @@ -16,7 +16,7 @@ ) -# ── VRAM Model ─────────────────────────────────────────────────────── +# -- VRAM Model ------------------------------------------------------- class TestVRAMModel: @@ -47,7 +47,44 @@ def test_vram_positive(self, bundled_models): assert v > 0, f"VRAM should be positive for {model} {quant}" -# ── Throughput Model ───────────────────────────────────────────────── +# -- KV-cache-bound concurrency (0.6.0) ------------------------------- + + +class TestMaxConcurrentSeqs: + ARCH = {"n_layers": 28, "n_kv_heads": 8, "d_head": 128} # llama3.2-3b + + def test_kv_cache_scales_with_context_batch_and_heads(self): + base = VRAMModel.kv_cache_gb(self.ARCH, 2048, 1) + assert VRAMModel.kv_cache_gb(self.ARCH, 4096, 1) == pytest.approx(2 * base) + assert VRAMModel.kv_cache_gb(self.ARCH, 2048, 4) == pytest.approx(4 * base) + wide = {**self.ARCH, "n_kv_heads": 16} + assert VRAMModel.kv_cache_gb(wide, 2048, 1) == pytest.approx(2 * base) + + def test_bigger_gpu_holds_more_seqs(self): + m = VRAMModel() + small = m.max_concurrent_seqs(3.21, "Q4_K_M", self.ARCH, 2048, 12.0) + big = m.max_concurrent_seqs(3.21, "Q4_K_M", self.ARCH, 2048, 24.0) + assert big > small > 0 + + def test_longer_context_fewer_seqs(self): + m = VRAMModel() + short = m.max_concurrent_seqs(3.21, "Q4_K_M", self.ARCH, 1024, 24.0) + long = m.max_concurrent_seqs(3.21, "Q4_K_M", self.ARCH, 8192, 24.0) + assert short > long + + def test_weights_dont_fit_returns_zero(self): + m = VRAMModel() + # 70B FP16 (~154 GB) cannot fit a 24 GB card. + assert m.max_concurrent_seqs(70.0, "FP16", self.ARCH, 2048, 24.0) == 0 + + def test_quantization_increases_capacity(self): + m = VRAMModel() + fp16 = m.max_concurrent_seqs(3.21, "FP16", self.ARCH, 2048, 24.0) + q4 = m.max_concurrent_seqs(3.21, "Q4_K_M", self.ARCH, 2048, 24.0) + assert q4 > fp16 # smaller weights leave more VRAM for KV + + +# -- Throughput Model ------------------------------------------------- class TestThroughputModel: @@ -79,8 +116,17 @@ def test_minimum_throughput(self, bundled_models): tps = bundled_models.throughput.predict("nonexistent-model", "ollama", "FP16") assert tps >= 0.1 + def test_roofline_fp32_half_of_fp16(self, bundled_models): + # FP32 streams 2x the weight bytes per token with no dequant speedup, so + # its roofline decode rate must be ~half FP16 -- not equal (the bug: the + # nearest-bpw fallback wrongly picked FP16=1.0 for above-FP16 precision). + tp = bundled_models.throughput + f16 = tp.roofline_tps(7.0, "FP16", "RTX 4090 24GB") + f32 = tp.roofline_tps(7.0, "FP32", "RTX 4090 24GB") + assert f32 == pytest.approx(f16 * 0.5, rel=0.02) -# ── Scaling Model ──────────────────────────────────────────────────── + +# -- Scaling Model ---------------------------------------------------- class TestScalingModel: @@ -100,7 +146,7 @@ def test_unknown_model_uses_defaults(self): assert 0 < eta < 1 -# ── Quality Model ──────────────────────────────────────────────────── +# -- Quality Model ---------------------------------------------------- class TestQualityModel: @@ -117,13 +163,27 @@ def test_quality_tier_fp16(self, bundled_models): tier = bundled_models.quality.quality_tier("llama3.2-1b", "FP16") assert tier == "negligible" + def test_quality_tier_family_aware(self, bundled_models): + # An off-registry model whose family matches the registry must get a real + # tier (consistent with estimate()), not "unknown" -- so the engine's + # "concerning drop" advisory can actually fire for it. + q = bundled_models.quality + _, src = q.estimate("qwen2.5:7b", "Q3_K_S", "qwen2.5") + tier = q.quality_tier("qwen2.5:7b", "Q3_K_S", "qwen2.5") + assert src == "estimated" + assert tier != "unknown" + + def test_quality_tier_unknown_without_family(self, bundled_models): + # No name match and no family -> honestly unknown. + assert bundled_models.quality.quality_tier("totally-novel-9000", "Q4_K_M") == "unknown" + def test_unknown_model_returns_default(self): m = QualityModel() q = m.predict("nonexistent", "FP16") assert q == 0.5 -# ── Safety Model ───────────────────────────────────────────────────── +# -- Safety Model ----------------------------------------------------- class TestSafetyModel: @@ -136,7 +196,7 @@ def test_refusal_lookup_collapse_cell(self, bundled_models): assert q2k < fp16 def test_refusal_safe_cell(self, bundled_models): - # Q4_K_M holds refusal high — should clear a 0.8 safety bar. + # Q4_K_M holds refusal high - should clear a 0.8 safety bar. q4 = bundled_models.safety.predict_refusal("llama3.2-1b", "Q4_K_M") assert q4 == pytest.approx(0.905, abs=0.01) assert q4 >= 0.8 @@ -195,7 +255,7 @@ def test_empty_model_defaults(self): assert m.fitted is False -# ── Cost Model ─────────────────────────────────────────────────────── +# -- Cost Model ------------------------------------------------------- class TestCostModel: @@ -224,7 +284,7 @@ def test_override_hw_cost(self): assert cost1 > cost2 -# ── Latency Model ──────────────────────────────────────────────────── +# -- Latency Model ---------------------------------------------------- class TestLatencyModel: @@ -260,6 +320,78 @@ def test_saturated_flag(self, bundled_models): assert result["saturated"] +# -- Prefill / TTFT (0.6.0) --------------------------------------------------- + + +class TestPrefillTTFT: + def test_ttft_scales_with_params(self): + m = LatencyModel() + small = m.predict_ttft_ms(7.0, 512, "RTX 4090 24GB") + big = m.predict_ttft_ms(14.0, 512, "RTX 4090 24GB") + assert big == pytest.approx(2 * small, rel=1e-3) # linear in params + + def test_ttft_scales_with_prompt_length(self): + m = LatencyModel() + short = m.predict_ttft_ms(7.0, 512, "RTX 4090 24GB") + long = m.predict_ttft_ms(7.0, 2048, "RTX 4090 24GB") + assert long == pytest.approx(4 * short, rel=1e-3) # linear in prompt tokens + + def test_faster_gpu_lower_ttft(self): + m = LatencyModel() + slow = m.predict_ttft_ms(7.0, 512, "T4 16GB") # 65 TFLOPS + fast = m.predict_ttft_ms(7.0, 512, "H100 80GB") # 989 TFLOPS + assert fast < slow + + def test_unknown_gpu_returns_zero(self): + # No compute data -> 0.0 so the caller omits prefill rather than guessing. + assert LatencyModel().predict_ttft_ms(7.0, 512, "Some Unknown GPU") == 0.0 + + def test_zero_params_or_prompt_returns_zero(self): + m = LatencyModel() + assert m.predict_ttft_ms(0, 512, "RTX 4090 24GB") == 0.0 + assert m.predict_ttft_ms(7.0, 0, "RTX 4090 24GB") == 0.0 + + def test_p95_includes_prefill(self, bundled_models): + # Same config, with vs without a prefill term -> prefill adds to service. + base = bundled_models.latency.predict_p95( + "llama3.2-3b", "ollama", request_rate=0.01, n1_tps=100.0, avg_tokens=128 + ) + with_prefill = bundled_models.latency.predict_p95( + "llama3.2-3b", "ollama", request_rate=0.01, n1_tps=100.0, avg_tokens=128, ttft_ms=200.0 + ) + assert with_prefill["service_ms"] == pytest.approx(base["service_ms"] + 200.0, rel=1e-3) + + +# -- Variance-aware queueing (0.6.0) ------------------------------------------ + + +class TestVarianceQueueing: + def _p95(self, models, cv2): + return models.latency.predict_p95( + "llama3.2-3b", + "ollama", + request_rate=0.4, + n_agents=1, + avg_tokens=128, + n1_tps=100.0, + service_cv2=cv2, + )["p95_ms"] + + def test_cv2_zero_is_the_md1_default(self, bundled_models): + # service_cv2=0 must reproduce the prior M/D/1 behaviour (no arg = 0). + explicit = self._p95(bundled_models, 0.0) + default = bundled_models.latency.predict_p95( + "llama3.2-3b", "ollama", request_rate=0.4, n_agents=1, avg_tokens=128, n1_tps=100.0 + )["p95_ms"] + assert explicit == pytest.approx(default) + + def test_higher_variance_raises_tail(self, bundled_models): + low = self._p95(bundled_models, 0.0) + chat = self._p95(bundled_models, 1.0) + agent = self._p95(bundled_models, 8.0) + assert agent > chat > low # heavier tail with more service variance + + # -- Scaling Model Edge Cases ------------------------------------------------- diff --git a/tests/test_refit.py b/tests/test_refit.py index 480c13e5..bb43a18a 100644 --- a/tests/test_refit.py +++ b/tests/test_refit.py @@ -1,4 +1,4 @@ -"""ChimeraForge Refit — unit tests. +"""ChimeraForge Refit - unit tests. Tests the refit pipeline: loading bench results, extracting throughput lookups / quant multipliers / service times, fitting power law, @@ -35,7 +35,7 @@ def test_degenerate_inputs(self): # --------------------------------------------------------------------------- -# Helpers — synthetic bench result dicts +# Helpers - synthetic bench result dicts # --------------------------------------------------------------------------- @@ -505,6 +505,30 @@ def test_zero_runs(self): # w = 0 -> fully existing assert blended["a|b|FP16"] == pytest.approx(100.0) + def test_per_key_weight_uses_own_run_count(self): + # Each entry must be weighted by ITS OWN sample size, not the global total: + # a config measured with 50 runs gets full confidence; one with 5 stays + # near the prior -- even though both are in the same refit (global=55). + from chimeraforge.refit.fitter import bayesian_blend_throughput + + existing = {"hi|b|FP16": 100.0, "lo|b|FP16": 100.0} + measured = {"hi|b|FP16": 200.0, "lo|b|FP16": 200.0} + blended = bayesian_blend_throughput( + existing, measured, n_total_runs=55, runs_per_key={"hi|b|FP16": 50, "lo|b|FP16": 5} + ) + assert blended["hi|b|FP16"] == pytest.approx(200.0) # w=1.0 + assert blended["lo|b|FP16"] == pytest.approx(110.0) # w=0.1 -> 0.9*100 + 0.1*200 + + def test_per_key_falls_back_to_global(self): + # Without a per-key map, behaviour is the prior global-weight blend. + from chimeraforge.refit.fitter import bayesian_blend_throughput + + existing = {"a|b|FP16": 100.0} + measured = {"a|b|FP16": 200.0} + assert bayesian_blend_throughput(existing, measured, n_total_runs=25)[ + "a|b|FP16" + ] == pytest.approx(150.0) + # --------------------------------------------------------------------------- # TestHardwareOffsets diff --git a/tests/test_report.py b/tests/test_report.py index e505848a..a7bbbc34 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -1,4 +1,4 @@ -"""ChimeraForge Report — unit tests. +"""ChimeraForge Report - unit tests. Tests statistical analysis, report generation (Markdown, HTML, Rich), loading, saving, and CLI integration. diff --git a/tests/test_safety.py b/tests/test_safety.py index 21a73f14..e1c963b5 100644 --- a/tests/test_safety.py +++ b/tests/test_safety.py @@ -1,4 +1,4 @@ -"""Tests for the live safety screen — refusal classifier + screen runner.""" +"""Tests for the live safety screen - refusal classifier + screen runner.""" from __future__ import annotations @@ -8,7 +8,7 @@ from chimeraforge.safety.runner import SafetyScreenResult -# ── Refusal classifier ─────────────────────────────────────────────── +# -- Refusal classifier ----------------------------------------------- class TestRefusalClassifier: @@ -52,7 +52,7 @@ def test_pattern_count(self): assert refusal_pattern_count() >= 8 -# ── Screen runner (mocked backend, no live model) ──────────────────── +# -- Screen runner (mocked backend, no live model) -------------------- class _FakeBackend: @@ -168,7 +168,7 @@ async def test_to_dict(self, monkeypatch): assert d["refusal_rate"] == 1.0 and d["n_prompts"] == 2 -# ── CLI command (fail-loud paths + mocked success) ─────────────────── +# -- CLI command (fail-loud paths + mocked success) ------------------- async def _stub_screen( @@ -273,7 +273,7 @@ def test_comparison_resolves_ollama_tag(self, tmp_path, monkeypatch): assert data["rtsi_risk"] == "HIGH" -# ── Model identity / resolution ────────────────────────────────────── +# -- Model identity / resolution -------------------------------------- class TestModelResolution: @@ -296,10 +296,14 @@ def test_resolve(self, identifier, expected): assert resolve_model(identifier) == expected - @pytest.mark.parametrize("identifier", ["gpt-4o", "mystery-model", "gemma2:9b", ""]) + @pytest.mark.parametrize( + "identifier", ["gpt-4o", "mystery-model", "gemma2:9b", "", "qwen2.5-0b"] + ) def test_unresolvable(self, identifier): from chimeraforge.planner.identity import resolve_model + # "qwen2.5-0b": a degenerate 0-param token must return None, not raise + # ZeroDivisionError on the relative-error division. assert resolve_model(identifier) is None @pytest.mark.parametrize(