diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 428b0c63..a686ce6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ".[all]" + pip install -e ".[all,dev]" - name: Lint run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index a3068fe8..d001da6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.6.1] - 2026-07-04 + +### Fixed +- **Clean errors for missing extras.** `bench` / `measure` / `safety` (and + `plan --measure`) now fail with a clear `install "chimeraforge[bench|safety]"` + message instead of a raw `ModuleNotFoundError` traceback when the serving + backends' `httpx` dependency is absent (the backends import it at module load). +- Install-hint error messages no longer have their `[extra]` swallowed by Rich + markup (the resolver hint rendered as `pip install chimeraforge`, dropping + `[resolve]`); dynamic error text is now escaped. + +### Changed +- **Corrected optional-dependency groups.** `[bench]` dropped `psutil`, `pyyaml`, + and `structlog` (none are imported by the shipped `chimeraforge` package) and + added `pynvml` (used for GPU environment metadata, previously undeclared, so + `[bench]` silently lacked it). `psutil` and `structlog` moved to `[dev]` (they + are test-only, for the `banterhearts` monitoring subsystem the suite exercises); + `pyyaml` removed entirely (unused). `[refit]` added `platformdirs` (used by its + output-path resolution). `[all]` no longer pulls the `dev` tools (pytest/ruff) + onto end users; CI installs `.[all,dev]`. + ## [0.6.0] - 2026-06-25 State-of-the-art serving model: the planner now reflects how LLM inference diff --git a/CLAUDE.md b/CLAUDE.md index f3fa6ba1..be79f3ff 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ ChimeraForge is an LLM inference benchmarking and deployment planning platform, broken out from the Banterhearts program. It provides quantified, reproducible answers to LLM deployment decisions, backed by ~204,000 real measurements on consumer GPUs. Ships both research artifacts (32 technical reports, TR108-TR137 + TR142/TR146) and production CLI tools (`chimeraforge plan` and `chimeraforge bench`). -**Version:** 0.6.0 | **License:** MIT | **Python:** >=3.10 | **Rust:** >=1.70 +**Version:** 0.6.1 | **License:** MIT | **Python:** >=3.10 | **Rust:** >=1.70 ## Quick Reference @@ -53,7 +53,7 @@ cd src/rust/demo_multiagent && cargo build --release ``` src/ chimeraforge/ # CLI tool + capacity planner (pip-installable) - __init__.py # Exports __version__ = "0.6.0" + __init__.py # Exports __version__ = "0.6.1" cli.py # Typer entry point, registers plan/suggest/safety/... (lazy imports) commands/ # One module per CLI command (plan.py, suggest.py, safety.py, ...) planner/ diff --git a/pyproject.toml b/pyproject.toml index 33a4cbcc..70d99143 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "chimeraforge" -version = "0.6.0" +version = "0.6.1" description = "LLM deployment optimizer (performance, cost, and safety) — backed by ~204,000 real measurements on consumer GPUs" readme = "README.md" requires-python = ">=3.10" @@ -35,13 +35,13 @@ Homepage = "https://github.com/Sahil170595/Chimeraforge" Repository = "https://github.com/Sahil170595/Chimeraforge" [project.optional-dependencies] -bench = ["psutil>=5.9", "pyyaml>=6.0,<7.0", "httpx>=0.25", "platformdirs>=4.0,<5.0", "structlog>=23.0"] +bench = ["httpx>=0.25", "platformdirs>=4.0,<5.0", "pynvml>=11.0"] eval = ["evaluate>=0.4"] -refit = ["numpy>=1.24,<3.0", "scipy>=1.11,<2.0"] +refit = ["numpy>=1.24,<3.0", "scipy>=1.11,<2.0", "platformdirs>=4.0,<5.0"] safety = ["httpx>=0.25"] resolve = ["httpx>=0.25"] -dev = ["pytest>=7.4,<9.0", "pytest-cov>=4.1,<6.0", "pytest-asyncio>=0.21,<1.0", "ruff>=0.3,<1.0"] -all = ["chimeraforge[bench,eval,refit,safety,resolve,dev]"] +dev = ["pytest>=7.4,<9.0", "pytest-cov>=4.1,<6.0", "pytest-asyncio>=0.21,<1.0", "ruff>=0.3,<1.0", "psutil>=5.9", "structlog>=23.0"] +all = ["chimeraforge[bench,eval,refit,safety,resolve]"] [tool.setuptools.packages.find] where = ["src"] diff --git a/src/chimeraforge/__init__.py b/src/chimeraforge/__init__.py index 722b970c..498b97bf 100644 --- a/src/chimeraforge/__init__.py +++ b/src/chimeraforge/__init__.py @@ -5,4 +5,4 @@ inference deployments. """ -__version__ = "0.6.0" +__version__ = "0.6.1" diff --git a/src/chimeraforge/commands/_deps.py b/src/chimeraforge/commands/_deps.py new file mode 100644 index 00000000..f4bc83e2 --- /dev/null +++ b/src/chimeraforge/commands/_deps.py @@ -0,0 +1,36 @@ +"""Shared helpers for clean optional-dependency errors in CLI commands. + +The serving backends import third-party deps (``httpx``) at module load, so a +command that needs an optional extra would otherwise surface a raw +``ModuleNotFoundError`` traceback instead of a clear "install the extra" +message. ``require_extra`` checks availability *without* importing (so no +traceback) and fails loud-and-clean, matching the resolver's behaviour. +""" + +from __future__ import annotations + +import importlib.util + +import typer +from rich.console import Console +from rich.markup import escape + +_console = Console() + + +def require_extra(extra: str, *modules: str) -> None: + """Exit cleanly (code 1) if any *modules* for an optional *extra* is missing. + + Args: + extra: The optional-dependency group name (e.g. ``"bench"``). + modules: Import names the command needs (e.g. ``"httpx"``). + """ + missing = [m for m in modules if importlib.util.find_spec(m) is None] + if missing: + # escape() so Rich does not swallow the ``[extra]`` as a style tag. + hint = escape(f'pip install "chimeraforge[{extra}]"') + _console.print( + f"[red]Error:[/] this command needs the '{extra}' extra " + f"({', '.join(missing)} not installed). {hint}" + ) + raise typer.Exit(code=1) diff --git a/src/chimeraforge/commands/bench.py b/src/chimeraforge/commands/bench.py index 2a361ad8..1a0a7f6b 100644 --- a/src/chimeraforge/commands/bench.py +++ b/src/chimeraforge/commands/bench.py @@ -86,6 +86,10 @@ def bench( from rich.progress import Progress, SpinnerColumn, TextColumn from rich.table import Table + from chimeraforge.commands._deps import require_extra + + require_extra("bench", "httpx") # backends import httpx at module load + from chimeraforge.bench.metrics import result_to_dict from chimeraforge.bench.runner import ( run_benchmark as _run_benchmark, diff --git a/src/chimeraforge/commands/measure.py b/src/chimeraforge/commands/measure.py index 9761fd9b..4f6b73f1 100644 --- a/src/chimeraforge/commands/measure.py +++ b/src/chimeraforge/commands/measure.py @@ -46,6 +46,10 @@ def measure( from rich.progress import Progress, SpinnerColumn, TextColumn + from chimeraforge.commands._deps import require_extra + + require_extra("bench", "httpx") # measure runs through the bench backends (httpx) + from chimeraforge.measure import measure_model logging.basicConfig( diff --git a/src/chimeraforge/commands/plan.py b/src/chimeraforge/commands/plan.py index 2facf5eb..b02b9548 100644 --- a/src/chimeraforge/commands/plan.py +++ b/src/chimeraforge/commands/plan.py @@ -4,6 +4,7 @@ import typer from rich.console import Console +from rich.markup import escape console = Console() @@ -227,6 +228,10 @@ def plan( if measure_first and model: import asyncio + from chimeraforge.commands._deps import require_extra + + require_extra("bench", "httpx") # --measure runs the bench backends (httpx) + from chimeraforge.measure import measure_model for ident in model: @@ -234,7 +239,7 @@ def plan( try: mres = asyncio.run(measure_model(ident, backend="ollama", ollama_url=ollama_url)) except RuntimeError as exc: - console.print(f"[red]Error measuring '{ident}':[/] {exc}") + console.print(f"[red]Error measuring '{escape(ident)}':[/] {escape(str(exc))}") raise typer.Exit(code=1) console.print( f"[green]Measured[/] {ident}: {mres.tps_n1} tok/s" @@ -279,7 +284,7 @@ def plan( allow_network=not no_network, ) except ResolverError as exc: - console.print(f"[red]Error resolving '{ident}':[/] {exc}") + console.print(f"[red]Error resolving '{escape(ident)}':[/] {escape(str(exc))}") raise typer.Exit(code=1) specs[ident] = spec if not output_json: diff --git a/src/chimeraforge/commands/safety.py b/src/chimeraforge/commands/safety.py index e2d29546..5a64638c 100644 --- a/src/chimeraforge/commands/safety.py +++ b/src/chimeraforge/commands/safety.py @@ -89,6 +89,10 @@ def safety( console.print(f"[red]Error:[/] prompts file '{prompts_file}' has no non-empty lines.") raise typer.Exit(code=1) + from chimeraforge.commands._deps import require_extra + + require_extra("safety", "httpx") # backends import httpx at module load + from chimeraforge.safety import run_safety_screen try: diff --git a/src/chimeraforge/commands/suggest.py b/src/chimeraforge/commands/suggest.py index 2f5189be..227560be 100644 --- a/src/chimeraforge/commands/suggest.py +++ b/src/chimeraforge/commands/suggest.py @@ -105,7 +105,9 @@ def suggest( live_sources, ollama_url=eff_ollama_url, hf_token=hf_token, hf_limit=hf_limit ) except ResolverError as exc: - console.print(f"[red]Error:[/] {exc}") + from rich.markup import escape + + console.print(f"[red]Error:[/] {escape(str(exc))}") raise typer.Exit(code=1) if identifiers: with console.status(f"Resolving {len(identifiers)} models..."): diff --git a/tests/test_cli_fail_loud.py b/tests/test_cli_fail_loud.py index 54bce67a..234876dc 100644 --- a/tests/test_cli_fail_loud.py +++ b/tests/test_cli_fail_loud.py @@ -8,8 +8,13 @@ from __future__ import annotations +import importlib.util +import io from pathlib import Path +import pytest +import typer +from rich.console import Console from typer.testing import CliRunner from chimeraforge.cli import app @@ -80,3 +85,35 @@ def test_unknown_backend_fails_clean(self): r = runner.invoke(app, ["bench", "--model", "x", "--backend", "nope"]) assert r.exit_code == 1 assert not isinstance(r.exception, LEAKED) + + +class TestRequireExtra: + """A missing optional extra must fail loud (exit 1) with a clean hint -- + not a raw ModuleNotFoundError from a backend's module-level `import httpx`.""" + + def test_missing_dep_exits_clean(self): + from chimeraforge.commands._deps import require_extra + + with pytest.raises(typer.Exit) as ei: + require_extra("bench", "a_module_that_does_not_exist_xyz") + assert ei.value.exit_code == 1 + + def test_present_dep_passes(self): + from chimeraforge.commands._deps import require_extra + + require_extra("bench", "typer") # always installed -> no raise + + def test_hint_shows_bracketed_extra(self, monkeypatch): + # The install hint must contain the literal 'chimeraforge[bench]' -- + # Rich must not swallow the [bench] as a style tag. + from chimeraforge.commands import _deps + + buf = io.StringIO() + monkeypatch.setattr(_deps, "_console", Console(file=buf, width=200)) + real = importlib.util.find_spec + monkeypatch.setattr( + importlib.util, "find_spec", lambda m: None if m == "httpx" else real(m) + ) + with pytest.raises(typer.Exit): + _deps.require_extra("bench", "httpx") + assert "chimeraforge[bench]" in buf.getvalue()