Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[all]"
pip install -e ".[all,dev]"

- name: Lint
run: |
Expand Down
21 changes: 21 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

## [0.6.1] - 2026-07-04

### Fixed
- **Clean errors for missing extras.** `bench` / `measure` / `safety` (and
`plan --measure`) now fail with a clear `install "chimeraforge[bench|safety]"`
message instead of a raw `ModuleNotFoundError` traceback when the serving
backends' `httpx` dependency is absent (the backends import it at module load).
- Install-hint error messages no longer have their `[extra]` swallowed by Rich
markup (the resolver hint rendered as `pip install chimeraforge`, dropping
`[resolve]`); dynamic error text is now escaped.

### Changed
- **Corrected optional-dependency groups.** `[bench]` dropped `psutil`, `pyyaml`,
and `structlog` (none are imported by the shipped `chimeraforge` package) and
added `pynvml` (used for GPU environment metadata, previously undeclared, so
`[bench]` silently lacked it). `psutil` and `structlog` moved to `[dev]` (they
are test-only, for the `banterhearts` monitoring subsystem the suite exercises);
`pyyaml` removed entirely (unused). `[refit]` added `platformdirs` (used by its
output-path resolution). `[all]` no longer pulls the `dev` tools (pytest/ruff)
onto end users; CI installs `.[all,dev]`.

## [0.6.0] - 2026-06-25

State-of-the-art serving model: the planner now reflects how LLM inference
Expand Down
4 changes: 2 additions & 2 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

ChimeraForge is an LLM inference benchmarking and deployment planning platform, broken out from the Banterhearts program. It provides quantified, reproducible answers to LLM deployment decisions, backed by ~204,000 real measurements on consumer GPUs. Ships both research artifacts (32 technical reports, TR108-TR137 + TR142/TR146) and production CLI tools (`chimeraforge plan` and `chimeraforge bench`).

**Version:** 0.6.0 | **License:** MIT | **Python:** >=3.10 | **Rust:** >=1.70
**Version:** 0.6.1 | **License:** MIT | **Python:** >=3.10 | **Rust:** >=1.70

## Quick Reference

Expand Down Expand Up @@ -53,7 +53,7 @@ cd src/rust/demo_multiagent && cargo build --release
```
src/
chimeraforge/ # CLI tool + capacity planner (pip-installable)
__init__.py # Exports __version__ = "0.6.0"
__init__.py # Exports __version__ = "0.6.1"
cli.py # Typer entry point, registers plan/suggest/safety/... (lazy imports)
commands/ # One module per CLI command (plan.py, suggest.py, safety.py, ...)
planner/
Expand Down
10 changes: 5 additions & 5 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "chimeraforge"
version = "0.6.0"
version = "0.6.1"
description = "LLM deployment optimizer (performance, cost, and safety) — backed by ~204,000 real measurements on consumer GPUs"
readme = "README.md"
requires-python = ">=3.10"
Expand Down Expand Up @@ -35,13 +35,13 @@ Homepage = "https://github.com/Sahil170595/Chimeraforge"
Repository = "https://github.com/Sahil170595/Chimeraforge"

[project.optional-dependencies]
bench = ["psutil>=5.9", "pyyaml>=6.0,<7.0", "httpx>=0.25", "platformdirs>=4.0,<5.0", "structlog>=23.0"]
bench = ["httpx>=0.25", "platformdirs>=4.0,<5.0", "pynvml>=11.0"]
eval = ["evaluate>=0.4"]
refit = ["numpy>=1.24,<3.0", "scipy>=1.11,<2.0"]
refit = ["numpy>=1.24,<3.0", "scipy>=1.11,<2.0", "platformdirs>=4.0,<5.0"]
safety = ["httpx>=0.25"]
resolve = ["httpx>=0.25"]
dev = ["pytest>=7.4,<9.0", "pytest-cov>=4.1,<6.0", "pytest-asyncio>=0.21,<1.0", "ruff>=0.3,<1.0"]
all = ["chimeraforge[bench,eval,refit,safety,resolve,dev]"]
dev = ["pytest>=7.4,<9.0", "pytest-cov>=4.1,<6.0", "pytest-asyncio>=0.21,<1.0", "ruff>=0.3,<1.0", "psutil>=5.9", "structlog>=23.0"]
all = ["chimeraforge[bench,eval,refit,safety,resolve]"]

[tool.setuptools.packages.find]
where = ["src"]
Expand Down
2 changes: 1 addition & 1 deletion src/chimeraforge/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
inference deployments.
"""

__version__ = "0.6.0"
__version__ = "0.6.1"
36 changes: 36 additions & 0 deletions src/chimeraforge/commands/_deps.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
"""Shared helpers for clean optional-dependency errors in CLI commands.

The serving backends import third-party deps (``httpx``) at module load, so a
command that needs an optional extra would otherwise surface a raw
``ModuleNotFoundError`` traceback instead of a clear "install the extra"
message. ``require_extra`` checks availability *without* importing (so no
traceback) and fails loud-and-clean, matching the resolver's behaviour.
"""

from __future__ import annotations

import importlib.util

import typer
from rich.console import Console
from rich.markup import escape

_console = Console()


def require_extra(extra: str, *modules: str) -> None:
"""Exit cleanly (code 1) if any *modules* for an optional *extra* is missing.

Args:
extra: The optional-dependency group name (e.g. ``"bench"``).
modules: Import names the command needs (e.g. ``"httpx"``).
"""
missing = [m for m in modules if importlib.util.find_spec(m) is None]
if missing:
# escape() so Rich does not swallow the ``[extra]`` as a style tag.
hint = escape(f'pip install "chimeraforge[{extra}]"')
_console.print(
f"[red]Error:[/] this command needs the '{extra}' extra "
f"({', '.join(missing)} not installed). {hint}"
)
raise typer.Exit(code=1)
4 changes: 4 additions & 0 deletions src/chimeraforge/commands/bench.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ def bench(
from rich.progress import Progress, SpinnerColumn, TextColumn
from rich.table import Table

from chimeraforge.commands._deps import require_extra

require_extra("bench", "httpx") # backends import httpx at module load

from chimeraforge.bench.metrics import result_to_dict
from chimeraforge.bench.runner import (
run_benchmark as _run_benchmark,
Expand Down
4 changes: 4 additions & 0 deletions src/chimeraforge/commands/measure.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ def measure(

from rich.progress import Progress, SpinnerColumn, TextColumn

from chimeraforge.commands._deps import require_extra

require_extra("bench", "httpx") # measure runs through the bench backends (httpx)

from chimeraforge.measure import measure_model

logging.basicConfig(
Expand Down
9 changes: 7 additions & 2 deletions src/chimeraforge/commands/plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import typer
from rich.console import Console
from rich.markup import escape

console = Console()

Expand Down Expand Up @@ -227,14 +228,18 @@ def plan(
if measure_first and model:
import asyncio

from chimeraforge.commands._deps import require_extra

require_extra("bench", "httpx") # --measure runs the bench backends (httpx)

from chimeraforge.measure import measure_model

for ident in model:
console.print(f"[dim]Measuring {ident} on ollama (live)...[/]")
try:
mres = asyncio.run(measure_model(ident, backend="ollama", ollama_url=ollama_url))
except RuntimeError as exc:
console.print(f"[red]Error measuring '{ident}':[/] {exc}")
console.print(f"[red]Error measuring '{escape(ident)}':[/] {escape(str(exc))}")
raise typer.Exit(code=1)
console.print(
f"[green]Measured[/] {ident}: {mres.tps_n1} tok/s"
Expand Down Expand Up @@ -279,7 +284,7 @@ def plan(
allow_network=not no_network,
)
except ResolverError as exc:
console.print(f"[red]Error resolving '{ident}':[/] {exc}")
console.print(f"[red]Error resolving '{escape(ident)}':[/] {escape(str(exc))}")
raise typer.Exit(code=1)
specs[ident] = spec
if not output_json:
Expand Down
4 changes: 4 additions & 0 deletions src/chimeraforge/commands/safety.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,10 @@ def safety(
console.print(f"[red]Error:[/] prompts file '{prompts_file}' has no non-empty lines.")
raise typer.Exit(code=1)

from chimeraforge.commands._deps import require_extra

require_extra("safety", "httpx") # backends import httpx at module load

from chimeraforge.safety import run_safety_screen

try:
Expand Down
4 changes: 3 additions & 1 deletion src/chimeraforge/commands/suggest.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,9 @@ def suggest(
live_sources, ollama_url=eff_ollama_url, hf_token=hf_token, hf_limit=hf_limit
)
except ResolverError as exc:
console.print(f"[red]Error:[/] {exc}")
from rich.markup import escape

console.print(f"[red]Error:[/] {escape(str(exc))}")
raise typer.Exit(code=1)
if identifiers:
with console.status(f"Resolving {len(identifiers)} models..."):
Expand Down
37 changes: 37 additions & 0 deletions tests/test_cli_fail_loud.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,13 @@

from __future__ import annotations

import importlib.util
import io
from pathlib import Path

import pytest
import typer
from rich.console import Console
from typer.testing import CliRunner

from chimeraforge.cli import app
Expand Down Expand Up @@ -80,3 +85,35 @@ def test_unknown_backend_fails_clean(self):
r = runner.invoke(app, ["bench", "--model", "x", "--backend", "nope"])
assert r.exit_code == 1
assert not isinstance(r.exception, LEAKED)


class TestRequireExtra:
"""A missing optional extra must fail loud (exit 1) with a clean hint --
not a raw ModuleNotFoundError from a backend's module-level `import httpx`."""

def test_missing_dep_exits_clean(self):
from chimeraforge.commands._deps import require_extra

with pytest.raises(typer.Exit) as ei:
require_extra("bench", "a_module_that_does_not_exist_xyz")
assert ei.value.exit_code == 1

def test_present_dep_passes(self):
from chimeraforge.commands._deps import require_extra

require_extra("bench", "typer") # always installed -> no raise

def test_hint_shows_bracketed_extra(self, monkeypatch):
# The install hint must contain the literal 'chimeraforge[bench]' --
# Rich must not swallow the [bench] as a style tag.
from chimeraforge.commands import _deps

buf = io.StringIO()
monkeypatch.setattr(_deps, "_console", Console(file=buf, width=200))
real = importlib.util.find_spec
monkeypatch.setattr(
importlib.util, "find_spec", lambda m: None if m == "httpx" else real(m)
)
with pytest.raises(typer.Exit):
_deps.require_extra("bench", "httpx")
assert "chimeraforge[bench]" in buf.getvalue()
Loading