From 9ecc099890e0fb700fee6c952fbb7edc9deccb21 Mon Sep 17 00:00:00 2001 From: Rafael Pini Date: Wed, 15 Jul 2026 18:00:14 +0000 Subject: [PATCH 1/2] feat(registry): add GPT-5.6 capability sheets (sol/terra/luna) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit model-identity.toml already registers the three GPT-5.6 tiers, but registry/model-capabilities/ had no sheets for them — gpt-5.5.toml was the newest OpenAI entry. This fills that gap so the family Ringer can already name is also one it can reason about. Each sheet follows the gpt-5.5.toml section layout (model/api/caching/ tool_calling/reasoning/limits/openrouter/pricing/sources) and carries a [[sources]] entry per claim with URL, quote, and access date. Sourcing is deliberately conservative — where OpenAI does not document something, the sheet says so rather than guessing: - Per-tier reasoning effort ceilings (Sol alone exposing `max`/`ultra`) are marked harness-observed via the Codex CLI config surface, not OpenAI docs, which confirm reasoning token support but never enumerate the effort scale. - Luna's [limits].long_context_caveat records that no OpenAI or third-party tier-by-tier long-context recall benchmark could be found (searched 2026-07-14), so the conservative slot for Luna is short-context work. It is flagged as an operational caveat, explicitly not a sourced benchmark. - The GA date (2026-07-09) is third-party; the model pages carry the Feb 16 2026 knowledge cutoff but no in-page GA date. Pricing is as_of 2026-07-14 from the OpenAI pricing page. Co-Authored-By: Claude Opus 4.8 (1M context) --- registry/model-capabilities/gpt-5.6-luna.toml | 102 +++++++++++++++++ registry/model-capabilities/gpt-5.6-sol.toml | 106 ++++++++++++++++++ .../model-capabilities/gpt-5.6-terra.toml | 105 +++++++++++++++++ 3 files changed, 313 insertions(+) create mode 100644 registry/model-capabilities/gpt-5.6-luna.toml create mode 100644 registry/model-capabilities/gpt-5.6-sol.toml create mode 100644 registry/model-capabilities/gpt-5.6-terra.toml diff --git a/registry/model-capabilities/gpt-5.6-luna.toml b/registry/model-capabilities/gpt-5.6-luna.toml new file mode 100644 index 0000000..4237328 --- /dev/null +++ b/registry/model-capabilities/gpt-5.6-luna.toml @@ -0,0 +1,102 @@ +[model] +key = "gpt-5.6-luna" +display = "GPT-5.6 Luna" +vendor = "OpenAI" +release_date = "2026-07-09" +identity_verification = "OpenAI's developer model page for gpt-5.6-luna lists it as the fastest/most-affordable member of the GPT-5.6 family (Sol/Terra/Luna) with a 1,050,000 context window, 128,000 max output tokens, reasoning token support, and a Feb 16, 2026 knowledge cutoff (fine-tuning listed as Not supported). GA of the GPT-5.6 family is widely reported as 2026-07-09 (third-party); the model page carries the Feb 16 2026 knowledge cutoff but no in-page GA date." + +[api] +endpoint_families = ["responses", "chat-completions", "batch"] +auth = "In this setup, Codex CLI signs in with ChatGPT/OAuth plan credentials and uses ChatGPT workspace permissions, plan access, and ChatGPT-managed Codex entitlements. Codex CLI can also sign in with an OpenAI API key; API-key usage is billed through the OpenAI Platform account at standard API rates instead of included ChatGPT plan credits." +streaming = "Supported. API streaming is available with stream=true on Responses and Chat Completions; streaming responses use server-sent events. Luna targets high-throughput/low-latency use; the model page lists fine-tuning as Not supported for this tier." + +[caching] +supported = true +mode = "implicit" +how_enabled = "Prompt caching works automatically for eligible requests with no code changes. GPT-5.6 introduces more predictable prompt caching, including support for explicit cache breakpoints. Caching starts at 1024 prompt tokens; repeated prompt prefixes, tools, and structured-output schemas can contribute to the cached prefix." +ttl = "GPT-5.6 introduces a 30-minute minimum cache life (per the openai.com GPT-5.6 overview), a change from the GPT-5.5 24h prompt_cache_retention model. Explicit cache breakpoints let callers mark segments to cache and reuse." +read_pricing_per_m = 0.10 +write_pricing_per_m = 1.25 +min_cacheable_prefix_tokens = 1024 + +[tool_calling] +format = "OpenAI Responses API tools and function calls with JSON Schema tool parameters. Chat Completions function calling is also supported. Structured Outputs are supported, including strict JSON Schema mode with strict=true. Family-level GPT-5.6 behavior inherited from the OpenAI platform; not separately re-verified on the per-tier model page." +parallel = true +strict_json_schema = true +quirks = [ + "For reasoning models using function calling through the Responses API, pass back reasoning items, function call items, and function call output items during tool loops.", + "Prompt-only JSON mode can still produce incomplete JSON when output is cut off by max_output_tokens or a content filter; use Structured Outputs for schema adherence.", + "Fine-tuning is listed as Not supported on the Luna model page.", +] + +[reasoning] +params = ["reasoning.effort", "text.verbosity", "max_output_tokens", "model_reasoning_effort", "model_reasoning_summary", "model_verbosity"] +defaults = "The gpt-5.6-luna model page confirms reasoning token support but does not enumerate the effort scale. Luna is the fast/cheap tier, routed at low/medium/high via `engine_args [\"-c\",\"model_reasoning_effort=...\"]`; `max` has not been observed on any tier but Sol. Harness-observed through the Codex CLI config surface, NOT OpenAI public model docs — treat the per-tier effort ceiling as such." +max_output_interaction = "max_output_tokens limits total generated tokens, including reasoning tokens, visible output tokens, and non-visible formatting tokens. If the cap is exhausted during reasoning, the response can be incomplete before any visible output is produced." + +[limits] +context_window = 1050000 +max_output_tokens = 128000 +long_context_caveat = "OPERATIONAL CAVEAT, NOT A SOURCED BENCHMARK: Luna's nominal context window is 1.05M, the same as Sol and Terra, but a 2026-07-14 search found no tier-by-tier long-context recall benchmark from OpenAI or any third party — so per-tier long-context behavior is unverified in both directions. Until a scoreboard accumulates real evidence, the conservative slot for Luna is short-context, low-stakes, latency-sensitive work." +rate_limits = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request (per the gpt-5.6-luna model page). Knowledge cutoff Feb 16, 2026. ChatGPT-plan Codex OAuth usage is governed by plan entitlements, not API billing tiers; per-tier long-context API rate-limit tiers are not published on the model page." + +[openrouter] +upstream_providers = ["openai"] +require_parameters_behavior = "This registry entry is for OpenAI direct/Codex OAuth access. If served via OpenRouter, the model slug would pass through the opencode engine's model field." + +[pricing] +prompt_per_m = 1.00 +completion_per_m = 6.00 +cache_read_per_m = 0.10 +cache_write_per_m = 1.25 +as_of = "2026-07-14" + +[[sources]] +claim = "model: gpt-5.6-luna context window" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-luna" +accessed = "2026-07-14" +quote = "1,050,000 context window" + +[[sources]] +claim = "model: gpt-5.6-luna max output tokens" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-luna" +accessed = "2026-07-14" +quote = "128,000 max output tokens" + +[[sources]] +claim = "pricing: gpt-5.6-luna input, cached input, and output per 1M" +url = "https://developers.openai.com/api/docs/pricing" +accessed = "2026-07-14" +quote = "Input $1.00 Cached input $0.10 Output $6.00" + +[[sources]] +claim = "limits: gpt-5.6-luna large-prompt (>272K) pricing rule" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-luna" +accessed = "2026-07-14" +quote = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request" + +[[sources]] +claim = "model: gpt-5.6-luna knowledge cutoff and fine-tuning support" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-luna" +accessed = "2026-07-14" +quote = "Feb 16, 2026 knowledge cutoff" + +[[sources]] +claim = "caching: GPT-5.6 introduces explicit cache breakpoints and a 30-minute minimum cache life" +url = "https://openai.com/index/gpt-5-6/" +accessed = "2026-07-14" +quote = "more predictable prompt caching, including support for explicit cache breakpoints and a 30-minute minimum cache life" +note = "openai.com/index/gpt-5-6 returned HTTP 403 to direct fetch; quote relayed via Perplexity sonar-pro search 2026-07-14. The 90% cache-read discount is independently corroborated by the pricing page ($0.10 cached = 0.1x $1.00 input)." + +[[sources]] +claim = "long_context: Luna recall-cliff figures are UNVERIFIED (no reproducible benchmark)" +url = "https://www.mindstudio.ai/blog/what-is-gpt-5-6-openai-three-model-tiers" +accessed = "2026-07-14" +quote = "Luna is the fastest and most affordable model" +note = "A 2026-07-14 search found no OpenAI or third-party tier-by-tier long-context recall benchmark for the GPT-5.6 family. Luna's long-context behavior relative to Terra/Sol is therefore unverified; the sheet carries this as an operational caveat only, not a sourced claim." + +[[sources]] +claim = "release: GPT-5.6 family (Sol/Terra/Luna) GA date (third-party, corroborated)" +url = "https://www.digitalapplied.com/blog/gpt-5-6-sol-terra-luna-public-ga" +accessed = "2026-07-14" +quote = "GPT-5.6 went to general availability on July 9, 2026 — across ChatGPT, Codex, and the OpenAI API" diff --git a/registry/model-capabilities/gpt-5.6-sol.toml b/registry/model-capabilities/gpt-5.6-sol.toml new file mode 100644 index 0000000..7d6eeae --- /dev/null +++ b/registry/model-capabilities/gpt-5.6-sol.toml @@ -0,0 +1,106 @@ +[model] +key = "gpt-5.6-sol" +display = "GPT-5.6 Sol" +vendor = "OpenAI" +release_date = "2026-07-09" +identity_verification = "OpenAI's developer model page for gpt-5.6-sol lists it as the flagship member of the GPT-5.6 family (Sol/Terra/Luna) with a 1,050,000 context window, 128,000 max output tokens, reasoning token support, and a Feb 16, 2026 knowledge cutoff. GA of the GPT-5.6 family is widely reported as 2026-07-09 (third-party); the model page carries the Feb 16 2026 knowledge cutoff but no in-page GA date." + +[api] +endpoint_families = ["responses", "chat-completions", "batch"] +auth = "In this setup, Codex CLI signs in with ChatGPT/OAuth plan credentials and uses ChatGPT workspace permissions, plan access, and ChatGPT-managed Codex entitlements. Codex CLI can also sign in with an OpenAI API key; API-key usage is billed through the OpenAI Platform account at standard API rates instead of included ChatGPT plan credits." +streaming = "Supported. API streaming is available with stream=true on Responses and Chat Completions; streaming responses use server-sent events. The Sol model page also lists Realtime, Assistants, Fine-tuning, Embeddings, and image/audio endpoints among the broadest endpoint set in the family; Ringer uses Responses/Chat via Codex." + +[caching] +supported = true +mode = "implicit" +how_enabled = "Prompt caching works automatically for eligible requests with no code changes. GPT-5.6 introduces more predictable prompt caching, including support for explicit cache breakpoints. Caching starts at 1024 prompt tokens; repeated prompt prefixes, tools, and structured-output schemas can contribute to the cached prefix." +ttl = "GPT-5.6 introduces a 30-minute minimum cache life (per the openai.com GPT-5.6 overview), a change from the GPT-5.5 24h prompt_cache_retention model. Explicit cache breakpoints let callers mark segments to cache and reuse." +read_pricing_per_m = 0.50 +write_pricing_per_m = 6.25 +min_cacheable_prefix_tokens = 1024 + +[tool_calling] +format = "OpenAI Responses API tools and function calls with JSON Schema tool parameters. Chat Completions function calling is also supported. Structured Outputs are supported, including strict JSON Schema mode with strict=true. Family-level GPT-5.6 behavior inherited from the OpenAI platform; not separately re-verified on the per-tier model page." +parallel = true +strict_json_schema = true +quirks = [ + "For reasoning models using function calling through the Responses API, pass back reasoning items, function call items, and function call output items during tool loops.", + "Prompt-only JSON mode can still produce incomplete JSON when output is cut off by max_output_tokens or a content filter; use Structured Outputs for schema adherence.", +] + +[reasoning] +params = ["reasoning.effort", "text.verbosity", "max_output_tokens", "model_reasoning_effort", "model_reasoning_summary", "model_verbosity"] +defaults = "The gpt-5.6-sol model page confirms reasoning token support but does not enumerate the effort scale. Sol is the deepest-reasoning tier and the only one observed to expose `model_reasoning_effort=max` (and a codex `ultra` multi-agent mode). Harness-observed through the Codex CLI config surface, NOT OpenAI public model docs. Route Sol per-task for gnarly, long-horizon, or long-context work; treat the per-tier effort ceiling as such." +max_output_interaction = "max_output_tokens limits total generated tokens, including reasoning tokens, visible output tokens, and non-visible formatting tokens. If the cap is exhausted during reasoning, the response can be incomplete before any visible output is produced. Sol's deeper reasoning consumes proportionally more of the output budget as reasoning tokens." + +[limits] +context_window = 1050000 +max_output_tokens = 128000 +rate_limits = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request (per the gpt-5.6-sol model page). Knowledge cutoff Feb 16, 2026. ChatGPT-plan Codex OAuth usage is governed by plan entitlements, not API billing tiers; per-tier long-context API rate-limit tiers are not published on the model page." + +[openrouter] +upstream_providers = ["openai"] +require_parameters_behavior = "This registry entry is for OpenAI direct/Codex OAuth access. If served via OpenRouter, the model slug would pass through the opencode engine's model field." + +[pricing] +prompt_per_m = 5.00 +completion_per_m = 30.00 +cache_read_per_m = 0.50 +cache_write_per_m = 6.25 +as_of = "2026-07-14" + +[[sources]] +claim = "model: gpt-5.6-sol context window" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-sol" +accessed = "2026-07-14" +quote = "1,050,000 context window" + +[[sources]] +claim = "model: gpt-5.6-sol max output tokens" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-sol" +accessed = "2026-07-14" +quote = "128,000 max output tokens" + +[[sources]] +claim = "model: gpt-5.6-sol reasoning token support" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-sol" +accessed = "2026-07-14" +quote = "Reasoning token support" + +[[sources]] +claim = "pricing: gpt-5.6-sol input, cached input, and output per 1M" +url = "https://developers.openai.com/api/docs/pricing" +accessed = "2026-07-14" +quote = "Input $5.00 Cached input $0.50 Output $30.00" + +[[sources]] +claim = "limits: gpt-5.6-sol large-prompt (>272K) pricing rule" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-sol" +accessed = "2026-07-14" +quote = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request." + +[[sources]] +claim = "model: gpt-5.6-sol knowledge cutoff" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-sol" +accessed = "2026-07-14" +quote = "Feb 16, 2026 knowledge cutoff" + +[[sources]] +claim = "caching: GPT-5.6 introduces explicit cache breakpoints and a 30-minute minimum cache life" +url = "https://openai.com/index/gpt-5-6/" +accessed = "2026-07-14" +quote = "more predictable prompt caching, including support for explicit cache breakpoints and a 30-minute minimum cache life" +note = "openai.com/index/gpt-5-6 returned HTTP 403 to direct fetch; quote relayed via Perplexity sonar-pro search 2026-07-14. The 90% cache-read discount is independently corroborated by the pricing page ($0.50 cached = 0.1x $5.00 input)." + +[[sources]] +claim = "reasoning: `max` effort tier for Sol is harness-observed, not OpenAI-documented" +url = "https://developers.openai.com/codex/config-reference" +accessed = "2026-07-14" +quote = "model_reasoning_effort" +note = "OpenAI's gpt-5.6-sol model page confirms reasoning token support but does NOT enumerate effort levels or state that Sol uniquely supports `max`. The `max`/`ultra` claim is harness-observed through the Codex CLI config surface (2026-07-13) and should be treated as such until OpenAI publishes a per-tier effort scale." + +[[sources]] +claim = "release: GPT-5.6 family (Sol/Terra/Luna) GA date (third-party, corroborated)" +url = "https://www.digitalapplied.com/blog/gpt-5-6-sol-terra-luna-public-ga" +accessed = "2026-07-14" +quote = "GPT-5.6 went to general availability on July 9, 2026 — across ChatGPT, Codex, and the OpenAI API" diff --git a/registry/model-capabilities/gpt-5.6-terra.toml b/registry/model-capabilities/gpt-5.6-terra.toml new file mode 100644 index 0000000..c7817cb --- /dev/null +++ b/registry/model-capabilities/gpt-5.6-terra.toml @@ -0,0 +1,105 @@ +[model] +key = "gpt-5.6-terra" +display = "GPT-5.6 Terra" +vendor = "OpenAI" +release_date = "2026-07-09" +identity_verification = "OpenAI's developer model page for gpt-5.6-terra lists it as a member of the GPT-5.6 family (Sol/Terra/Luna) with a 1,050,000 context window, 128,000 max output tokens, and a Feb 16, 2026 knowledge cutoff. Terra is the balanced middle tier of the family. GA of the GPT-5.6 family is widely reported as 2026-07-09 (third-party); OpenAI's model page states the Feb 16 2026 knowledge cutoff but no in-page GA date." + +[api] +endpoint_families = ["responses", "chat-completions", "batch"] +auth = "In this setup, Codex CLI signs in with ChatGPT/OAuth plan credentials and uses ChatGPT workspace permissions, plan access, and ChatGPT-managed Codex entitlements. Codex CLI can also sign in with an OpenAI API key; API-key usage is billed through the OpenAI Platform account at standard API rates instead of included ChatGPT plan credits." +streaming = "Supported. API streaming is available with stream=true on Responses and Chat Completions; streaming responses use server-sent events. Codex provider config exposes SSE idle timeout/retry settings and a Responses WebSocket support flag for providers that support it. Family-level (GPT-5.6) behavior; the per-tier model page does not separately enumerate a streaming flag." + +[caching] +supported = true +mode = "implicit" +how_enabled = "Prompt caching works automatically for eligible requests with no code changes. GPT-5.6 introduces more predictable prompt caching, including support for explicit cache breakpoints. Caching starts at 1024 prompt tokens; repeated prompt prefixes, tools, and structured-output schemas can contribute to the cached prefix." +ttl = "GPT-5.6 introduces a 30-minute minimum cache life (per the openai.com GPT-5.6 overview), a change from the GPT-5.5 24h prompt_cache_retention model. Explicit cache breakpoints let callers mark segments to cache and reuse." +read_pricing_per_m = 0.25 +write_pricing_per_m = 3.125 +min_cacheable_prefix_tokens = 1024 + +[tool_calling] +format = "OpenAI Responses API tools and function calls with JSON Schema tool parameters. Chat Completions function calling is also supported. Structured Outputs are supported, including strict JSON Schema mode with strict=true. Family-level GPT-5.6 behavior inherited from the OpenAI platform; not separately re-verified on the per-tier model page." +parallel = true +strict_json_schema = true +quirks = [ + "For reasoning models using function calling through the Responses API, pass back reasoning items, function call items, and function call output items during tool loops.", + "Prompt-only JSON mode can still produce incomplete JSON when output is cut off by max_output_tokens or a content filter; use Structured Outputs for schema adherence.", +] + +[reasoning] +params = ["reasoning.effort", "text.verbosity", "max_output_tokens", "model_reasoning_effort", "model_reasoning_summary", "model_verbosity"] +defaults = "The gpt-5.6-terra model page confirms reasoning token support but does not enumerate the effort scale. Terra is routed at low/medium/high via `engine_args [\"-c\",\"model_reasoning_effort=...\"]`; `max` effort has not been observed on Terra (only on Sol). Harness-observed through the Codex CLI config surface, NOT OpenAI public model docs — treat the per-tier effort ceiling as such." +max_output_interaction = "max_output_tokens limits total generated tokens, including reasoning tokens, visible output tokens, and non-visible formatting tokens. If the cap is exhausted during reasoning, the response can be incomplete before any visible output is produced." + +[limits] +context_window = 1050000 +max_output_tokens = 128000 +rate_limits = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request (per the gpt-5.6-terra model page). Knowledge cutoff Feb 16, 2026. ChatGPT-plan Codex OAuth usage is governed by plan entitlements, not API billing tiers; per-tier long-context API rate-limit tiers are not published on the model page." + +[openrouter] +upstream_providers = ["openai"] +require_parameters_behavior = "gpt-5.6-terra is also served on OpenRouter (openai/gpt-5.6-terra); this registry entry is for OpenAI direct/Codex OAuth access. OpenRouter routing would pass the model slug through the opencode engine's model field." + +[pricing] +prompt_per_m = 2.50 +completion_per_m = 15.00 +cache_read_per_m = 0.25 +cache_write_per_m = 3.125 +as_of = "2026-07-14" + +[[sources]] +claim = "model: gpt-5.6-terra context window, max output, and knowledge cutoff" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-terra" +accessed = "2026-07-14" +quote = "1,050,000 context window" + +[[sources]] +claim = "model: gpt-5.6-terra max output tokens" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-terra" +accessed = "2026-07-14" +quote = "128,000 max output tokens" + +[[sources]] +claim = "pricing: gpt-5.6-terra input, cached input, and output per 1M" +url = "https://developers.openai.com/api/docs/pricing" +accessed = "2026-07-14" +quote = "Input $2.50 Cached input $0.25 Output $15.00" + +[[sources]] +claim = "limits: gpt-5.6-terra large-prompt (>272K) pricing rule" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-terra" +accessed = "2026-07-14" +quote = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request." + +[[sources]] +claim = "model: gpt-5.6-terra knowledge cutoff" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-terra" +accessed = "2026-07-14" +quote = "Feb 16, 2026 knowledge cutoff" + +[[sources]] +claim = "caching: GPT-5.6 introduces explicit cache breakpoints and a 30-minute minimum cache life" +url = "https://openai.com/index/gpt-5-6/" +accessed = "2026-07-14" +quote = "more predictable prompt caching, including support for explicit cache breakpoints and a 30-minute minimum cache life" +note = "openai.com/index/gpt-5-6 returned HTTP 403 to direct fetch; quote relayed via Perplexity sonar-pro search 2026-07-14. The 90% cache-read discount is independently corroborated by the pricing page ($0.25 cached = 0.1x $2.50 input)." + +[[sources]] +claim = "reasoning: Codex config exposes model reasoning and verbosity controls" +url = "https://developers.openai.com/codex/config-reference" +accessed = "2026-07-14" +quote = "model_reasoning_effort" + +[[sources]] +claim = "tool_calling: strict structured outputs use JSON Schema strict true (platform, version-independent)" +url = "https://developers.openai.com/api/docs/guides/structured-outputs" +accessed = "2026-07-14" +quote = "strict: true" + +[[sources]] +claim = "release: GPT-5.6 family (Sol/Terra/Luna) GA date (third-party, corroborated)" +url = "https://www.digitalapplied.com/blog/gpt-5-6-sol-terra-luna-public-ga" +accessed = "2026-07-14" +quote = "GPT-5.6 went to general availability on July 9, 2026 — across ChatGPT, Codex, and the OpenAI API" From ee7ca6e9b56ec485047fcb5c7f4fb91e614a1ecd Mon Sep 17 00:00:00 2001 From: Rafael Pini Date: Thu, 16 Jul 2026 18:26:14 +0000 Subject: [PATCH 2/2] =?UTF-8?q?fix(registry):=20tighten=20GPT-5.6=20sheets?= =?UTF-8?q?=20per=20review=20=E2=80=94=20sourced=20cache-write,=20per-surf?= =?UTF-8?q?ace=20effort=20scales?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cache-write prices now cite the pricing page's Cache writes column and the prompt-caching guide's 1.25x rule directly (replaces the search-relayed quote from the 403'd overview page) - reasoning effort scales stated per surface with quotes (Codex CLI enum minimal|low|medium|high|xhigh; OpenRouter supported_efforts incl. max on all three tiers); removed 'ultra' and the Sol-only 'max' claim, which both cited surfaces contradict — the harness observation moved to docs/MODEL-NOTES.md as a dated entry - OpenRouter listings for all three tiers sourced via the models API - Luna limits.long_context_caveat removed; dated observation added to docs/MODEL-NOTES.md with fact and routing judgment explicitly separated - positioning language replaced with plain facts throughout - tests/test_capability_sheets.py: every sheet parses; required sections; unique model.key; sources carry claim/quote/http(s) url/ISO accessed date; mirrored caching/pricing values must agree when numeric Co-Authored-By: Claude Fable 5 --- docs/MODEL-NOTES.md | 14 ++ registry/model-capabilities/gpt-5.6-luna.toml | 87 +++++++--- registry/model-capabilities/gpt-5.6-sol.toml | 77 ++++++--- .../model-capabilities/gpt-5.6-terra.toml | 70 +++++--- tests/test_capability_sheets.py | 153 ++++++++++++++++++ 5 files changed, 333 insertions(+), 68 deletions(-) create mode 100644 tests/test_capability_sheets.py diff --git a/docs/MODEL-NOTES.md b/docs/MODEL-NOTES.md index d7306dc..920c0cc 100644 --- a/docs/MODEL-NOTES.md +++ b/docs/MODEL-NOTES.md @@ -287,6 +287,13 @@ checks and raw logs support — no vibes, no worker self-reports. ## gpt-5.6-sol (codex) - 2026-07-15 ringer-self-update run (3 serial tasks, direct-repo-edit mode): code-fix baseline-test repair 1/1 first-try (61k tokens, 1.6m); code-feature self-update mechanism (git fetch/ff-pull/re-exec + HUD staleness restart + 20-test suite) 1/1 first-try at high effort (153k, 8.1m); code-feature signal-contract (all 3 scoreboard surfaces + canonical-route lint enforcement) passed on retry (358k, 13.7m) — attempt 1 died on stale old-column assertions in pre-existing tests it hadn't finished updating; the retry prompt's injected FAIL list was enough to close it out. Lesson: when a task rewrites a display contract, name every test file asserting the old contract in the spec's ownership list AND tell it to update them FIRST. +- 2026-07-13 reasoning-effort surfaces (dated observation): codex CLI on this + box accepted `model_reasoning_effort=max` for gpt-5.6-sol. As of 2026-07-16 + the Codex config reference types the field `minimal|low|medium|high|xhigh` + (no `max`), while OpenRouter metadata lists supported_efforts including + `max` for ALL three 5.6 tiers. The surfaces disagree — per-tier effort + ceilings are unsettled; the capability sheets carry only the per-surface + documented facts. - 2026-07-09 code-feature/code-fix (ringside-overhaul): 4/4 first-try — a ringer.py logging change with tests, a 265-line stdlib backfill CLI (atomic rewrite, dry-run, idempotence all check-verified), a ~1500-line single-file HTML redesign (running-now pills + worker-card grid + multi-expansion refactor, 30KB patch, node --check + contract greps + unittest), and a render-gating change where it correctly UPDATED tests asserting the old behavior instead of gaming the check. Medium/high reasoning, 65–120k tokens/task. - Same day, different session (bench-harness-patches, code-fix): 0.29 first-try over 7 tasks on a Next.js/Turbopack harness. Spec and check quality dominate model choice — see the scoreboard before generalizing either number. @@ -309,6 +316,13 @@ checks and raw logs support — no vibes, no worker self-reports. ## gpt-5.6-luna (codex) - 2026-07-09 code-feature (unlock-ai guide-format conversion, strict type-contract check): 1/1 first-try, 42.6k tokens, 80s. Followed a multi-file TS pattern precisely at $1/$6 pricing. Good candidate for mechanical codegen/docs lanes; audition in adjacent types. +- 2026-07-14 long-context (dated observation, no run): Luna's nominal window + is 1.05M, same as Sol/Terra, but a search that day found no tier-by-tier + long-context recall benchmark from OpenAI or any third party — per-tier + long-context behavior is unverified in both directions. Routing judgment, + not scoreboard evidence: until run evidence accumulates, the conservative + slot for Luna is short-context, low-stakes, latency-sensitive work; treat + long-context Luna assignments as exploration, not proven routing. ## opencode / z-ai glm-5.2 (via openrouter) - 2026-07-09 (aicred-invoice-downloads, 4 code-fix tasks + 1 follow-up, worktrees+npm ci checks): systematic attempt-1 NO-OP — all 4 parallel workers produced zero edits and no summary on first attempt, then completed cleanly on attempt 2 after retry-prompt injection (34k-69k tokens each). Follow-up single task passed attempt 1. Suspect first-invocation session warm-up in opencode-sandboxed under parallel spawn; budget for 2 attempts on parallel GLM batches. Output quality on Next.js/Stripe route+test work: solid, spec-faithful, one boss-caught design gap (used user-scoped supabase client where RLS demanded service role — spec didn't say explicitly; say it explicitly). diff --git a/registry/model-capabilities/gpt-5.6-luna.toml b/registry/model-capabilities/gpt-5.6-luna.toml index 4237328..e1b9230 100644 --- a/registry/model-capabilities/gpt-5.6-luna.toml +++ b/registry/model-capabilities/gpt-5.6-luna.toml @@ -3,18 +3,18 @@ key = "gpt-5.6-luna" display = "GPT-5.6 Luna" vendor = "OpenAI" release_date = "2026-07-09" -identity_verification = "OpenAI's developer model page for gpt-5.6-luna lists it as the fastest/most-affordable member of the GPT-5.6 family (Sol/Terra/Luna) with a 1,050,000 context window, 128,000 max output tokens, reasoning token support, and a Feb 16, 2026 knowledge cutoff (fine-tuning listed as Not supported). GA of the GPT-5.6 family is widely reported as 2026-07-09 (third-party); the model page carries the Feb 16 2026 knowledge cutoff but no in-page GA date." +identity_verification = "OpenAI's developer model page for gpt-5.6-luna lists it as a member of the GPT-5.6 family (Sol/Terra/Luna) with a 1,050,000 context window, 128,000 max output tokens, reasoning token support, and a Feb 16, 2026 knowledge cutoff (fine-tuning listed as Not supported). GA of the GPT-5.6 family is reported as 2026-07-09 by third-party coverage (single source); the model page carries the Feb 16 2026 knowledge cutoff but no in-page GA date." [api] endpoint_families = ["responses", "chat-completions", "batch"] auth = "In this setup, Codex CLI signs in with ChatGPT/OAuth plan credentials and uses ChatGPT workspace permissions, plan access, and ChatGPT-managed Codex entitlements. Codex CLI can also sign in with an OpenAI API key; API-key usage is billed through the OpenAI Platform account at standard API rates instead of included ChatGPT plan credits." -streaming = "Supported. API streaming is available with stream=true on Responses and Chat Completions; streaming responses use server-sent events. Luna targets high-throughput/low-latency use; the model page lists fine-tuning as Not supported for this tier." +streaming = "Supported. API streaming is available with stream=true on Responses and Chat Completions; streaming responses use server-sent events. Family-level (GPT-5.6) behavior; the per-tier model page does not separately enumerate a streaming flag." [caching] supported = true mode = "implicit" -how_enabled = "Prompt caching works automatically for eligible requests with no code changes. GPT-5.6 introduces more predictable prompt caching, including support for explicit cache breakpoints. Caching starts at 1024 prompt tokens; repeated prompt prefixes, tools, and structured-output schemas can contribute to the cached prefix." -ttl = "GPT-5.6 introduces a 30-minute minimum cache life (per the openai.com GPT-5.6 overview), a change from the GPT-5.5 24h prompt_cache_retention model. Explicit cache breakpoints let callers mark segments to cache and reuse." +how_enabled = "Prompt caching works automatically for eligible requests with no code changes; prompt_cache_options.mode defaults to implicit (the mode field records this default). For GPT-5.6, explicit cache breakpoints are additionally available in both the Responses and Chat Completions APIs, and prompt_cache_key must be set to get the more reliable cache matching. Caching starts at 1024 prompt tokens (cached_tokens is zero for requests under 1024); repeated prompt prefixes, tools, and structured-output schemas can contribute to the cached prefix." +ttl = "For GPT-5.6, prompt_cache_options.ttl sets a MINIMUM cache lifetime; the only supported value is 30m (also the default) — a cached prefix remains eligible for reuse for at least 30 minutes and may be retained longer. The prompt_cache_retention field used by earlier families (24h on GPT-5.5) is deprecated for GPT-5.6 and later." read_pricing_per_m = 0.10 write_pricing_per_m = 1.25 min_cacheable_prefix_tokens = 1024 @@ -26,30 +26,28 @@ strict_json_schema = true quirks = [ "For reasoning models using function calling through the Responses API, pass back reasoning items, function call items, and function call output items during tool loops.", "Prompt-only JSON mode can still produce incomplete JSON when output is cut off by max_output_tokens or a content filter; use Structured Outputs for schema adherence.", - "Fine-tuning is listed as Not supported on the Luna model page.", ] [reasoning] params = ["reasoning.effort", "text.verbosity", "max_output_tokens", "model_reasoning_effort", "model_reasoning_summary", "model_verbosity"] -defaults = "The gpt-5.6-luna model page confirms reasoning token support but does not enumerate the effort scale. Luna is the fast/cheap tier, routed at low/medium/high via `engine_args [\"-c\",\"model_reasoning_effort=...\"]`; `max` has not been observed on any tier but Sol. Harness-observed through the Codex CLI config surface, NOT OpenAI public model docs — treat the per-tier effort ceiling as such." +defaults = "The gpt-5.6-luna model page confirms reasoning token support but does not enumerate an effort scale. Effort scales are surface-specific: the Codex CLI config reference types model_reasoning_effort as minimal|low|medium|high|xhigh (xhigh is model-dependent), and OpenRouter endpoint metadata lists supported_efforts [max, xhigh, high, medium, low, none] with default medium for openai/gpt-5.6-luna. OpenAI publishes no per-tier effort ceiling; Ringer passes effort per-task via engine_args [\"-c\",\"model_reasoning_effort=...\"]." max_output_interaction = "max_output_tokens limits total generated tokens, including reasoning tokens, visible output tokens, and non-visible formatting tokens. If the cap is exhausted during reasoning, the response can be incomplete before any visible output is produced." [limits] context_window = 1050000 max_output_tokens = 128000 -long_context_caveat = "OPERATIONAL CAVEAT, NOT A SOURCED BENCHMARK: Luna's nominal context window is 1.05M, the same as Sol and Terra, but a 2026-07-14 search found no tier-by-tier long-context recall benchmark from OpenAI or any third party — so per-tier long-context behavior is unverified in both directions. Until a scoreboard accumulates real evidence, the conservative slot for Luna is short-context, low-stakes, latency-sensitive work." -rate_limits = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request (per the gpt-5.6-luna model page). Knowledge cutoff Feb 16, 2026. ChatGPT-plan Codex OAuth usage is governed by plan entitlements, not API billing tiers; per-tier long-context API rate-limit tiers are not published on the model page." +rate_limits = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request (per the gpt-5.6-luna model page). ChatGPT-plan Codex OAuth usage is governed by plan entitlements, not API billing tiers; per-tier long-context API rate-limit tiers are not published on the model page." [openrouter] upstream_providers = ["openai"] -require_parameters_behavior = "This registry entry is for OpenAI direct/Codex OAuth access. If served via OpenRouter, the model slug would pass through the opencode engine's model field." +require_parameters_behavior = "not applicable; this registry entry is for OpenAI direct/Codex OAuth access. gpt-5.6-luna is also listed on OpenRouter as openai/gpt-5.6-luna (see sources); OpenRouter routing would pass the model slug through the opencode engine's model field." [pricing] prompt_per_m = 1.00 completion_per_m = 6.00 cache_read_per_m = 0.10 cache_write_per_m = 1.25 -as_of = "2026-07-14" +as_of = "2026-07-16" [[sources]] claim = "model: gpt-5.6-luna context window" @@ -64,10 +62,11 @@ accessed = "2026-07-14" quote = "128,000 max output tokens" [[sources]] -claim = "pricing: gpt-5.6-luna input, cached input, and output per 1M" +claim = "pricing: gpt-5.6-luna input, cached input, cache writes, and output per 1M" url = "https://developers.openai.com/api/docs/pricing" -accessed = "2026-07-14" -quote = "Input $1.00 Cached input $0.10 Output $6.00" +accessed = "2026-07-16" +quote = "Input Cached input Cache writes Output ... gpt-5.6-luna $1.00 $0.10 $1.25 $6.00" +note = "The pricing page's GPT-5.6 rows carry an explicit Cache writes column (older families show '-'). The $1.25/M cache-write price is independently corroborated by OpenRouter endpoint metadata (input_cache_write = 0.00000125 for openai/gpt-5.6-luna, accessed 2026-07-16)." [[sources]] claim = "limits: gpt-5.6-luna large-prompt (>272K) pricing rule" @@ -76,27 +75,65 @@ accessed = "2026-07-14" quote = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request" [[sources]] -claim = "model: gpt-5.6-luna knowledge cutoff and fine-tuning support" +claim = "model: gpt-5.6-luna knowledge cutoff" url = "https://developers.openai.com/api/docs/models/gpt-5.6-luna" accessed = "2026-07-14" quote = "Feb 16, 2026 knowledge cutoff" [[sources]] -claim = "caching: GPT-5.6 introduces explicit cache breakpoints and a 30-minute minimum cache life" -url = "https://openai.com/index/gpt-5-6/" -accessed = "2026-07-14" -quote = "more predictable prompt caching, including support for explicit cache breakpoints and a 30-minute minimum cache life" -note = "openai.com/index/gpt-5-6 returned HTTP 403 to direct fetch; quote relayed via Perplexity sonar-pro search 2026-07-14. The 90% cache-read discount is independently corroborated by the pricing page ($0.10 cached = 0.1x $1.00 input)." +claim = "model: gpt-5.6-luna fine-tuning not supported" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-luna" +accessed = "2026-07-16" +quote = "Fine-tuning Not supported" [[sources]] -claim = "long_context: Luna recall-cliff figures are UNVERIFIED (no reproducible benchmark)" -url = "https://www.mindstudio.ai/blog/what-is-gpt-5-6-openai-three-model-tiers" -accessed = "2026-07-14" -quote = "Luna is the fastest and most affordable model" -note = "A 2026-07-14 search found no OpenAI or third-party tier-by-tier long-context recall benchmark for the GPT-5.6 family. Luna's long-context behavior relative to Terra/Sol is therefore unverified; the sheet carries this as an operational caveat only, not a sourced claim." +claim = "reasoning: Codex CLI reasoning-effort scale" +url = "https://developers.openai.com/codex/config-reference" +accessed = "2026-07-16" +quote = "minimal | low | medium | high | xhigh" +note = "The config reference types model_reasoning_effort with this enum ('xhigh is model-dependent') and does not include 'max'." + +[[sources]] +claim = "reasoning: OpenRouter supported reasoning efforts for openai/gpt-5.6-luna" +url = "https://openrouter.ai/api/v1/models" +accessed = "2026-07-16" +quote = "\"supported_efforts\":[\"max\",\"xhigh\",\"high\",\"medium\",\"low\",\"none\"],\"default_effort\":\"medium\"" + +[[sources]] +claim = "tool_calling: strict structured outputs use JSON Schema strict true (platform, version-independent)" +url = "https://developers.openai.com/api/docs/guides/structured-outputs" +accessed = "2026-07-16" +quote = "strict: true" [[sources]] -claim = "release: GPT-5.6 family (Sol/Terra/Luna) GA date (third-party, corroborated)" +claim = "caching: GPT-5.6 cache writes are billed at 1.25x the uncached input rate" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "On GPT-5.6 models and later model families, cache writes are billed at 1.25× the uncached input token rate and reported in `cache_write_tokens`." +note = "Direct OpenAI source (replaces an earlier search-relayed quote from openai.com/index/gpt-5-6, which 403s to direct fetch). 1.25x of $1.00 input = $1.25/M, matching the pricing page's Cache writes column." + +[[sources]] +claim = "caching: explicit cache breakpoints and 30m minimum cache life" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "use `prompt_cache_options.ttl` to set the minimum lifetime of all breakpoints written by the request. The only supported value is `30m`, which is also the default." +note = "The guide also states: 'you can mark the end of a reusable prompt prefix with an explicit cache breakpoint. Breakpoints are available in both the Responses API and Chat Completions API.', documents `prompt_cache_options.mode` with implicit as the default, requires prompt_cache_key for the more reliable matching, and deprecates prompt_cache_retention for GPT-5.6 and later." + +[[sources]] +claim = "caching: 1024-token floor for cached_tokens" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "For requests under 1024 tokens, `cached_tokens` is zero." + +[[sources]] +claim = "release: GPT-5.6 family (Sol/Terra/Luna) GA date (third-party, single source)" url = "https://www.digitalapplied.com/blog/gpt-5-6-sol-terra-luna-public-ga" accessed = "2026-07-14" quote = "GPT-5.6 went to general availability on July 9, 2026 — across ChatGPT, Codex, and the OpenAI API" +note = "release_date records the public GA date, not first availability (pre-GA access via Codex existed for the family; see docs/MODEL-NOTES.md, 'GPT-5.5 (codex) — attribution caveat'). Do not use release_date as a lower bound for scoreboard row attribution." + +[[sources]] +claim = "openrouter: gpt-5.6-luna listing and cache-write price metadata" +url = "https://openrouter.ai/api/v1/models" +accessed = "2026-07-16" +quote = "\"id\": \"openai/gpt-5.6-luna\", \"name\": \"OpenAI: GPT-5.6 Luna\" ... \"input_cache_read\": \"0.0000001\", \"input_cache_write\": \"0.00000125\"" diff --git a/registry/model-capabilities/gpt-5.6-sol.toml b/registry/model-capabilities/gpt-5.6-sol.toml index 7d6eeae..7c546b0 100644 --- a/registry/model-capabilities/gpt-5.6-sol.toml +++ b/registry/model-capabilities/gpt-5.6-sol.toml @@ -3,18 +3,18 @@ key = "gpt-5.6-sol" display = "GPT-5.6 Sol" vendor = "OpenAI" release_date = "2026-07-09" -identity_verification = "OpenAI's developer model page for gpt-5.6-sol lists it as the flagship member of the GPT-5.6 family (Sol/Terra/Luna) with a 1,050,000 context window, 128,000 max output tokens, reasoning token support, and a Feb 16, 2026 knowledge cutoff. GA of the GPT-5.6 family is widely reported as 2026-07-09 (third-party); the model page carries the Feb 16 2026 knowledge cutoff but no in-page GA date." +identity_verification = "OpenAI's developer model page for gpt-5.6-sol lists it as a member of the GPT-5.6 family (Sol/Terra/Luna) with a 1,050,000 context window, 128,000 max output tokens, reasoning token support, and a Feb 16, 2026 knowledge cutoff. GA of the GPT-5.6 family is reported as 2026-07-09 by third-party coverage (single source); the model page carries the Feb 16 2026 knowledge cutoff but no in-page GA date." [api] endpoint_families = ["responses", "chat-completions", "batch"] auth = "In this setup, Codex CLI signs in with ChatGPT/OAuth plan credentials and uses ChatGPT workspace permissions, plan access, and ChatGPT-managed Codex entitlements. Codex CLI can also sign in with an OpenAI API key; API-key usage is billed through the OpenAI Platform account at standard API rates instead of included ChatGPT plan credits." -streaming = "Supported. API streaming is available with stream=true on Responses and Chat Completions; streaming responses use server-sent events. The Sol model page also lists Realtime, Assistants, Fine-tuning, Embeddings, and image/audio endpoints among the broadest endpoint set in the family; Ringer uses Responses/Chat via Codex." +streaming = "Supported. API streaming is available with stream=true on Responses and Chat Completions; streaming responses use server-sent events. Family-level (GPT-5.6) behavior; the per-tier model page does not separately enumerate a streaming flag." [caching] supported = true mode = "implicit" -how_enabled = "Prompt caching works automatically for eligible requests with no code changes. GPT-5.6 introduces more predictable prompt caching, including support for explicit cache breakpoints. Caching starts at 1024 prompt tokens; repeated prompt prefixes, tools, and structured-output schemas can contribute to the cached prefix." -ttl = "GPT-5.6 introduces a 30-minute minimum cache life (per the openai.com GPT-5.6 overview), a change from the GPT-5.5 24h prompt_cache_retention model. Explicit cache breakpoints let callers mark segments to cache and reuse." +how_enabled = "Prompt caching works automatically for eligible requests with no code changes; prompt_cache_options.mode defaults to implicit (the mode field records this default). For GPT-5.6, explicit cache breakpoints are additionally available in both the Responses and Chat Completions APIs, and prompt_cache_key must be set to get the more reliable cache matching. Caching starts at 1024 prompt tokens (cached_tokens is zero for requests under 1024); repeated prompt prefixes, tools, and structured-output schemas can contribute to the cached prefix." +ttl = "For GPT-5.6, prompt_cache_options.ttl sets a MINIMUM cache lifetime; the only supported value is 30m (also the default) — a cached prefix remains eligible for reuse for at least 30 minutes and may be retained longer. The prompt_cache_retention field used by earlier families (24h on GPT-5.5) is deprecated for GPT-5.6 and later." read_pricing_per_m = 0.50 write_pricing_per_m = 6.25 min_cacheable_prefix_tokens = 1024 @@ -30,24 +30,24 @@ quirks = [ [reasoning] params = ["reasoning.effort", "text.verbosity", "max_output_tokens", "model_reasoning_effort", "model_reasoning_summary", "model_verbosity"] -defaults = "The gpt-5.6-sol model page confirms reasoning token support but does not enumerate the effort scale. Sol is the deepest-reasoning tier and the only one observed to expose `model_reasoning_effort=max` (and a codex `ultra` multi-agent mode). Harness-observed through the Codex CLI config surface, NOT OpenAI public model docs. Route Sol per-task for gnarly, long-horizon, or long-context work; treat the per-tier effort ceiling as such." -max_output_interaction = "max_output_tokens limits total generated tokens, including reasoning tokens, visible output tokens, and non-visible formatting tokens. If the cap is exhausted during reasoning, the response can be incomplete before any visible output is produced. Sol's deeper reasoning consumes proportionally more of the output budget as reasoning tokens." +defaults = "The gpt-5.6-sol model page confirms reasoning token support but does not enumerate an effort scale. Effort scales are surface-specific: the Codex CLI config reference types model_reasoning_effort as minimal|low|medium|high|xhigh (xhigh is model-dependent), and OpenRouter endpoint metadata lists supported_efforts [max, xhigh, high, medium, low, none] with default medium for openai/gpt-5.6-sol. OpenAI publishes no per-tier effort ceiling; Ringer passes effort per-task via engine_args [\"-c\",\"model_reasoning_effort=...\"]." +max_output_interaction = "max_output_tokens limits total generated tokens, including reasoning tokens, visible output tokens, and non-visible formatting tokens. If the cap is exhausted during reasoning, the response can be incomplete before any visible output is produced." [limits] context_window = 1050000 max_output_tokens = 128000 -rate_limits = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request (per the gpt-5.6-sol model page). Knowledge cutoff Feb 16, 2026. ChatGPT-plan Codex OAuth usage is governed by plan entitlements, not API billing tiers; per-tier long-context API rate-limit tiers are not published on the model page." +rate_limits = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request (per the gpt-5.6-sol model page). ChatGPT-plan Codex OAuth usage is governed by plan entitlements, not API billing tiers; per-tier long-context API rate-limit tiers are not published on the model page." [openrouter] upstream_providers = ["openai"] -require_parameters_behavior = "This registry entry is for OpenAI direct/Codex OAuth access. If served via OpenRouter, the model slug would pass through the opencode engine's model field." +require_parameters_behavior = "not applicable; this registry entry is for OpenAI direct/Codex OAuth access. gpt-5.6-sol is also listed on OpenRouter as openai/gpt-5.6-sol (see sources); OpenRouter routing would pass the model slug through the opencode engine's model field." [pricing] prompt_per_m = 5.00 completion_per_m = 30.00 cache_read_per_m = 0.50 cache_write_per_m = 6.25 -as_of = "2026-07-14" +as_of = "2026-07-16" [[sources]] claim = "model: gpt-5.6-sol context window" @@ -68,10 +68,11 @@ accessed = "2026-07-14" quote = "Reasoning token support" [[sources]] -claim = "pricing: gpt-5.6-sol input, cached input, and output per 1M" +claim = "pricing: gpt-5.6-sol input, cached input, cache writes, and output per 1M" url = "https://developers.openai.com/api/docs/pricing" -accessed = "2026-07-14" -quote = "Input $5.00 Cached input $0.50 Output $30.00" +accessed = "2026-07-16" +quote = "Input Cached input Cache writes Output gpt-5.6-sol $5.00 $0.50 $6.25 $30.00" +note = "The pricing page's GPT-5.6 rows carry an explicit Cache writes column (older families show '-'). The $6.25/M cache-write price is independently corroborated by OpenRouter endpoint metadata (input_cache_write = 0.00000625 for openai/gpt-5.6-sol, accessed 2026-07-16)." [[sources]] claim = "limits: gpt-5.6-sol large-prompt (>272K) pricing rule" @@ -86,21 +87,53 @@ accessed = "2026-07-14" quote = "Feb 16, 2026 knowledge cutoff" [[sources]] -claim = "caching: GPT-5.6 introduces explicit cache breakpoints and a 30-minute minimum cache life" -url = "https://openai.com/index/gpt-5-6/" -accessed = "2026-07-14" -quote = "more predictable prompt caching, including support for explicit cache breakpoints and a 30-minute minimum cache life" -note = "openai.com/index/gpt-5-6 returned HTTP 403 to direct fetch; quote relayed via Perplexity sonar-pro search 2026-07-14. The 90% cache-read discount is independently corroborated by the pricing page ($0.50 cached = 0.1x $5.00 input)." +claim = "caching: GPT-5.6 cache writes are billed at 1.25x the uncached input rate" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "On GPT-5.6 models and later model families, cache writes are billed at 1.25× the uncached input token rate and reported in `cache_write_tokens`." +note = "Direct OpenAI source (replaces an earlier search-relayed quote from openai.com/index/gpt-5-6, which 403s to direct fetch). 1.25x of $5.00 input = $6.25/M, matching the pricing page's Cache writes column." + +[[sources]] +claim = "caching: explicit cache breakpoints and 30m minimum cache life" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "use `prompt_cache_options.ttl` to set the minimum lifetime of all breakpoints written by the request. The only supported value is `30m`, which is also the default." +note = "The guide also states: 'you can mark the end of a reusable prompt prefix with an explicit cache breakpoint. Breakpoints are available in both the Responses API and Chat Completions API.', documents `prompt_cache_options.mode` with implicit as the default, requires prompt_cache_key for the more reliable matching, and deprecates prompt_cache_retention for GPT-5.6 and later." [[sources]] -claim = "reasoning: `max` effort tier for Sol is harness-observed, not OpenAI-documented" +claim = "caching: 1024-token floor for cached_tokens" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "For requests under 1024 tokens, `cached_tokens` is zero." + +[[sources]] +claim = "reasoning: Codex CLI reasoning-effort scale" url = "https://developers.openai.com/codex/config-reference" -accessed = "2026-07-14" -quote = "model_reasoning_effort" -note = "OpenAI's gpt-5.6-sol model page confirms reasoning token support but does NOT enumerate effort levels or state that Sol uniquely supports `max`. The `max`/`ultra` claim is harness-observed through the Codex CLI config surface (2026-07-13) and should be treated as such until OpenAI publishes a per-tier effort scale." +accessed = "2026-07-16" +quote = "minimal | low | medium | high | xhigh" +note = "The config reference types model_reasoning_effort with this enum ('xhigh is model-dependent') and does not include 'max'. A 2026-07-13 harness observation on one machine had codex accept model_reasoning_effort=max with gpt-5.6-sol; the surfaces disagree — recorded as a dated observation in docs/MODEL-NOTES.md, not as a sheet capability." [[sources]] -claim = "release: GPT-5.6 family (Sol/Terra/Luna) GA date (third-party, corroborated)" +claim = "reasoning: OpenRouter supported reasoning efforts for openai/gpt-5.6-sol" +url = "https://openrouter.ai/api/v1/models" +accessed = "2026-07-16" +quote = "\"supported_efforts\":[\"max\",\"xhigh\",\"high\",\"medium\",\"low\",\"none\"],\"default_effort\":\"medium\"" + +[[sources]] +claim = "tool_calling: strict structured outputs use JSON Schema strict true (platform, version-independent)" +url = "https://developers.openai.com/api/docs/guides/structured-outputs" +accessed = "2026-07-16" +quote = "strict: true" + +[[sources]] +claim = "release: GPT-5.6 family (Sol/Terra/Luna) GA date (third-party, single source)" url = "https://www.digitalapplied.com/blog/gpt-5-6-sol-terra-luna-public-ga" accessed = "2026-07-14" quote = "GPT-5.6 went to general availability on July 9, 2026 — across ChatGPT, Codex, and the OpenAI API" +note = "release_date records the public GA date, not first availability: this machine's Codex default had moved to gpt-5.6-sol at an unknown earlier date (see docs/MODEL-NOTES.md, 'GPT-5.5 (codex) — attribution caveat'). Do not use release_date as a lower bound for scoreboard row attribution." + +[[sources]] +claim = "openrouter: gpt-5.6-sol listing and cache-write price metadata" +url = "https://openrouter.ai/api/v1/models" +accessed = "2026-07-16" +quote = "\"id\": \"openai/gpt-5.6-sol\", \"name\": \"OpenAI: GPT-5.6 Sol\" ... \"input_cache_read\": \"0.0000005\", \"input_cache_write\": \"0.00000625\"" diff --git a/registry/model-capabilities/gpt-5.6-terra.toml b/registry/model-capabilities/gpt-5.6-terra.toml index c7817cb..dc9ec2b 100644 --- a/registry/model-capabilities/gpt-5.6-terra.toml +++ b/registry/model-capabilities/gpt-5.6-terra.toml @@ -3,18 +3,18 @@ key = "gpt-5.6-terra" display = "GPT-5.6 Terra" vendor = "OpenAI" release_date = "2026-07-09" -identity_verification = "OpenAI's developer model page for gpt-5.6-terra lists it as a member of the GPT-5.6 family (Sol/Terra/Luna) with a 1,050,000 context window, 128,000 max output tokens, and a Feb 16, 2026 knowledge cutoff. Terra is the balanced middle tier of the family. GA of the GPT-5.6 family is widely reported as 2026-07-09 (third-party); OpenAI's model page states the Feb 16 2026 knowledge cutoff but no in-page GA date." +identity_verification = "OpenAI's developer model page for gpt-5.6-terra lists it as a member of the GPT-5.6 family (Sol/Terra/Luna) with a 1,050,000 context window, 128,000 max output tokens, and a Feb 16, 2026 knowledge cutoff. GA of the GPT-5.6 family is reported as 2026-07-09 by third-party coverage (single source); OpenAI's model page states the Feb 16 2026 knowledge cutoff but no in-page GA date." [api] endpoint_families = ["responses", "chat-completions", "batch"] auth = "In this setup, Codex CLI signs in with ChatGPT/OAuth plan credentials and uses ChatGPT workspace permissions, plan access, and ChatGPT-managed Codex entitlements. Codex CLI can also sign in with an OpenAI API key; API-key usage is billed through the OpenAI Platform account at standard API rates instead of included ChatGPT plan credits." -streaming = "Supported. API streaming is available with stream=true on Responses and Chat Completions; streaming responses use server-sent events. Codex provider config exposes SSE idle timeout/retry settings and a Responses WebSocket support flag for providers that support it. Family-level (GPT-5.6) behavior; the per-tier model page does not separately enumerate a streaming flag." +streaming = "Supported. API streaming is available with stream=true on Responses and Chat Completions; streaming responses use server-sent events. Family-level (GPT-5.6) behavior; the per-tier model page does not separately enumerate a streaming flag." [caching] supported = true mode = "implicit" -how_enabled = "Prompt caching works automatically for eligible requests with no code changes. GPT-5.6 introduces more predictable prompt caching, including support for explicit cache breakpoints. Caching starts at 1024 prompt tokens; repeated prompt prefixes, tools, and structured-output schemas can contribute to the cached prefix." -ttl = "GPT-5.6 introduces a 30-minute minimum cache life (per the openai.com GPT-5.6 overview), a change from the GPT-5.5 24h prompt_cache_retention model. Explicit cache breakpoints let callers mark segments to cache and reuse." +how_enabled = "Prompt caching works automatically for eligible requests with no code changes; prompt_cache_options.mode defaults to implicit (the mode field records this default). For GPT-5.6, explicit cache breakpoints are additionally available in both the Responses and Chat Completions APIs, and prompt_cache_key must be set to get the more reliable cache matching. Caching starts at 1024 prompt tokens (cached_tokens is zero for requests under 1024); repeated prompt prefixes, tools, and structured-output schemas can contribute to the cached prefix." +ttl = "For GPT-5.6, prompt_cache_options.ttl sets a MINIMUM cache lifetime; the only supported value is 30m (also the default) — a cached prefix remains eligible for reuse for at least 30 minutes and may be retained longer. The prompt_cache_retention field used by earlier families (24h on GPT-5.5) is deprecated for GPT-5.6 and later." read_pricing_per_m = 0.25 write_pricing_per_m = 3.125 min_cacheable_prefix_tokens = 1024 @@ -30,27 +30,27 @@ quirks = [ [reasoning] params = ["reasoning.effort", "text.verbosity", "max_output_tokens", "model_reasoning_effort", "model_reasoning_summary", "model_verbosity"] -defaults = "The gpt-5.6-terra model page confirms reasoning token support but does not enumerate the effort scale. Terra is routed at low/medium/high via `engine_args [\"-c\",\"model_reasoning_effort=...\"]`; `max` effort has not been observed on Terra (only on Sol). Harness-observed through the Codex CLI config surface, NOT OpenAI public model docs — treat the per-tier effort ceiling as such." +defaults = "The gpt-5.6-terra model page confirms reasoning token support but does not enumerate an effort scale. Effort scales are surface-specific: the Codex CLI config reference types model_reasoning_effort as minimal|low|medium|high|xhigh (xhigh is model-dependent), and OpenRouter endpoint metadata lists supported_efforts [max, xhigh, high, medium, low, none] with default medium for openai/gpt-5.6-terra. OpenAI publishes no per-tier effort ceiling; Ringer passes effort per-task via engine_args [\"-c\",\"model_reasoning_effort=...\"]." max_output_interaction = "max_output_tokens limits total generated tokens, including reasoning tokens, visible output tokens, and non-visible formatting tokens. If the cap is exhausted during reasoning, the response can be incomplete before any visible output is produced." [limits] context_window = 1050000 max_output_tokens = 128000 -rate_limits = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request (per the gpt-5.6-terra model page). Knowledge cutoff Feb 16, 2026. ChatGPT-plan Codex OAuth usage is governed by plan entitlements, not API billing tiers; per-tier long-context API rate-limit tiers are not published on the model page." +rate_limits = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request (per the gpt-5.6-terra model page). ChatGPT-plan Codex OAuth usage is governed by plan entitlements, not API billing tiers; per-tier long-context API rate-limit tiers are not published on the model page." [openrouter] upstream_providers = ["openai"] -require_parameters_behavior = "gpt-5.6-terra is also served on OpenRouter (openai/gpt-5.6-terra); this registry entry is for OpenAI direct/Codex OAuth access. OpenRouter routing would pass the model slug through the opencode engine's model field." +require_parameters_behavior = "not applicable; this registry entry is for OpenAI direct/Codex OAuth access. gpt-5.6-terra is also listed on OpenRouter as openai/gpt-5.6-terra (see sources); OpenRouter routing would pass the model slug through the opencode engine's model field." [pricing] prompt_per_m = 2.50 completion_per_m = 15.00 cache_read_per_m = 0.25 cache_write_per_m = 3.125 -as_of = "2026-07-14" +as_of = "2026-07-16" [[sources]] -claim = "model: gpt-5.6-terra context window, max output, and knowledge cutoff" +claim = "model: gpt-5.6-terra context window" url = "https://developers.openai.com/api/docs/models/gpt-5.6-terra" accessed = "2026-07-14" quote = "1,050,000 context window" @@ -62,10 +62,11 @@ accessed = "2026-07-14" quote = "128,000 max output tokens" [[sources]] -claim = "pricing: gpt-5.6-terra input, cached input, and output per 1M" +claim = "pricing: gpt-5.6-terra input, cached input, cache writes, and output per 1M" url = "https://developers.openai.com/api/docs/pricing" -accessed = "2026-07-14" -quote = "Input $2.50 Cached input $0.25 Output $15.00" +accessed = "2026-07-16" +quote = "Input Cached input Cache writes Output ... gpt-5.6-terra $2.50 $0.25 $3.125 $15.00" +note = "The pricing page's GPT-5.6 rows carry an explicit Cache writes column (older families show '-'). The $3.125/M cache-write price is independently corroborated by OpenRouter endpoint metadata (input_cache_write = 0.000003125 for openai/gpt-5.6-terra, accessed 2026-07-16)." [[sources]] claim = "limits: gpt-5.6-terra large-prompt (>272K) pricing rule" @@ -80,17 +81,37 @@ accessed = "2026-07-14" quote = "Feb 16, 2026 knowledge cutoff" [[sources]] -claim = "caching: GPT-5.6 introduces explicit cache breakpoints and a 30-minute minimum cache life" -url = "https://openai.com/index/gpt-5-6/" -accessed = "2026-07-14" -quote = "more predictable prompt caching, including support for explicit cache breakpoints and a 30-minute minimum cache life" -note = "openai.com/index/gpt-5-6 returned HTTP 403 to direct fetch; quote relayed via Perplexity sonar-pro search 2026-07-14. The 90% cache-read discount is independently corroborated by the pricing page ($0.25 cached = 0.1x $2.50 input)." +claim = "caching: GPT-5.6 cache writes are billed at 1.25x the uncached input rate" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "On GPT-5.6 models and later model families, cache writes are billed at 1.25× the uncached input token rate and reported in `cache_write_tokens`." +note = "Direct OpenAI source (replaces an earlier search-relayed quote from openai.com/index/gpt-5-6, which 403s to direct fetch). 1.25x of $2.50 input = $3.125/M, matching the pricing page's Cache writes column." + +[[sources]] +claim = "caching: explicit cache breakpoints and 30m minimum cache life" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "use `prompt_cache_options.ttl` to set the minimum lifetime of all breakpoints written by the request. The only supported value is `30m`, which is also the default." +note = "The guide also states: 'you can mark the end of a reusable prompt prefix with an explicit cache breakpoint. Breakpoints are available in both the Responses API and Chat Completions API.', documents `prompt_cache_options.mode` with implicit as the default, requires prompt_cache_key for the more reliable matching, and deprecates prompt_cache_retention for GPT-5.6 and later." + +[[sources]] +claim = "caching: 1024-token floor for cached_tokens" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "For requests under 1024 tokens, `cached_tokens` is zero." [[sources]] -claim = "reasoning: Codex config exposes model reasoning and verbosity controls" +claim = "reasoning: Codex CLI reasoning-effort scale" url = "https://developers.openai.com/codex/config-reference" -accessed = "2026-07-14" -quote = "model_reasoning_effort" +accessed = "2026-07-16" +quote = "minimal | low | medium | high | xhigh" +note = "The config reference types model_reasoning_effort with this enum ('xhigh is model-dependent') and does not include 'max'." + +[[sources]] +claim = "reasoning: OpenRouter supported reasoning efforts for openai/gpt-5.6-terra" +url = "https://openrouter.ai/api/v1/models" +accessed = "2026-07-16" +quote = "\"supported_efforts\":[\"max\",\"xhigh\",\"high\",\"medium\",\"low\",\"none\"],\"default_effort\":\"medium\"" [[sources]] claim = "tool_calling: strict structured outputs use JSON Schema strict true (platform, version-independent)" @@ -99,7 +120,14 @@ accessed = "2026-07-14" quote = "strict: true" [[sources]] -claim = "release: GPT-5.6 family (Sol/Terra/Luna) GA date (third-party, corroborated)" +claim = "release: GPT-5.6 family (Sol/Terra/Luna) GA date (third-party, single source)" url = "https://www.digitalapplied.com/blog/gpt-5-6-sol-terra-luna-public-ga" accessed = "2026-07-14" quote = "GPT-5.6 went to general availability on July 9, 2026 — across ChatGPT, Codex, and the OpenAI API" +note = "release_date records the public GA date, not first availability (pre-GA access via Codex existed for the family; see docs/MODEL-NOTES.md, 'GPT-5.5 (codex) — attribution caveat'). Do not use release_date as a lower bound for scoreboard row attribution." + +[[sources]] +claim = "openrouter: gpt-5.6-terra listing and cache-write price metadata" +url = "https://openrouter.ai/api/v1/models" +accessed = "2026-07-16" +quote = "\"id\": \"openai/gpt-5.6-terra\", \"name\": \"OpenAI: GPT-5.6 Terra\" ... \"input_cache_read\": \"0.00000025\", \"input_cache_write\": \"0.000003125\"" diff --git a/tests/test_capability_sheets.py b/tests/test_capability_sheets.py new file mode 100644 index 0000000..e1c3fd2 --- /dev/null +++ b/tests/test_capability_sheets.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Structural checks for registry/model-capabilities/*.toml. + +Every model sheet must parse as TOML, carry the required sections, declare a +unique model key, and back its claims with source records that carry a claim, +an http(s) url, an ISO accessed date, and a quote. Platform sheets (no [model] +table) must be explicitly listed in PLATFORM_SHEETS and only need well-formed +sources — any other sheet missing its [model] table fails instead of being +silently exempted. +""" +from __future__ import annotations + +import tomllib +import unittest +from datetime import date +from pathlib import Path +from urllib.parse import urlparse + +ROOT = Path(__file__).resolve().parents[1] +SHEETS_DIR = ROOT / "registry" / "model-capabilities" + +# Sheets that intentionally have no [model] table (platform-level facts). +PLATFORM_SHEETS = {"openrouter-platform.toml"} + +REQUIRED_MODEL_SECTIONS = { + "model", + "api", + "caching", + "tool_calling", + "reasoning", + "limits", + "pricing", + "sources", +} + +# (caching key, pricing key) pairs that duplicate the same fact; when both are +# numeric they must agree. +MIRRORED_PRICE_FIELDS = ( + ("read_pricing_per_m", "cache_read_per_m"), + ("write_pricing_per_m", "cache_write_per_m"), +) + +_SHEETS: dict[Path, dict] | None = None + + +def load_sheets() -> dict[Path, dict]: + """Parse every sheet once. Unparseable files are skipped here so the other + tests still cover the rest of the corpus; test_every_sheet_parses is the + single test that attributes the parse error to its sheet.""" + global _SHEETS + if _SHEETS is None: + _SHEETS = {} + for path in sorted(SHEETS_DIR.glob("*.toml")): + try: + with path.open("rb") as fh: + _SHEETS[path] = tomllib.load(fh) + except tomllib.TOMLDecodeError: + continue + return _SHEETS + + +def model_sheets() -> dict[Path, dict]: + return { + path: data + for path, data in load_sheets().items() + if path.name not in PLATFORM_SHEETS + } + + +class TestCapabilitySheets(unittest.TestCase): + def test_sheets_dir_has_sheets(self) -> None: + self.assertTrue( + sorted(SHEETS_DIR.glob("*.toml")), f"no sheets found in {SHEETS_DIR}" + ) + + def test_every_sheet_parses(self) -> None: + for path in sorted(SHEETS_DIR.glob("*.toml")): + with self.subTest(sheet=path.name): + with path.open("rb") as fh: + tomllib.load(fh) + + def test_model_sheets_have_required_sections(self) -> None: + for path, data in model_sheets().items(): + with self.subTest(sheet=path.name): + missing = REQUIRED_MODEL_SECTIONS - set(data) + self.assertFalse( + missing, + f"{path.name} is missing required sections: {sorted(missing)}" + " (platform sheets must be listed in PLATFORM_SHEETS)", + ) + + def test_model_keys_present_and_unique(self) -> None: + seen: dict[str, str] = {} + for path, data in model_sheets().items(): + with self.subTest(sheet=path.name): + key = data.get("model", {}).get("key") + self.assertTrue( + isinstance(key, str) and key.strip(), + f"{path.name} has no non-empty model.key", + ) + self.assertNotIn( + key, + seen, + f"model.key {key!r} appears in both {seen.get(key)} and {path.name}", + ) + seen[key] = path.name + + def test_mirrored_cache_prices_agree(self) -> None: + for path, data in model_sheets().items(): + caching = data.get("caching", {}) + pricing = data.get("pricing", {}) + for caching_key, pricing_key in MIRRORED_PRICE_FIELDS: + a, b = caching.get(caching_key), pricing.get(pricing_key) + if isinstance(a, (int, float)) and isinstance(b, (int, float)): + with self.subTest(sheet=path.name, field=pricing_key): + self.assertEqual( + a, + b, + f"{path.name}: caching.{caching_key}={a} disagrees " + f"with pricing.{pricing_key}={b}", + ) + + def test_sources_are_complete_records(self) -> None: + for path, data in load_sheets().items(): + sources = data.get("sources", []) + with self.subTest(sheet=path.name): + self.assertTrue(sources, f"{path.name} has no [[sources]] records") + for i, src in enumerate(sources): + with self.subTest(sheet=path.name, source=i): + for field in ("claim", "quote"): + value = src.get(field) + self.assertTrue( + isinstance(value, str) and value.strip(), + f"{path.name} source #{i} has no non-empty {field}", + ) + url = src.get("url", "") + parsed = urlparse(url if isinstance(url, str) else "") + self.assertTrue( + parsed.scheme in ("http", "https") and parsed.netloc, + f"{path.name} source #{i} url {url!r} is not http(s)", + ) + accessed = str(src.get("accessed", "")) + try: + date.fromisoformat(accessed) + except ValueError: + self.fail( + f"{path.name} source #{i} accessed date {accessed!r} " + "is not a valid ISO date" + ) + + +if __name__ == "__main__": + unittest.main()