diff --git a/docs/MODEL-NOTES.md b/docs/MODEL-NOTES.md index d7306dc..920c0cc 100644 --- a/docs/MODEL-NOTES.md +++ b/docs/MODEL-NOTES.md @@ -287,6 +287,13 @@ checks and raw logs support — no vibes, no worker self-reports. ## gpt-5.6-sol (codex) - 2026-07-15 ringer-self-update run (3 serial tasks, direct-repo-edit mode): code-fix baseline-test repair 1/1 first-try (61k tokens, 1.6m); code-feature self-update mechanism (git fetch/ff-pull/re-exec + HUD staleness restart + 20-test suite) 1/1 first-try at high effort (153k, 8.1m); code-feature signal-contract (all 3 scoreboard surfaces + canonical-route lint enforcement) passed on retry (358k, 13.7m) — attempt 1 died on stale old-column assertions in pre-existing tests it hadn't finished updating; the retry prompt's injected FAIL list was enough to close it out. Lesson: when a task rewrites a display contract, name every test file asserting the old contract in the spec's ownership list AND tell it to update them FIRST. +- 2026-07-13 reasoning-effort surfaces (dated observation): codex CLI on this + box accepted `model_reasoning_effort=max` for gpt-5.6-sol. As of 2026-07-16 + the Codex config reference types the field `minimal|low|medium|high|xhigh` + (no `max`), while OpenRouter metadata lists supported_efforts including + `max` for ALL three 5.6 tiers. The surfaces disagree — per-tier effort + ceilings are unsettled; the capability sheets carry only the per-surface + documented facts. - 2026-07-09 code-feature/code-fix (ringside-overhaul): 4/4 first-try — a ringer.py logging change with tests, a 265-line stdlib backfill CLI (atomic rewrite, dry-run, idempotence all check-verified), a ~1500-line single-file HTML redesign (running-now pills + worker-card grid + multi-expansion refactor, 30KB patch, node --check + contract greps + unittest), and a render-gating change where it correctly UPDATED tests asserting the old behavior instead of gaming the check. Medium/high reasoning, 65–120k tokens/task. - Same day, different session (bench-harness-patches, code-fix): 0.29 first-try over 7 tasks on a Next.js/Turbopack harness. Spec and check quality dominate model choice — see the scoreboard before generalizing either number. @@ -309,6 +316,13 @@ checks and raw logs support — no vibes, no worker self-reports. ## gpt-5.6-luna (codex) - 2026-07-09 code-feature (unlock-ai guide-format conversion, strict type-contract check): 1/1 first-try, 42.6k tokens, 80s. Followed a multi-file TS pattern precisely at $1/$6 pricing. Good candidate for mechanical codegen/docs lanes; audition in adjacent types. +- 2026-07-14 long-context (dated observation, no run): Luna's nominal window + is 1.05M, same as Sol/Terra, but a search that day found no tier-by-tier + long-context recall benchmark from OpenAI or any third party — per-tier + long-context behavior is unverified in both directions. Routing judgment, + not scoreboard evidence: until run evidence accumulates, the conservative + slot for Luna is short-context, low-stakes, latency-sensitive work; treat + long-context Luna assignments as exploration, not proven routing. ## opencode / z-ai glm-5.2 (via openrouter) - 2026-07-09 (aicred-invoice-downloads, 4 code-fix tasks + 1 follow-up, worktrees+npm ci checks): systematic attempt-1 NO-OP — all 4 parallel workers produced zero edits and no summary on first attempt, then completed cleanly on attempt 2 after retry-prompt injection (34k-69k tokens each). Follow-up single task passed attempt 1. Suspect first-invocation session warm-up in opencode-sandboxed under parallel spawn; budget for 2 attempts on parallel GLM batches. Output quality on Next.js/Stripe route+test work: solid, spec-faithful, one boss-caught design gap (used user-scoped supabase client where RLS demanded service role — spec didn't say explicitly; say it explicitly). diff --git a/registry/model-capabilities/gpt-5.6-luna.toml b/registry/model-capabilities/gpt-5.6-luna.toml new file mode 100644 index 0000000..e1b9230 --- /dev/null +++ b/registry/model-capabilities/gpt-5.6-luna.toml @@ -0,0 +1,139 @@ +[model] +key = "gpt-5.6-luna" +display = "GPT-5.6 Luna" +vendor = "OpenAI" +release_date = "2026-07-09" +identity_verification = "OpenAI's developer model page for gpt-5.6-luna lists it as a member of the GPT-5.6 family (Sol/Terra/Luna) with a 1,050,000 context window, 128,000 max output tokens, reasoning token support, and a Feb 16, 2026 knowledge cutoff (fine-tuning listed as Not supported). GA of the GPT-5.6 family is reported as 2026-07-09 by third-party coverage (single source); the model page carries the Feb 16 2026 knowledge cutoff but no in-page GA date." + +[api] +endpoint_families = ["responses", "chat-completions", "batch"] +auth = "In this setup, Codex CLI signs in with ChatGPT/OAuth plan credentials and uses ChatGPT workspace permissions, plan access, and ChatGPT-managed Codex entitlements. Codex CLI can also sign in with an OpenAI API key; API-key usage is billed through the OpenAI Platform account at standard API rates instead of included ChatGPT plan credits." +streaming = "Supported. API streaming is available with stream=true on Responses and Chat Completions; streaming responses use server-sent events. Family-level (GPT-5.6) behavior; the per-tier model page does not separately enumerate a streaming flag." + +[caching] +supported = true +mode = "implicit" +how_enabled = "Prompt caching works automatically for eligible requests with no code changes; prompt_cache_options.mode defaults to implicit (the mode field records this default). For GPT-5.6, explicit cache breakpoints are additionally available in both the Responses and Chat Completions APIs, and prompt_cache_key must be set to get the more reliable cache matching. Caching starts at 1024 prompt tokens (cached_tokens is zero for requests under 1024); repeated prompt prefixes, tools, and structured-output schemas can contribute to the cached prefix." +ttl = "For GPT-5.6, prompt_cache_options.ttl sets a MINIMUM cache lifetime; the only supported value is 30m (also the default) — a cached prefix remains eligible for reuse for at least 30 minutes and may be retained longer. The prompt_cache_retention field used by earlier families (24h on GPT-5.5) is deprecated for GPT-5.6 and later." +read_pricing_per_m = 0.10 +write_pricing_per_m = 1.25 +min_cacheable_prefix_tokens = 1024 + +[tool_calling] +format = "OpenAI Responses API tools and function calls with JSON Schema tool parameters. Chat Completions function calling is also supported. Structured Outputs are supported, including strict JSON Schema mode with strict=true. Family-level GPT-5.6 behavior inherited from the OpenAI platform; not separately re-verified on the per-tier model page." +parallel = true +strict_json_schema = true +quirks = [ + "For reasoning models using function calling through the Responses API, pass back reasoning items, function call items, and function call output items during tool loops.", + "Prompt-only JSON mode can still produce incomplete JSON when output is cut off by max_output_tokens or a content filter; use Structured Outputs for schema adherence.", +] + +[reasoning] +params = ["reasoning.effort", "text.verbosity", "max_output_tokens", "model_reasoning_effort", "model_reasoning_summary", "model_verbosity"] +defaults = "The gpt-5.6-luna model page confirms reasoning token support but does not enumerate an effort scale. Effort scales are surface-specific: the Codex CLI config reference types model_reasoning_effort as minimal|low|medium|high|xhigh (xhigh is model-dependent), and OpenRouter endpoint metadata lists supported_efforts [max, xhigh, high, medium, low, none] with default medium for openai/gpt-5.6-luna. OpenAI publishes no per-tier effort ceiling; Ringer passes effort per-task via engine_args [\"-c\",\"model_reasoning_effort=...\"]." +max_output_interaction = "max_output_tokens limits total generated tokens, including reasoning tokens, visible output tokens, and non-visible formatting tokens. If the cap is exhausted during reasoning, the response can be incomplete before any visible output is produced." + +[limits] +context_window = 1050000 +max_output_tokens = 128000 +rate_limits = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request (per the gpt-5.6-luna model page). ChatGPT-plan Codex OAuth usage is governed by plan entitlements, not API billing tiers; per-tier long-context API rate-limit tiers are not published on the model page." + +[openrouter] +upstream_providers = ["openai"] +require_parameters_behavior = "not applicable; this registry entry is for OpenAI direct/Codex OAuth access. gpt-5.6-luna is also listed on OpenRouter as openai/gpt-5.6-luna (see sources); OpenRouter routing would pass the model slug through the opencode engine's model field." + +[pricing] +prompt_per_m = 1.00 +completion_per_m = 6.00 +cache_read_per_m = 0.10 +cache_write_per_m = 1.25 +as_of = "2026-07-16" + +[[sources]] +claim = "model: gpt-5.6-luna context window" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-luna" +accessed = "2026-07-14" +quote = "1,050,000 context window" + +[[sources]] +claim = "model: gpt-5.6-luna max output tokens" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-luna" +accessed = "2026-07-14" +quote = "128,000 max output tokens" + +[[sources]] +claim = "pricing: gpt-5.6-luna input, cached input, cache writes, and output per 1M" +url = "https://developers.openai.com/api/docs/pricing" +accessed = "2026-07-16" +quote = "Input Cached input Cache writes Output ... gpt-5.6-luna $1.00 $0.10 $1.25 $6.00" +note = "The pricing page's GPT-5.6 rows carry an explicit Cache writes column (older families show '-'). The $1.25/M cache-write price is independently corroborated by OpenRouter endpoint metadata (input_cache_write = 0.00000125 for openai/gpt-5.6-luna, accessed 2026-07-16)." + +[[sources]] +claim = "limits: gpt-5.6-luna large-prompt (>272K) pricing rule" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-luna" +accessed = "2026-07-14" +quote = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request" + +[[sources]] +claim = "model: gpt-5.6-luna knowledge cutoff" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-luna" +accessed = "2026-07-14" +quote = "Feb 16, 2026 knowledge cutoff" + +[[sources]] +claim = "model: gpt-5.6-luna fine-tuning not supported" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-luna" +accessed = "2026-07-16" +quote = "Fine-tuning Not supported" + +[[sources]] +claim = "reasoning: Codex CLI reasoning-effort scale" +url = "https://developers.openai.com/codex/config-reference" +accessed = "2026-07-16" +quote = "minimal | low | medium | high | xhigh" +note = "The config reference types model_reasoning_effort with this enum ('xhigh is model-dependent') and does not include 'max'." + +[[sources]] +claim = "reasoning: OpenRouter supported reasoning efforts for openai/gpt-5.6-luna" +url = "https://openrouter.ai/api/v1/models" +accessed = "2026-07-16" +quote = "\"supported_efforts\":[\"max\",\"xhigh\",\"high\",\"medium\",\"low\",\"none\"],\"default_effort\":\"medium\"" + +[[sources]] +claim = "tool_calling: strict structured outputs use JSON Schema strict true (platform, version-independent)" +url = "https://developers.openai.com/api/docs/guides/structured-outputs" +accessed = "2026-07-16" +quote = "strict: true" + +[[sources]] +claim = "caching: GPT-5.6 cache writes are billed at 1.25x the uncached input rate" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "On GPT-5.6 models and later model families, cache writes are billed at 1.25× the uncached input token rate and reported in `cache_write_tokens`." +note = "Direct OpenAI source (replaces an earlier search-relayed quote from openai.com/index/gpt-5-6, which 403s to direct fetch). 1.25x of $1.00 input = $1.25/M, matching the pricing page's Cache writes column." + +[[sources]] +claim = "caching: explicit cache breakpoints and 30m minimum cache life" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "use `prompt_cache_options.ttl` to set the minimum lifetime of all breakpoints written by the request. The only supported value is `30m`, which is also the default." +note = "The guide also states: 'you can mark the end of a reusable prompt prefix with an explicit cache breakpoint. Breakpoints are available in both the Responses API and Chat Completions API.', documents `prompt_cache_options.mode` with implicit as the default, requires prompt_cache_key for the more reliable matching, and deprecates prompt_cache_retention for GPT-5.6 and later." + +[[sources]] +claim = "caching: 1024-token floor for cached_tokens" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "For requests under 1024 tokens, `cached_tokens` is zero." + +[[sources]] +claim = "release: GPT-5.6 family (Sol/Terra/Luna) GA date (third-party, single source)" +url = "https://www.digitalapplied.com/blog/gpt-5-6-sol-terra-luna-public-ga" +accessed = "2026-07-14" +quote = "GPT-5.6 went to general availability on July 9, 2026 — across ChatGPT, Codex, and the OpenAI API" +note = "release_date records the public GA date, not first availability (pre-GA access via Codex existed for the family; see docs/MODEL-NOTES.md, 'GPT-5.5 (codex) — attribution caveat'). Do not use release_date as a lower bound for scoreboard row attribution." + +[[sources]] +claim = "openrouter: gpt-5.6-luna listing and cache-write price metadata" +url = "https://openrouter.ai/api/v1/models" +accessed = "2026-07-16" +quote = "\"id\": \"openai/gpt-5.6-luna\", \"name\": \"OpenAI: GPT-5.6 Luna\" ... \"input_cache_read\": \"0.0000001\", \"input_cache_write\": \"0.00000125\"" diff --git a/registry/model-capabilities/gpt-5.6-sol.toml b/registry/model-capabilities/gpt-5.6-sol.toml new file mode 100644 index 0000000..7c546b0 --- /dev/null +++ b/registry/model-capabilities/gpt-5.6-sol.toml @@ -0,0 +1,139 @@ +[model] +key = "gpt-5.6-sol" +display = "GPT-5.6 Sol" +vendor = "OpenAI" +release_date = "2026-07-09" +identity_verification = "OpenAI's developer model page for gpt-5.6-sol lists it as a member of the GPT-5.6 family (Sol/Terra/Luna) with a 1,050,000 context window, 128,000 max output tokens, reasoning token support, and a Feb 16, 2026 knowledge cutoff. GA of the GPT-5.6 family is reported as 2026-07-09 by third-party coverage (single source); the model page carries the Feb 16 2026 knowledge cutoff but no in-page GA date." + +[api] +endpoint_families = ["responses", "chat-completions", "batch"] +auth = "In this setup, Codex CLI signs in with ChatGPT/OAuth plan credentials and uses ChatGPT workspace permissions, plan access, and ChatGPT-managed Codex entitlements. Codex CLI can also sign in with an OpenAI API key; API-key usage is billed through the OpenAI Platform account at standard API rates instead of included ChatGPT plan credits." +streaming = "Supported. API streaming is available with stream=true on Responses and Chat Completions; streaming responses use server-sent events. Family-level (GPT-5.6) behavior; the per-tier model page does not separately enumerate a streaming flag." + +[caching] +supported = true +mode = "implicit" +how_enabled = "Prompt caching works automatically for eligible requests with no code changes; prompt_cache_options.mode defaults to implicit (the mode field records this default). For GPT-5.6, explicit cache breakpoints are additionally available in both the Responses and Chat Completions APIs, and prompt_cache_key must be set to get the more reliable cache matching. Caching starts at 1024 prompt tokens (cached_tokens is zero for requests under 1024); repeated prompt prefixes, tools, and structured-output schemas can contribute to the cached prefix." +ttl = "For GPT-5.6, prompt_cache_options.ttl sets a MINIMUM cache lifetime; the only supported value is 30m (also the default) — a cached prefix remains eligible for reuse for at least 30 minutes and may be retained longer. The prompt_cache_retention field used by earlier families (24h on GPT-5.5) is deprecated for GPT-5.6 and later." +read_pricing_per_m = 0.50 +write_pricing_per_m = 6.25 +min_cacheable_prefix_tokens = 1024 + +[tool_calling] +format = "OpenAI Responses API tools and function calls with JSON Schema tool parameters. Chat Completions function calling is also supported. Structured Outputs are supported, including strict JSON Schema mode with strict=true. Family-level GPT-5.6 behavior inherited from the OpenAI platform; not separately re-verified on the per-tier model page." +parallel = true +strict_json_schema = true +quirks = [ + "For reasoning models using function calling through the Responses API, pass back reasoning items, function call items, and function call output items during tool loops.", + "Prompt-only JSON mode can still produce incomplete JSON when output is cut off by max_output_tokens or a content filter; use Structured Outputs for schema adherence.", +] + +[reasoning] +params = ["reasoning.effort", "text.verbosity", "max_output_tokens", "model_reasoning_effort", "model_reasoning_summary", "model_verbosity"] +defaults = "The gpt-5.6-sol model page confirms reasoning token support but does not enumerate an effort scale. Effort scales are surface-specific: the Codex CLI config reference types model_reasoning_effort as minimal|low|medium|high|xhigh (xhigh is model-dependent), and OpenRouter endpoint metadata lists supported_efforts [max, xhigh, high, medium, low, none] with default medium for openai/gpt-5.6-sol. OpenAI publishes no per-tier effort ceiling; Ringer passes effort per-task via engine_args [\"-c\",\"model_reasoning_effort=...\"]." +max_output_interaction = "max_output_tokens limits total generated tokens, including reasoning tokens, visible output tokens, and non-visible formatting tokens. If the cap is exhausted during reasoning, the response can be incomplete before any visible output is produced." + +[limits] +context_window = 1050000 +max_output_tokens = 128000 +rate_limits = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request (per the gpt-5.6-sol model page). ChatGPT-plan Codex OAuth usage is governed by plan entitlements, not API billing tiers; per-tier long-context API rate-limit tiers are not published on the model page." + +[openrouter] +upstream_providers = ["openai"] +require_parameters_behavior = "not applicable; this registry entry is for OpenAI direct/Codex OAuth access. gpt-5.6-sol is also listed on OpenRouter as openai/gpt-5.6-sol (see sources); OpenRouter routing would pass the model slug through the opencode engine's model field." + +[pricing] +prompt_per_m = 5.00 +completion_per_m = 30.00 +cache_read_per_m = 0.50 +cache_write_per_m = 6.25 +as_of = "2026-07-16" + +[[sources]] +claim = "model: gpt-5.6-sol context window" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-sol" +accessed = "2026-07-14" +quote = "1,050,000 context window" + +[[sources]] +claim = "model: gpt-5.6-sol max output tokens" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-sol" +accessed = "2026-07-14" +quote = "128,000 max output tokens" + +[[sources]] +claim = "model: gpt-5.6-sol reasoning token support" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-sol" +accessed = "2026-07-14" +quote = "Reasoning token support" + +[[sources]] +claim = "pricing: gpt-5.6-sol input, cached input, cache writes, and output per 1M" +url = "https://developers.openai.com/api/docs/pricing" +accessed = "2026-07-16" +quote = "Input Cached input Cache writes Output gpt-5.6-sol $5.00 $0.50 $6.25 $30.00" +note = "The pricing page's GPT-5.6 rows carry an explicit Cache writes column (older families show '-'). The $6.25/M cache-write price is independently corroborated by OpenRouter endpoint metadata (input_cache_write = 0.00000625 for openai/gpt-5.6-sol, accessed 2026-07-16)." + +[[sources]] +claim = "limits: gpt-5.6-sol large-prompt (>272K) pricing rule" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-sol" +accessed = "2026-07-14" +quote = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request." + +[[sources]] +claim = "model: gpt-5.6-sol knowledge cutoff" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-sol" +accessed = "2026-07-14" +quote = "Feb 16, 2026 knowledge cutoff" + +[[sources]] +claim = "caching: GPT-5.6 cache writes are billed at 1.25x the uncached input rate" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "On GPT-5.6 models and later model families, cache writes are billed at 1.25× the uncached input token rate and reported in `cache_write_tokens`." +note = "Direct OpenAI source (replaces an earlier search-relayed quote from openai.com/index/gpt-5-6, which 403s to direct fetch). 1.25x of $5.00 input = $6.25/M, matching the pricing page's Cache writes column." + +[[sources]] +claim = "caching: explicit cache breakpoints and 30m minimum cache life" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "use `prompt_cache_options.ttl` to set the minimum lifetime of all breakpoints written by the request. The only supported value is `30m`, which is also the default." +note = "The guide also states: 'you can mark the end of a reusable prompt prefix with an explicit cache breakpoint. Breakpoints are available in both the Responses API and Chat Completions API.', documents `prompt_cache_options.mode` with implicit as the default, requires prompt_cache_key for the more reliable matching, and deprecates prompt_cache_retention for GPT-5.6 and later." + +[[sources]] +claim = "caching: 1024-token floor for cached_tokens" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "For requests under 1024 tokens, `cached_tokens` is zero." + +[[sources]] +claim = "reasoning: Codex CLI reasoning-effort scale" +url = "https://developers.openai.com/codex/config-reference" +accessed = "2026-07-16" +quote = "minimal | low | medium | high | xhigh" +note = "The config reference types model_reasoning_effort with this enum ('xhigh is model-dependent') and does not include 'max'. A 2026-07-13 harness observation on one machine had codex accept model_reasoning_effort=max with gpt-5.6-sol; the surfaces disagree — recorded as a dated observation in docs/MODEL-NOTES.md, not as a sheet capability." + +[[sources]] +claim = "reasoning: OpenRouter supported reasoning efforts for openai/gpt-5.6-sol" +url = "https://openrouter.ai/api/v1/models" +accessed = "2026-07-16" +quote = "\"supported_efforts\":[\"max\",\"xhigh\",\"high\",\"medium\",\"low\",\"none\"],\"default_effort\":\"medium\"" + +[[sources]] +claim = "tool_calling: strict structured outputs use JSON Schema strict true (platform, version-independent)" +url = "https://developers.openai.com/api/docs/guides/structured-outputs" +accessed = "2026-07-16" +quote = "strict: true" + +[[sources]] +claim = "release: GPT-5.6 family (Sol/Terra/Luna) GA date (third-party, single source)" +url = "https://www.digitalapplied.com/blog/gpt-5-6-sol-terra-luna-public-ga" +accessed = "2026-07-14" +quote = "GPT-5.6 went to general availability on July 9, 2026 — across ChatGPT, Codex, and the OpenAI API" +note = "release_date records the public GA date, not first availability: this machine's Codex default had moved to gpt-5.6-sol at an unknown earlier date (see docs/MODEL-NOTES.md, 'GPT-5.5 (codex) — attribution caveat'). Do not use release_date as a lower bound for scoreboard row attribution." + +[[sources]] +claim = "openrouter: gpt-5.6-sol listing and cache-write price metadata" +url = "https://openrouter.ai/api/v1/models" +accessed = "2026-07-16" +quote = "\"id\": \"openai/gpt-5.6-sol\", \"name\": \"OpenAI: GPT-5.6 Sol\" ... \"input_cache_read\": \"0.0000005\", \"input_cache_write\": \"0.00000625\"" diff --git a/registry/model-capabilities/gpt-5.6-terra.toml b/registry/model-capabilities/gpt-5.6-terra.toml new file mode 100644 index 0000000..dc9ec2b --- /dev/null +++ b/registry/model-capabilities/gpt-5.6-terra.toml @@ -0,0 +1,133 @@ +[model] +key = "gpt-5.6-terra" +display = "GPT-5.6 Terra" +vendor = "OpenAI" +release_date = "2026-07-09" +identity_verification = "OpenAI's developer model page for gpt-5.6-terra lists it as a member of the GPT-5.6 family (Sol/Terra/Luna) with a 1,050,000 context window, 128,000 max output tokens, and a Feb 16, 2026 knowledge cutoff. GA of the GPT-5.6 family is reported as 2026-07-09 by third-party coverage (single source); OpenAI's model page states the Feb 16 2026 knowledge cutoff but no in-page GA date." + +[api] +endpoint_families = ["responses", "chat-completions", "batch"] +auth = "In this setup, Codex CLI signs in with ChatGPT/OAuth plan credentials and uses ChatGPT workspace permissions, plan access, and ChatGPT-managed Codex entitlements. Codex CLI can also sign in with an OpenAI API key; API-key usage is billed through the OpenAI Platform account at standard API rates instead of included ChatGPT plan credits." +streaming = "Supported. API streaming is available with stream=true on Responses and Chat Completions; streaming responses use server-sent events. Family-level (GPT-5.6) behavior; the per-tier model page does not separately enumerate a streaming flag." + +[caching] +supported = true +mode = "implicit" +how_enabled = "Prompt caching works automatically for eligible requests with no code changes; prompt_cache_options.mode defaults to implicit (the mode field records this default). For GPT-5.6, explicit cache breakpoints are additionally available in both the Responses and Chat Completions APIs, and prompt_cache_key must be set to get the more reliable cache matching. Caching starts at 1024 prompt tokens (cached_tokens is zero for requests under 1024); repeated prompt prefixes, tools, and structured-output schemas can contribute to the cached prefix." +ttl = "For GPT-5.6, prompt_cache_options.ttl sets a MINIMUM cache lifetime; the only supported value is 30m (also the default) — a cached prefix remains eligible for reuse for at least 30 minutes and may be retained longer. The prompt_cache_retention field used by earlier families (24h on GPT-5.5) is deprecated for GPT-5.6 and later." +read_pricing_per_m = 0.25 +write_pricing_per_m = 3.125 +min_cacheable_prefix_tokens = 1024 + +[tool_calling] +format = "OpenAI Responses API tools and function calls with JSON Schema tool parameters. Chat Completions function calling is also supported. Structured Outputs are supported, including strict JSON Schema mode with strict=true. Family-level GPT-5.6 behavior inherited from the OpenAI platform; not separately re-verified on the per-tier model page." +parallel = true +strict_json_schema = true +quirks = [ + "For reasoning models using function calling through the Responses API, pass back reasoning items, function call items, and function call output items during tool loops.", + "Prompt-only JSON mode can still produce incomplete JSON when output is cut off by max_output_tokens or a content filter; use Structured Outputs for schema adherence.", +] + +[reasoning] +params = ["reasoning.effort", "text.verbosity", "max_output_tokens", "model_reasoning_effort", "model_reasoning_summary", "model_verbosity"] +defaults = "The gpt-5.6-terra model page confirms reasoning token support but does not enumerate an effort scale. Effort scales are surface-specific: the Codex CLI config reference types model_reasoning_effort as minimal|low|medium|high|xhigh (xhigh is model-dependent), and OpenRouter endpoint metadata lists supported_efforts [max, xhigh, high, medium, low, none] with default medium for openai/gpt-5.6-terra. OpenAI publishes no per-tier effort ceiling; Ringer passes effort per-task via engine_args [\"-c\",\"model_reasoning_effort=...\"]." +max_output_interaction = "max_output_tokens limits total generated tokens, including reasoning tokens, visible output tokens, and non-visible formatting tokens. If the cap is exhausted during reasoning, the response can be incomplete before any visible output is produced." + +[limits] +context_window = 1050000 +max_output_tokens = 128000 +rate_limits = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request (per the gpt-5.6-terra model page). ChatGPT-plan Codex OAuth usage is governed by plan entitlements, not API billing tiers; per-tier long-context API rate-limit tiers are not published on the model page." + +[openrouter] +upstream_providers = ["openai"] +require_parameters_behavior = "not applicable; this registry entry is for OpenAI direct/Codex OAuth access. gpt-5.6-terra is also listed on OpenRouter as openai/gpt-5.6-terra (see sources); OpenRouter routing would pass the model slug through the opencode engine's model field." + +[pricing] +prompt_per_m = 2.50 +completion_per_m = 15.00 +cache_read_per_m = 0.25 +cache_write_per_m = 3.125 +as_of = "2026-07-16" + +[[sources]] +claim = "model: gpt-5.6-terra context window" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-terra" +accessed = "2026-07-14" +quote = "1,050,000 context window" + +[[sources]] +claim = "model: gpt-5.6-terra max output tokens" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-terra" +accessed = "2026-07-14" +quote = "128,000 max output tokens" + +[[sources]] +claim = "pricing: gpt-5.6-terra input, cached input, cache writes, and output per 1M" +url = "https://developers.openai.com/api/docs/pricing" +accessed = "2026-07-16" +quote = "Input Cached input Cache writes Output ... gpt-5.6-terra $2.50 $0.25 $3.125 $15.00" +note = "The pricing page's GPT-5.6 rows carry an explicit Cache writes column (older families show '-'). The $3.125/M cache-write price is independently corroborated by OpenRouter endpoint metadata (input_cache_write = 0.000003125 for openai/gpt-5.6-terra, accessed 2026-07-16)." + +[[sources]] +claim = "limits: gpt-5.6-terra large-prompt (>272K) pricing rule" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-terra" +accessed = "2026-07-14" +quote = "Prompts with >272K input tokens are priced at 2x input and 1.5x output for the full request." + +[[sources]] +claim = "model: gpt-5.6-terra knowledge cutoff" +url = "https://developers.openai.com/api/docs/models/gpt-5.6-terra" +accessed = "2026-07-14" +quote = "Feb 16, 2026 knowledge cutoff" + +[[sources]] +claim = "caching: GPT-5.6 cache writes are billed at 1.25x the uncached input rate" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "On GPT-5.6 models and later model families, cache writes are billed at 1.25× the uncached input token rate and reported in `cache_write_tokens`." +note = "Direct OpenAI source (replaces an earlier search-relayed quote from openai.com/index/gpt-5-6, which 403s to direct fetch). 1.25x of $2.50 input = $3.125/M, matching the pricing page's Cache writes column." + +[[sources]] +claim = "caching: explicit cache breakpoints and 30m minimum cache life" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "use `prompt_cache_options.ttl` to set the minimum lifetime of all breakpoints written by the request. The only supported value is `30m`, which is also the default." +note = "The guide also states: 'you can mark the end of a reusable prompt prefix with an explicit cache breakpoint. Breakpoints are available in both the Responses API and Chat Completions API.', documents `prompt_cache_options.mode` with implicit as the default, requires prompt_cache_key for the more reliable matching, and deprecates prompt_cache_retention for GPT-5.6 and later." + +[[sources]] +claim = "caching: 1024-token floor for cached_tokens" +url = "https://developers.openai.com/api/docs/guides/prompt-caching" +accessed = "2026-07-16" +quote = "For requests under 1024 tokens, `cached_tokens` is zero." + +[[sources]] +claim = "reasoning: Codex CLI reasoning-effort scale" +url = "https://developers.openai.com/codex/config-reference" +accessed = "2026-07-16" +quote = "minimal | low | medium | high | xhigh" +note = "The config reference types model_reasoning_effort with this enum ('xhigh is model-dependent') and does not include 'max'." + +[[sources]] +claim = "reasoning: OpenRouter supported reasoning efforts for openai/gpt-5.6-terra" +url = "https://openrouter.ai/api/v1/models" +accessed = "2026-07-16" +quote = "\"supported_efforts\":[\"max\",\"xhigh\",\"high\",\"medium\",\"low\",\"none\"],\"default_effort\":\"medium\"" + +[[sources]] +claim = "tool_calling: strict structured outputs use JSON Schema strict true (platform, version-independent)" +url = "https://developers.openai.com/api/docs/guides/structured-outputs" +accessed = "2026-07-14" +quote = "strict: true" + +[[sources]] +claim = "release: GPT-5.6 family (Sol/Terra/Luna) GA date (third-party, single source)" +url = "https://www.digitalapplied.com/blog/gpt-5-6-sol-terra-luna-public-ga" +accessed = "2026-07-14" +quote = "GPT-5.6 went to general availability on July 9, 2026 — across ChatGPT, Codex, and the OpenAI API" +note = "release_date records the public GA date, not first availability (pre-GA access via Codex existed for the family; see docs/MODEL-NOTES.md, 'GPT-5.5 (codex) — attribution caveat'). Do not use release_date as a lower bound for scoreboard row attribution." + +[[sources]] +claim = "openrouter: gpt-5.6-terra listing and cache-write price metadata" +url = "https://openrouter.ai/api/v1/models" +accessed = "2026-07-16" +quote = "\"id\": \"openai/gpt-5.6-terra\", \"name\": \"OpenAI: GPT-5.6 Terra\" ... \"input_cache_read\": \"0.00000025\", \"input_cache_write\": \"0.000003125\"" diff --git a/tests/test_capability_sheets.py b/tests/test_capability_sheets.py new file mode 100644 index 0000000..e1c3fd2 --- /dev/null +++ b/tests/test_capability_sheets.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Structural checks for registry/model-capabilities/*.toml. + +Every model sheet must parse as TOML, carry the required sections, declare a +unique model key, and back its claims with source records that carry a claim, +an http(s) url, an ISO accessed date, and a quote. Platform sheets (no [model] +table) must be explicitly listed in PLATFORM_SHEETS and only need well-formed +sources — any other sheet missing its [model] table fails instead of being +silently exempted. +""" +from __future__ import annotations + +import tomllib +import unittest +from datetime import date +from pathlib import Path +from urllib.parse import urlparse + +ROOT = Path(__file__).resolve().parents[1] +SHEETS_DIR = ROOT / "registry" / "model-capabilities" + +# Sheets that intentionally have no [model] table (platform-level facts). +PLATFORM_SHEETS = {"openrouter-platform.toml"} + +REQUIRED_MODEL_SECTIONS = { + "model", + "api", + "caching", + "tool_calling", + "reasoning", + "limits", + "pricing", + "sources", +} + +# (caching key, pricing key) pairs that duplicate the same fact; when both are +# numeric they must agree. +MIRRORED_PRICE_FIELDS = ( + ("read_pricing_per_m", "cache_read_per_m"), + ("write_pricing_per_m", "cache_write_per_m"), +) + +_SHEETS: dict[Path, dict] | None = None + + +def load_sheets() -> dict[Path, dict]: + """Parse every sheet once. Unparseable files are skipped here so the other + tests still cover the rest of the corpus; test_every_sheet_parses is the + single test that attributes the parse error to its sheet.""" + global _SHEETS + if _SHEETS is None: + _SHEETS = {} + for path in sorted(SHEETS_DIR.glob("*.toml")): + try: + with path.open("rb") as fh: + _SHEETS[path] = tomllib.load(fh) + except tomllib.TOMLDecodeError: + continue + return _SHEETS + + +def model_sheets() -> dict[Path, dict]: + return { + path: data + for path, data in load_sheets().items() + if path.name not in PLATFORM_SHEETS + } + + +class TestCapabilitySheets(unittest.TestCase): + def test_sheets_dir_has_sheets(self) -> None: + self.assertTrue( + sorted(SHEETS_DIR.glob("*.toml")), f"no sheets found in {SHEETS_DIR}" + ) + + def test_every_sheet_parses(self) -> None: + for path in sorted(SHEETS_DIR.glob("*.toml")): + with self.subTest(sheet=path.name): + with path.open("rb") as fh: + tomllib.load(fh) + + def test_model_sheets_have_required_sections(self) -> None: + for path, data in model_sheets().items(): + with self.subTest(sheet=path.name): + missing = REQUIRED_MODEL_SECTIONS - set(data) + self.assertFalse( + missing, + f"{path.name} is missing required sections: {sorted(missing)}" + " (platform sheets must be listed in PLATFORM_SHEETS)", + ) + + def test_model_keys_present_and_unique(self) -> None: + seen: dict[str, str] = {} + for path, data in model_sheets().items(): + with self.subTest(sheet=path.name): + key = data.get("model", {}).get("key") + self.assertTrue( + isinstance(key, str) and key.strip(), + f"{path.name} has no non-empty model.key", + ) + self.assertNotIn( + key, + seen, + f"model.key {key!r} appears in both {seen.get(key)} and {path.name}", + ) + seen[key] = path.name + + def test_mirrored_cache_prices_agree(self) -> None: + for path, data in model_sheets().items(): + caching = data.get("caching", {}) + pricing = data.get("pricing", {}) + for caching_key, pricing_key in MIRRORED_PRICE_FIELDS: + a, b = caching.get(caching_key), pricing.get(pricing_key) + if isinstance(a, (int, float)) and isinstance(b, (int, float)): + with self.subTest(sheet=path.name, field=pricing_key): + self.assertEqual( + a, + b, + f"{path.name}: caching.{caching_key}={a} disagrees " + f"with pricing.{pricing_key}={b}", + ) + + def test_sources_are_complete_records(self) -> None: + for path, data in load_sheets().items(): + sources = data.get("sources", []) + with self.subTest(sheet=path.name): + self.assertTrue(sources, f"{path.name} has no [[sources]] records") + for i, src in enumerate(sources): + with self.subTest(sheet=path.name, source=i): + for field in ("claim", "quote"): + value = src.get(field) + self.assertTrue( + isinstance(value, str) and value.strip(), + f"{path.name} source #{i} has no non-empty {field}", + ) + url = src.get("url", "") + parsed = urlparse(url if isinstance(url, str) else "") + self.assertTrue( + parsed.scheme in ("http", "https") and parsed.netloc, + f"{path.name} source #{i} url {url!r} is not http(s)", + ) + accessed = str(src.get("accessed", "")) + try: + date.fromisoformat(accessed) + except ValueError: + self.fail( + f"{path.name} source #{i} accessed date {accessed!r} " + "is not a valid ISO date" + ) + + +if __name__ == "__main__": + unittest.main()