Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/harness/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ pub use message::{ContentBlock, Message};
pub use model::{
CapabilitySet, Modalities, ModelProfile, ModelRequest, ModelResponse, ModelStatus, ModelStream,
ModelStreamItem, ProviderError, ResponseFormat, StreamAccumulator, ToolChoice,
collect_model_stream,
collect_model_stream, context_window_for_model_id,
};
pub use no_progress::{
DEFAULT_IDENTICAL_HALT_THRESHOLD, NoProgress, NoProgressTracker, ToolAttempt,
Expand Down
76 changes: 76 additions & 0 deletions src/harness/model/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,82 @@ use crate::harness::usage::Usage;

pub use types::*;

/// How a context-window pattern is matched against a model id.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ContextPatternMatch {
/// Pattern may appear anywhere in the lowercased model id.
Substring,
/// Pattern must be a complete segment delimited by common provider/id
/// separators. This avoids false positives for short model ids such as
/// `o1` and `o3`.
Segment,
}

/// Generic context-window hints for common provider model families.
///
/// These are deliberately provider-neutral fallbacks, not a pricing catalog.
/// Hosts should prefer authoritative provider/catalog metadata when available
/// and use this only when a raw model id needs a conservative pre-dispatch
/// budget.
///
/// Order matters: lookup returns the first matching entry, so more-specific
/// substrings such as `gpt-4.1` and `gpt-4-turbo` must stay before broader
/// patterns such as `gpt-4` that would otherwise shadow them.
const MODEL_CONTEXT_PATTERNS: &[(&str, ContextPatternMatch, u64)] = &[
("claude-haiku-4.5", ContextPatternMatch::Substring, 200_000),
("claude-haiku-4", ContextPatternMatch::Substring, 200_000),
("claude-haiku", ContextPatternMatch::Substring, 200_000),
("claude-sonnet-4", ContextPatternMatch::Substring, 200_000),
("claude-opus-4", ContextPatternMatch::Substring, 200_000),
("claude-3-5-sonnet", ContextPatternMatch::Substring, 200_000),
("claude-3-5-haiku", ContextPatternMatch::Substring, 200_000),
("claude-3-opus", ContextPatternMatch::Substring, 200_000),
("gpt-4.1", ContextPatternMatch::Substring, 1_047_576),
("gpt-4o", ContextPatternMatch::Substring, 128_000),
("gpt-4-turbo", ContextPatternMatch::Substring, 128_000),
("gpt-4", ContextPatternMatch::Substring, 128_000),

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Keep the gpt-4 fallback at 8k

OpenAI lists the plain gpt-4 model's context window as 8,192 tokens (https://developers.openai.com/api/docs/models/gpt-4), so context_window_for_model_id("gpt-4") now returns a 128k budget for a model that rejects prompts above 8k. Because this helper is intended for pre-dispatch budgeting, callers that use the plain gpt-4 id can skip summarization/compaction and hit provider context-limit errors; keep the 128k value scoped to gpt-4-turbo/gpt-4o variants.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Keep the gpt-4 fallback at 8k

OpenAI lists the plain gpt-4 model's context window as 8,192 tokens (https://developers.openai.com/api/docs/models/gpt-4), so context_window_for_model_id("gpt-4") now returns a 128k budget for a model that rejects prompts above 8k. Because this helper is intended for pre-dispatch budgeting, callers that use the plain gpt-4 id can skip summarization/compaction and hit provider context-limit errors; keep the 128k value scoped to gpt-4-turbo/gpt-4o variants.

Useful? React with 👍 / 👎.

("gpt-3.5", ContextPatternMatch::Substring, 16_385),
("o1", ContextPatternMatch::Segment, 200_000),
("o3", ContextPatternMatch::Segment, 200_000),
("deepseek", ContextPatternMatch::Substring, 128_000),
("gemma3", ContextPatternMatch::Substring, 8_192),

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Return Gemma 3's input context window

For Gemma 3 ids such as gemma3:4b (the added test covers this exact Ollama-style id), this returns 8,192 even though Google's Gemma 3 model card lists 128K input context for the 4B/12B/27B sizes and 32K for 1B/270M: https://ai.google.dev/gemma/docs/core/model_card_3. Since this helper is meant to feed pre-dispatch and capability budgeting, using the 8K value will unnecessarily reject/compact long prompts that those Gemma 3 models can accept; split Gemma 3 from the older Gemma fallback or make the pattern size-aware.

Useful? React with 👍 / 👎.

("gemma", ContextPatternMatch::Substring, 8_192),
("llama-3", ContextPatternMatch::Substring, 128_000),
("llama3", ContextPatternMatch::Substring, 128_000),
Comment on lines +70 to +71

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Don't assign Llama 3 the Llama 3.1 window

Meta's Llama 3 model card lists an 8k context length for the Llama 3 8B/70B models (https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct), while the 128k window belongs to later Llama 3.1 models. With these generic patterns, context_window_for_model_id("llama3:8b") returns 128k, so local/hosted Llama 3 callers can over-budget by 16x and fail at runtime; add separate llama-3.1/llama3.1 patterns or lower the plain Llama 3 fallback.

Useful? React with 👍 / 👎.

Comment on lines +70 to +71

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Don't assign Llama 3 the Llama 3.1 window

Meta's Llama 3 model card lists an 8k context length for the Llama 3 8B/70B models (https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct), while the 128k window belongs to later Llama 3.1 models. With these generic patterns, context_window_for_model_id("llama3:8b") returns 128k, so local/hosted Llama 3 callers can over-budget by 16x and fail at runtime; add separate llama-3.1/llama3.1 patterns or lower the plain Llama 3 fallback.

Useful? React with 👍 / 👎.

];

fn matches_context_pattern(lower: &str, pattern: &str, mode: ContextPatternMatch) -> bool {
match mode {
ContextPatternMatch::Substring => lower.contains(pattern),
ContextPatternMatch::Segment => {
let model_name = lower.rsplit(['/', ':']).next().unwrap_or(lower);
model_name
.split(['-', '_', '.'])
.next()
.is_some_and(|segment| segment == pattern)
}
}
}

/// Returns a generic context-window hint for a raw provider model id.
///
/// Returns `None` for unknown ids rather than guessing. Hosts with product tier
/// aliases, local runtime profiles, or authoritative provider catalogs should
/// check those first and use this helper as a last generic fallback.
pub fn context_window_for_model_id(model: &str) -> Option<u64> {
let normalized = model.trim();
if normalized.is_empty() {
return None;
}

let lower = normalized.to_ascii_lowercase();
MODEL_CONTEXT_PATTERNS
.iter()
.find_map(|(pattern, mode, window)| {
matches_context_pattern(&lower, pattern, *mode).then_some(*window)
})
}

impl std::fmt::Display for ProviderError {
/// Renders the same human-readable shape real provider adapters used to
/// build by hand before flattening it into a plain
Expand Down
37 changes: 37 additions & 0 deletions src/harness/model/test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,43 @@ fn lifecycle_helpers_gate_retired_and_deprecated_models() {
assert!(retired.is_deprecated());
}

#[test]
fn context_window_patterns_cover_common_provider_families() {
assert_eq!(context_window_for_model_id("gpt-4.1"), Some(1_047_576));
assert_eq!(
context_window_for_model_id("openai/gpt-4o-mini"),
Some(128_000)
);
assert_eq!(
context_window_for_model_id("github_copilot/claude-haiku-4.5"),
Some(200_000)
);
assert_eq!(context_window_for_model_id("deepseek-chat"), Some(128_000));
assert_eq!(context_window_for_model_id("gemma3:4b"), Some(8_192));
assert_eq!(context_window_for_model_id("llama3:8b"), Some(128_000));
assert_eq!(context_window_for_model_id("totally-unknown-model"), None);
assert_eq!(context_window_for_model_id(" "), None);
}

#[test]
fn o1_o3_context_patterns_require_segment_boundaries() {
assert_eq!(context_window_for_model_id("o1"), Some(200_000));
assert_eq!(context_window_for_model_id("o1-mini"), Some(200_000));
assert_eq!(context_window_for_model_id("o3-mini"), Some(200_000));
assert_eq!(
context_window_for_model_id("openai/o1-preview"),
Some(200_000)
);

assert_eq!(context_window_for_model_id("solo1-7b"), None);
assert_eq!(context_window_for_model_id("proto3-chat"), None);
assert_eq!(context_window_for_model_id("octo3thing"), None);
assert_eq!(
context_window_for_model_id("ollama/mistral-for-o1-benchmark"),
None
);
}

#[test]
fn cacheable_prefix_ids_in_order() {
let req = ModelRequest::new(vec![]).with_cache_segments(vec![
Expand Down