diff --git a/src/harness/providers/openai/sse.rs b/src/harness/providers/openai/sse.rs index e1cd506..5ae76e4 100644 --- a/src/harness/providers/openai/sse.rs +++ b/src/harness/providers/openai/sse.rs @@ -291,12 +291,15 @@ impl SseState { .or_else(|| error.get("type")) .and_then(Value::as_str) .map(str::to_string); + let retryable = + crate::harness::retry::classify_provider_failure(None, code.as_deref(), &message) + .is_retryable(); ProviderError { provider: self.provider.clone(), model: Some(self.model.clone()), code, message, - retryable: false, + retryable, raw: Some(error.clone()), ..ProviderError::default() } diff --git a/src/harness/providers/openai/transport.rs b/src/harness/providers/openai/transport.rs index a98ddb8..6674138 100644 --- a/src/harness/providers/openai/transport.rs +++ b/src/harness/providers/openai/transport.rs @@ -469,13 +469,16 @@ impl OpenAiModel { code: Option, raw: Option, ) -> ProviderError { - let retryable = status.is_some_and(|s| s == 408 || s == 409 || s == 429 || s >= 500); + let message = message.into(); + let retryable = + crate::harness::retry::classify_provider_failure(status, code.as_deref(), &message) + .is_retryable(); ProviderError { provider: self.provider.clone(), model: Some(self.model.clone()), status, code, - message: message.into(), + message, retryable, raw, } diff --git a/src/harness/retry/mod.rs b/src/harness/retry/mod.rs index 6b7f3f4..e23e40e 100644 --- a/src/harness/retry/mod.rs +++ b/src/harness/retry/mod.rs @@ -30,6 +30,217 @@ use std::sync::Mutex; use std::time::{Duration, Instant}; use crate::error::TinyAgentsError; +use crate::harness::model::ProviderError; + +// ── Provider failure classification ───────────────────────────────────────── + +fn parse_status_at(text: &str, start: usize) -> Option { + let digits: String = text + .get(start..)? + .trim_start() + .chars() + .take_while(|ch| ch.is_ascii_digit()) + .collect(); + if digits.len() == 3 { + digits.parse().ok() + } else { + None + } +} + +fn find_case_insensitive(haystack: &str, needle: &str) -> Option { + haystack + .to_ascii_lowercase() + .find(&needle.to_ascii_lowercase()) +} + +/// Extracts an HTTP status from structured provider error text. +/// +/// Recognized positions are provider envelopes like `API error (404): ...`, +/// explicit `HTTP 404`, `status: 404` / `status 404`, or a status code at the +/// beginning of the message. Free-text digit runs such as latency values or +/// model ids are ignored. +pub fn structured_http_status(message: &str) -> Option { + let trimmed = message.trim_start(); + if let Some(status) = parse_status_at(trimmed, 0) { + return Some(status); + } + + for (idx, _) in message.match_indices('(') { + if let Some(status) = parse_status_at(message, idx + 1) { + return Some(status); + } + } + + for marker in ["http ", "status:", "status "] { + if let Some(idx) = find_case_insensitive(message, marker) + && let Some(status) = parse_status_at(message, idx + marker.len()) + { + return Some(status); + } + } + + None +} + +fn is_retryable_status(status: u16) -> bool { + matches!(status, 408 | 409 | 429) || status >= 500 +} + +fn is_upstream_unhealthy_status(status: u16) -> bool { + matches!(status, 408 | 409) || status >= 500 +} + +fn text_indicates_rate_limit(lower: &str) -> bool { + lower.contains("429") + && (lower.contains("too many") || lower.contains("rate") || lower.contains("limit")) +} + +fn text_indicates_upstream_unhealthy(lower: &str) -> bool { + lower.contains("no healthy upstream") + || lower.contains("upstream unavailable") + || lower.contains("service unavailable") + || lower.contains("408 request timeout") + || lower.contains("409 conflict") + || lower.contains("500 internal server error") + || lower.contains("502 bad gateway") + || lower.contains("503 service unavailable") + || lower.contains("504 gateway timeout") +} + +fn text_indicates_non_retryable(lower: &str) -> bool { + let auth_failure_hints = [ + "invalid api key", + "incorrect api key", + "missing api key", + "api key not set", + "authentication failed", + "auth failed", + "unauthorized", + "forbidden", + "permission denied", + "access denied", + "invalid token", + ]; + if auth_failure_hints.iter().any(|hint| lower.contains(hint)) { + return true; + } + + lower.contains("model") + && (lower.contains("not found") + || lower.contains("unknown") + || lower.contains("unsupported") + || lower.contains("does not exist") + || lower.contains("invalid")) +} + +fn text_indicates_non_retryable_rate_limit(lower: &str) -> bool { + let business_hints = [ + "plan does not include", + "doesn't include", + "not include", + "insufficient balance", + "insufficient_balance", + "insufficient quota", + "insufficient_quota", + "quota exhausted", + "out of credits", + "no available package", + "package not active", + "purchase package", + "model not available for your plan", + ]; + if business_hints.iter().any(|hint| lower.contains(hint)) { + return true; + } + + lower.split(|ch: char| !ch.is_ascii_digit()).any(|token| { + token + .parse::() + .is_ok_and(|code| matches!(code, 1113 | 1311)) + }) +} + +/// Classifies a normalized provider failure from generic HTTP status, provider +/// error code/type, and message details. +/// +/// Host applications may layer their own account, billing, or product-specific +/// terminal rules before or after this helper. TinyAgents only classifies +/// provider-neutral retry behavior. +pub fn classify_provider_failure( + status: Option, + code: Option<&str>, + message: &str, +) -> ProviderFailureClass { + let status = status.or_else(|| structured_http_status(message)); + let lower = match code { + Some(code) if !code.trim().is_empty() => format!("{message} {code}").to_ascii_lowercase(), + _ => message.to_ascii_lowercase(), + }; + + if status == Some(429) || text_indicates_rate_limit(&lower) { + if text_indicates_non_retryable_rate_limit(&lower) { + return ProviderFailureClass::NonRetryableRateLimit; + } + return ProviderFailureClass::RateLimited; + } + + if let Some(status) = status { + if is_upstream_unhealthy_status(status) { + return ProviderFailureClass::UpstreamUnhealthy; + } + if (400..500).contains(&status) && !is_retryable_status(status) { + return ProviderFailureClass::NonRetryable; + } + } + + if text_indicates_upstream_unhealthy(&lower) { + return ProviderFailureClass::UpstreamUnhealthy; + } + if text_indicates_non_retryable(&lower) { + return ProviderFailureClass::NonRetryable; + } + + ProviderFailureClass::Retryable +} + +/// Classifies a normalized [`ProviderError`]. +pub fn classify_provider_error(error: &ProviderError) -> ProviderFailureClass { + classify_provider_failure(error.status, error.code.as_deref(), &error.message) +} + +/// Computes the retryability flag for a normalized [`ProviderError`]. +pub fn provider_error_is_retryable(error: &ProviderError) -> bool { + classify_provider_error(error).is_retryable() +} + +/// Parses a `Retry-After` / `retry_after` value from provider error text into +/// milliseconds. Integer and fractional seconds are accepted. +pub fn parse_retry_after_ms(message: &str) -> Option { + let lower = message.to_ascii_lowercase(); + for prefix in &[ + "retry-after:", + "retry_after:", + "retry-after ", + "retry_after ", + ] { + if let Some(pos) = lower.find(prefix) { + let after = &message[pos + prefix.len()..]; + let number: String = after + .trim_start() + .chars() + .take_while(|ch| ch.is_ascii_digit() || *ch == '.') + .collect(); + if let Ok(seconds) = number.parse::() + && seconds.is_finite() + && seconds >= 0.0 + { + return u64::try_from(Duration::from_secs_f64(seconds).as_millis()).ok(); + } + } + } + None +} // ── RetryPolicy ────────────────────────────────────────────────────────────── diff --git a/src/harness/retry/test.rs b/src/harness/retry/test.rs index adfcd7d..306fa53 100644 --- a/src/harness/retry/test.rs +++ b/src/harness/retry/test.rs @@ -5,7 +5,10 @@ use std::time::{Duration, Instant}; -use super::{FallbackPolicy, RateLimiter, RetryPolicy, is_retryable}; +use super::{ + FallbackPolicy, ProviderFailureClass, RateLimiter, RetryPolicy, classify_provider_error, + classify_provider_failure, is_retryable, parse_retry_after_ms, structured_http_status, +}; use crate::error::TinyAgentsError; #[test] @@ -166,6 +169,143 @@ fn provider_error_retryability_is_read_from_the_structured_flag_not_assumed() { )))); } +#[test] +fn structured_http_status_uses_only_anchored_positions() { + assert_eq!( + structured_http_status("custom_openai API error (403 Forbidden): nope"), + Some(403) + ); + assert_eq!(structured_http_status("HTTP 404 Not Found"), Some(404)); + assert_eq!(structured_http_status("status: 401"), Some(401)); + assert_eq!(structured_http_status("408 Request Timeout"), Some(408)); + + assert_eq!( + structured_http_status("upstream took 450ms to respond, retrying"), + None + ); + assert_eq!( + structured_http_status("gpt-4-0409 returned an empty completion"), + None + ); + assert_eq!( + structured_http_status("received 412 partial bytes before reset"), + None + ); +} + +#[test] +fn provider_failure_classifies_generic_http_statuses() { + assert_eq!( + classify_provider_failure(Some(401), None, "invalid api key"), + ProviderFailureClass::NonRetryable + ); + assert_eq!( + classify_provider_failure(Some(404), None, "model not found"), + ProviderFailureClass::NonRetryable + ); + assert_eq!( + classify_provider_failure(Some(429), None, "too many requests"), + ProviderFailureClass::RateLimited + ); + assert_eq!( + classify_provider_failure(Some(408), None, "request timeout"), + ProviderFailureClass::UpstreamUnhealthy + ); + assert_eq!( + classify_provider_failure(Some(502), None, "bad gateway"), + ProviderFailureClass::UpstreamUnhealthy + ); +} + +#[test] +fn provider_failure_classifies_message_hints_without_status() { + assert_eq!( + classify_provider_failure(None, None, "authentication failed"), + ProviderFailureClass::NonRetryable + ); + assert_eq!( + classify_provider_failure(None, None, "model glm-4.7 is unsupported"), + ProviderFailureClass::NonRetryable + ); + assert_eq!( + classify_provider_failure(None, None, "no healthy upstream available"), + ProviderFailureClass::UpstreamUnhealthy + ); + assert_eq!( + classify_provider_failure(None, None, "429 Too Many Requests: rate limit exceeded"), + ProviderFailureClass::RateLimited + ); +} + +#[test] +fn provider_failure_classifies_non_retryable_rate_limits() { + assert_eq!( + classify_provider_failure( + Some(429), + Some("1311"), + "the current account plan does not include glm-5" + ), + ProviderFailureClass::NonRetryableRateLimit + ); + assert_eq!( + classify_provider_failure(Some(429), None, "insufficient balance"), + ProviderFailureClass::NonRetryableRateLimit + ); +} + +#[test] +fn provider_failure_class_controls_retryability_and_reason_labels() { + assert!(ProviderFailureClass::RateLimited.is_retryable()); + assert!(ProviderFailureClass::UpstreamUnhealthy.is_retryable()); + assert!(!ProviderFailureClass::NonRetryable.is_retryable()); + assert!(!ProviderFailureClass::NonRetryableRateLimit.is_retryable()); + + assert_eq!(ProviderFailureClass::Retryable.reason(), "retryable"); + assert_eq!(ProviderFailureClass::NonRetryable.reason(), "non_retryable"); + assert_eq!(ProviderFailureClass::RateLimited.reason(), "rate_limited"); + assert_eq!( + ProviderFailureClass::NonRetryableRateLimit.reason(), + "rate_limited_non_retryable" + ); + assert_eq!( + ProviderFailureClass::UpstreamUnhealthy.reason(), + "upstream_unhealthy" + ); +} + +#[test] +fn classify_provider_error_reads_structured_error_fields() { + use crate::harness::model::ProviderError; + + let provider_error = ProviderError { + provider: "openai".to_string(), + model: Some("gpt-4o".to_string()), + status: Some(429), + code: Some("insufficient_quota".to_string()), + message: "insufficient quota".to_string(), + ..ProviderError::default() + }; + + assert_eq!( + classify_provider_error(&provider_error), + ProviderFailureClass::NonRetryableRateLimit + ); +} + +#[test] +fn retry_after_parser_accepts_integer_float_and_space_separators() { + assert_eq!( + parse_retry_after_ms("429 Too Many Requests, Retry-After: 5"), + Some(5_000) + ); + assert_eq!( + parse_retry_after_ms("Rate limited. retry_after: 2.5 seconds"), + Some(2_500) + ); + assert_eq!(parse_retry_after_ms("Retry-After 7"), Some(7_000)); + assert_eq!(parse_retry_after_ms("500 Internal Server Error"), None); +} + // ── FallbackPolicy::next_after ──────────────────────────────────────────────── #[test] diff --git a/src/harness/retry/types.rs b/src/harness/retry/types.rs index b0e373b..b88285e 100644 --- a/src/harness/retry/types.rs +++ b/src/harness/retry/types.rs @@ -118,3 +118,43 @@ pub(crate) struct RateLimiterState { /// Last time the bucket was refilled. pub(crate) last_refill: std::time::Instant, } + +/// Provider-failure class used for retry, fallback, and telemetry decisions. +/// +/// The class is intentionally provider-neutral: callers can layer product, +/// billing, or account-specific rules on top, while TinyAgents supplies the +/// generic HTTP/status/error-message behavior shared by hosted model adapters. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ProviderFailureClass { + /// A transient upstream failure such as timeout, rate limit, or 5xx. + Retryable, + /// A permanent caller/account/model error such as 400/401/403/404. + NonRetryable, + /// A generic 429 rate limit where retrying after backoff may succeed. + RateLimited, + /// A 429 carrying quota, balance, plan, or package exhaustion detail. + NonRetryableRateLimit, + /// A provider-side outage/capacity failure such as 408/409/5xx. + UpstreamUnhealthy, +} + +impl ProviderFailureClass { + /// Whether retrying the same request may succeed. + pub fn is_retryable(self) -> bool { + matches!( + self, + Self::Retryable | Self::RateLimited | Self::UpstreamUnhealthy + ) + } + + /// Stable reason label suitable for logs and telemetry dimensions. + pub fn reason(self) -> &'static str { + match self { + Self::Retryable => "retryable", + Self::NonRetryable => "non_retryable", + Self::RateLimited => "rate_limited", + Self::NonRetryableRateLimit => "rate_limited_non_retryable", + Self::UpstreamUnhealthy => "upstream_unhealthy", + } + } +}