fix: surface http errors correctly

engine/baml-runtime/src/cli/serve/error.rs

@@ -92,6 +92,9 @@ impl BamlError {
                     message: format!("Unexpected error from BAML: {err:?}"),
                 },
                 LLMResponse::LLMFailure(failed) => match &failed.code {
+                    crate::internal::llm_client::ErrorCode::FailedToConnect => Self::ClientError {
+                        message: format!("Failed to connect to the LLM provider: {err:?}"),
+                    },
                     crate::internal::llm_client::ErrorCode::Other(2) => Self::InternalError {
                         message: format!("Something went wrong with the LLM client: {err:?}"),
                     },

engine/baml-runtime/src/internal/llm_client/mod.rs

@@ -211,6 +211,12 @@ pub struct LLMErrorResponse {
 
 #[derive(Debug, Clone, Serialize, PartialEq)]
 pub enum ErrorCode {
+    // Failed to establish a connection
+    // Tends to happen when either (1) user enters the wrong connection details
+    // (e.g. wrong AWS region and therefore URL) or (2) in prod it's been
+    // chugging along and then the server provider goes hard down.
+    FailedToConnect,
+
     InvalidAuthentication, // 401
     NotSupported,          // 403
     RateLimited,           // 429
@@ -227,6 +233,7 @@ pub enum ErrorCode {
 impl std::fmt::Display for ErrorCode {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
+            ErrorCode::FailedToConnect => f.write_str("Failed while establishing connection"),
             ErrorCode::InvalidAuthentication => f.write_str("InvalidAuthentication (401)"),
             ErrorCode::NotSupported => f.write_str("NotSupported (403)"),
             ErrorCode::RateLimited => f.write_str("RateLimited (429)"),
@@ -263,6 +270,10 @@ impl ErrorCode {
 
     pub fn to_u16(&self) -> u16 {
         match self {
+            // FAILED_TO_CONNECT maps to 2 because the internal callsites usually used ErrorCode::Other(2)
+            // for connection errors. It's unclear if this actually makes its way out to users in any
+            // meaningful way.
+            ErrorCode::FailedToConnect => 2,
             ErrorCode::InvalidAuthentication => 401,
             ErrorCode::NotSupported => 403,
             ErrorCode::RateLimited => 429,

engine/baml-runtime/src/internal/llm_client/primitive/aws/aws_client.rs

@@ -1245,29 +1245,13 @@ impl WithChat for AwsClient {
                     latency: instant_start.elapsed(),
                     message: format!("{e:#?}"),
                     code: match e {
-                        SdkError::ConstructionFailure(_) => ErrorCode::Other(2),
+                        SdkError::ConstructionFailure(_) => ErrorCode::FailedToConnect,
                         SdkError::TimeoutError(_) => ErrorCode::Other(2),
-                        SdkError::DispatchFailure(_) => ErrorCode::Other(2),
+                        SdkError::DispatchFailure(_) => ErrorCode::FailedToConnect,
                         SdkError::ResponseError(e) => {
                             ErrorCode::UnsupportedResponse(e.raw().status().as_u16())
                         }
-                        SdkError::ServiceError(e) => {
-                            let status = e.raw().status();
-                            match status.as_u16() {
-                                400 => ErrorCode::InvalidAuthentication,
-                                403 => ErrorCode::NotSupported,
-                                429 => ErrorCode::RateLimited,
-                                500 => ErrorCode::ServerError,
-                                503 => ErrorCode::ServiceUnavailable,
-                                _ => {
-                                    if status.is_server_error() {
-                                        ErrorCode::ServerError
-                                    } else {
-                                        ErrorCode::Other(status.as_u16())
-                                    }
-                                }
-                            }
-                        }
+                        SdkError::ServiceError(e) => ErrorCode::from_u16(e.raw().status().as_u16()),
                         _ => ErrorCode::Other(2),
                     },
                 });

engine/baml-runtime/src/internal/llm_client/primitive/openai/openai_client.rs

@@ -488,7 +488,12 @@ impl WithStreamChat for OpenAIClient {
 }
 
 macro_rules! make_openai_client {
-    ($client:ident, $properties:ident, $provider:expr, dynamic) => {
+    ($client:ident, $properties:ident, $provider:expr, dynamic) => {{
+        let resolve_pdf_urls = if $provider == "openai-responses" {
+            ResolveMediaUrls::Never
+        } else {
+            ResolveMediaUrls::Always
+        };
         Ok(Self {
             name: $client.name.clone(),
             provider: $provider.into(),
@@ -506,16 +511,21 @@ macro_rules! make_openai_client {
                 max_one_system_prompt: false,
                 resolve_audio_urls: ResolveMediaUrls::Always,
                 resolve_image_urls: ResolveMediaUrls::Never,
-                resolve_pdf_urls: ResolveMediaUrls::Never,
+                resolve_pdf_urls,
                 resolve_video_urls: ResolveMediaUrls::Never,
                 allowed_metadata: $properties.allowed_metadata.clone(),
             },
             properties: $properties,
             retry_policy: $client.retry_policy.clone(),
             client: create_client()?,
         })
-    };
-    ($client:ident, $properties:ident, $provider:expr) => {
+    }};
+    ($client:ident, $properties:ident, $provider:expr) => {{
+        let resolve_pdf_urls = if $provider == "openai-responses" {
+            ResolveMediaUrls::Never
+        } else {
+            ResolveMediaUrls::Always
+        };
         Ok(Self {
             name: $client.name().into(),
             provider: $provider.into(),
@@ -533,7 +543,7 @@ macro_rules! make_openai_client {
                 max_one_system_prompt: false,
                 resolve_audio_urls: ResolveMediaUrls::Always,
                 resolve_image_urls: ResolveMediaUrls::Never,
-                resolve_pdf_urls: ResolveMediaUrls::Never,
+                resolve_pdf_urls,
                 resolve_video_urls: ResolveMediaUrls::Never,
                 allowed_metadata: $properties.allowed_metadata.clone(),
             },
@@ -545,7 +555,7 @@ macro_rules! make_openai_client {
                 .map(|s| s.to_string()),
             client: create_client()?,
         })
-    };
+    }};
 }
 
 impl OpenAIClient {
@@ -714,8 +724,12 @@ impl ToProviderMessage for OpenAIClient {
                 match &media.content {
                     BamlMediaContent::Url(url_content) => {
                         // For URLs, we need to resolve them to base64 first
-                        anyhow::bail!(
-                            "BAML internal error (openai): Pdf URL are not supported by OpenAI use base64."
+                        content.insert(
+                            payload_key.into(),
+                            json!({
+                                "type": "input_file",
+                                "file_url": url_content.url
+                            }),
                         );
                     }
                     BamlMediaContent::Base64(b64_media) => {

engine/baml-runtime/src/internal/llm_client/primitive/request.rs

@@ -182,7 +182,11 @@ pub(crate) async fn build_and_log_outbound_request(
                 request_options: client.request_options().clone(),
                 latency: instant_now.elapsed(),
                 message: format!("Failed to build request: {e:#?}"),
-                code: ErrorCode::Other(2),
+                code: if e.is_connect() {
+                    ErrorCode::FailedToConnect
+                } else {
+                    ErrorCode::Other(2)
+                },
             }));
         }
     };
@@ -253,9 +257,12 @@ pub async fn execute_request(
                         )
                     }
                 },
-                code: e
-                    .status()
-                    .map_or(ErrorCode::Other(2), ErrorCode::from_status),
+                code: if e.is_connect() {
+                    ErrorCode::FailedToConnect
+                } else {
+                    e.status()
+                        .map_or(ErrorCode::Other(2), ErrorCode::from_status)
+                },
             }));
         }
     };

engine/baml-runtime/src/types/response.rs

@@ -123,9 +123,26 @@ impl FunctionResult {
     }
 
     fn format_err(&self, err: &anyhow::Error) -> ExposedError {
+        // panic!("format_err {:?}", err);
         if let Some(exposed_error) = err.downcast_ref::<ExposedError>() {
             return exposed_error.clone();
         }
+
+        if let LLMResponse::LLMFailure(err) = self.llm_response() {
+            if err.code == ErrorCode::FailedToConnect {
+                let actual_error = err.message.clone();
+                return ExposedError::ClientHttpError {
+                    client_name: match self.llm_response() {
+                        LLMResponse::Success(resp) => resp.client.clone(),
+                        LLMResponse::LLMFailure(err) => err.client.clone(),
+                        _ => "unknown".to_string(),
+                    },
+                    message: actual_error,
+                    status_code: ErrorCode::FailedToConnect,
+                };
+            }
+        }
+
         // Capture the actual error to preserve its details
         let actual_error = err.to_string();
         // TODO: HACK! Figure out why now connection errors dont get converted into ExposedError. Instead of converting to a validation error, check for connection errors here. We probably are missing a lot of other connection failures that should NOT be validation errors.
@@ -137,9 +154,10 @@ impl FunctionResult {
                     _ => "unknown".to_string(),
                 },
                 message: actual_error,
-                status_code: ErrorCode::ServiceUnavailable,
+                status_code: ErrorCode::FailedToConnect,
             };
         }
+
         ExposedError::ValidationError {
             prompt: match self.llm_response() {
                 LLMResponse::Success(resp) => resp.prompt.to_string(),

engine/language_client_python/src/errors.rs

@@ -131,6 +131,7 @@ impl BamlError {
                         ))
                     }
                     baml_runtime::internal::llm_client::ErrorCode::Other(_)
+                    | baml_runtime::internal::llm_client::ErrorCode::FailedToConnect
                     | baml_runtime::internal::llm_client::ErrorCode::InvalidAuthentication
                     | baml_runtime::internal::llm_client::ErrorCode::NotSupported
                     | baml_runtime::internal::llm_client::ErrorCode::RateLimited

engine/language_client_typescript/src/errors.rs

@@ -57,6 +57,7 @@ pub fn from_anyhow_error(err: anyhow::Error) -> napi::Error {
                     ),
                 ),
                 baml_runtime::internal::llm_client::ErrorCode::Other(_)
+                | baml_runtime::internal::llm_client::ErrorCode::FailedToConnect
                 | baml_runtime::internal::llm_client::ErrorCode::InvalidAuthentication
                 | baml_runtime::internal::llm_client::ErrorCode::NotSupported
                 | baml_runtime::internal::llm_client::ErrorCode::RateLimited

integ-tests/baml_src/test-files/abort-handlers/abort-handlers.baml

@@ -73,8 +73,8 @@ function FnFailRetryExponentialDelay(retries: int, initial_delay_ms: int) -> str
 function TestAbortFallbackChain(input: string) -> string {
   client AbortTestFallback
   prompt #"
-    This is a test for fallback chain cancellation.
-    Please fail so we test the fallback behavior.
+    Tell me a 200-word story about tigers.
+
     Input: {{ input }}
   "#
 }