diff --git a/integrations/langchain-py/src/braintrust_langchain/__init__.py b/integrations/langchain-py/src/braintrust_langchain/__init__.py index 2feeb7bc7..6345802d7 100644 --- a/integrations/langchain-py/src/braintrust_langchain/__init__.py +++ b/integrations/langchain-py/src/braintrust_langchain/__init__.py @@ -1,4 +1,4 @@ from .callbacks import BraintrustCallbackHandler -from .context import set_global_handler +from .context import clear_global_handler, set_global_handler -__all__ = ["BraintrustCallbackHandler", "set_global_handler"] +__all__ = ["BraintrustCallbackHandler", "set_global_handler", "clear_global_handler"] diff --git a/py/Makefile b/py/Makefile index 305fabf6f..fce06984e 100644 --- a/py/Makefile +++ b/py/Makefile @@ -55,7 +55,8 @@ install-dev: install-build-deps python -m uv pip install -r requirements-dev.txt install-optional: - python -m uv pip install anthropic openai pydantic_ai litellm agno google-genai dspy langsmith + python -m uv pip install anthropic openai pydantic_ai litellm agno google-genai dspy langsmith \ + langchain langchain-openai tenacity python -m uv pip install -e .[temporal,otel] .DEFAULT_GOAL := help diff --git a/py/examples/auto_instrument.py b/py/examples/auto_instrument.py index 33fa278a8..6e494c191 100644 --- a/py/examples/auto_instrument.py +++ b/py/examples/auto_instrument.py @@ -13,6 +13,7 @@ - Agno - Claude Agent SDK - DSPy +- LangChain """ import braintrust diff --git a/py/examples/langchain/auto.py b/py/examples/langchain/auto.py new file mode 100644 index 000000000..2433ab733 --- /dev/null +++ b/py/examples/langchain/auto.py @@ -0,0 +1,35 @@ +""" +Example: LangChain with auto_instrument() + +This example demonstrates automatic tracing of LangChain operations +using braintrust.auto_instrument(). + +Run with: python examples/langchain/auto.py +""" + +import braintrust + +# One-line instrumentation - call this BEFORE importing LangChain +results = braintrust.auto_instrument() +print(f"LangChain instrumented: {results.get('langchain', False)}") + +# Initialize logging +logger = braintrust.init_logger(project="langchain-auto-example") + +# Now import LangChain - all operations are automatically traced +from langchain_core.prompts import ChatPromptTemplate +from langchain_openai import ChatOpenAI + +# Create a simple chain +prompt = ChatPromptTemplate.from_template("What is {number} + {number}?") +model = ChatOpenAI(model="gpt-4o-mini") +chain = prompt | model + +# Wrap in a span to get a link +with braintrust.start_span(name="langchain_auto_example") as span: + print("Running LangChain chain...") + result = chain.invoke({"number": "5"}) + print(f"Result: {result.content}") + span.log(output=result.content) + +print(f"\nView trace: {span.link()}") diff --git a/py/examples/langchain/manual.py b/py/examples/langchain/manual.py new file mode 100644 index 000000000..080a92514 --- /dev/null +++ b/py/examples/langchain/manual.py @@ -0,0 +1,62 @@ +""" +Example: LangChain with manual setup + +This example demonstrates using setup_langchain() for global handler registration +and BraintrustCallbackHandler for per-call tracing. + +Run with: python examples/langchain/manual.py +""" + +import braintrust +from braintrust.wrappers.langchain import ( + BraintrustCallbackHandler, + set_global_handler, + setup_langchain, +) + +# Initialize logging +logger = braintrust.init_logger(project="langchain-manual-example") + +# Method 1: Global handler via setup_langchain() +# This registers a handler that traces ALL LangChain operations automatically +print("Method 1: Global handler") +setup_langchain() + +from langchain_core.prompts import ChatPromptTemplate +from langchain_openai import ChatOpenAI + +prompt = ChatPromptTemplate.from_template("What is the capital of {country}?") +model = ChatOpenAI(model="gpt-4o-mini") +chain = prompt | model + +# All operations are traced automatically +result = chain.invoke({"country": "France"}) +print(f" Capital: {result.content}\n") + + +# Method 2: Per-call handler +# This is useful when you want more control over which calls are traced +print("Method 2: Per-call handler") + +# Create a handler with a specific logger +handler = BraintrustCallbackHandler(logger=logger) + +# Pass the handler explicitly to chain.invoke() +result = chain.invoke( + {"country": "Japan"}, + config={"callbacks": [handler]} +) +print(f" Capital: {result.content}\n") + + +# Method 3: Global handler with custom handler instance +print("Method 3: Custom global handler") + +# Create a custom handler and set it globally +custom_handler = BraintrustCallbackHandler(logger=logger) +set_global_handler(custom_handler) + +result = chain.invoke({"country": "Brazil"}) +print(f" Capital: {result.content}\n") + +print("Check Braintrust dashboard for traces!") diff --git a/py/noxfile.py b/py/noxfile.py index 33d5644e8..9c60d1743 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -43,6 +43,7 @@ "openai", "openai-agents", # pydantic_ai is NOT included here - it has dedicated test sessions with version-specific handling + # langchain is NOT included here - it has dedicated test sessions with version-specific handling "autoevals", "braintrust_core", "litellm", @@ -73,6 +74,8 @@ DSPY_VERSIONS = (LATEST,) # temporalio 1.19.0+ requires Python >= 3.10; skip Python 3.9 entirely TEMPORAL_VERSIONS = (LATEST, "1.20.0", "1.19.0") +# langchain requires Python >= 3.10 +LANGCHAIN_VERSIONS = (LATEST, "0.3.27") @nox.session() @@ -193,6 +196,20 @@ def test_dspy(session, version): _run_tests(session, f"{WRAPPER_DIR}/test_dspy.py") +@nox.session() +@nox.parametrize("version", LANGCHAIN_VERSIONS, ids=LANGCHAIN_VERSIONS) +def test_langchain(session, version): + """Test LangChain integration.""" + # langchain requires Python >= 3.10 + if sys.version_info < (3, 10): + session.skip("langchain tests require Python >= 3.10") + _install_test_deps(session) + _install(session, "langchain", version) + session.install("langchain-openai", "langchain-anthropic", "langgraph") + _run_tests(session, f"{WRAPPER_DIR}/test_langchain.py") + _run_core_tests(session) + + @nox.session() @nox.parametrize("version", AUTOEVALS_VERSIONS, ids=AUTOEVALS_VERSIONS) def test_autoevals(session, version): @@ -267,6 +284,8 @@ def pylint(session): session.install("opentelemetry.instrumentation.openai") # langsmith is needed for the wrapper module but not in VENDOR_PACKAGES session.install("langsmith") + # langchain dependencies for the langchain wrapper + session.install("langchain", "langchain-openai", "langchain-anthropic", "langgraph") result = session.run("git", "ls-files", "**/*.py", silent=True, log=False) files = result.strip().splitlines() diff --git a/py/src/braintrust/auto.py b/py/src/braintrust/auto.py index b1b552f8c..9f9e31d18 100644 --- a/py/src/braintrust/auto.py +++ b/py/src/braintrust/auto.py @@ -35,6 +35,7 @@ def auto_instrument( agno: bool = True, claude_agent_sdk: bool = True, dspy: bool = True, + langchain: bool = True, ) -> dict[str, bool]: """ Auto-instrument supported AI/ML libraries for Braintrust tracing. @@ -54,6 +55,7 @@ def auto_instrument( agno: Enable Agno instrumentation (default: True) claude_agent_sdk: Enable Claude Agent SDK instrumentation (default: True) dspy: Enable DSPy instrumentation (default: True) + langchain: Enable LangChain instrumentation (default: True) Returns: Dict mapping integration name to whether it was successfully instrumented. @@ -91,6 +93,11 @@ def auto_instrument( from google.genai import Client client = Client() client.models.generate_content(model="gemini-2.0-flash", contents="Hello!") + + # LangChain + from langchain_openai import ChatOpenAI + model = ChatOpenAI(model="gpt-4o-mini") + model.invoke("Hello!") ``` """ results = {} @@ -111,6 +118,8 @@ def auto_instrument( results["claude_agent_sdk"] = _instrument_claude_agent_sdk() if dspy: results["dspy"] = _instrument_dspy() + if langchain: + results["langchain"] = _instrument_langchain() return results @@ -177,3 +186,11 @@ def _instrument_dspy() -> bool: return patch_dspy() return False + + +def _instrument_langchain() -> bool: + with _try_patch(): + from braintrust.wrappers.langchain import setup_langchain + + return setup_langchain() + return False diff --git a/py/src/braintrust/wrappers/cassettes/test_langchain/test_async_langchain_invoke b/py/src/braintrust/wrappers/cassettes/test_langchain/test_async_langchain_invoke new file mode 100644 index 000000000..3ecc362e9 --- /dev/null +++ b/py/src/braintrust/wrappers/cassettes/test_langchain/test_async_langchain_invoke @@ -0,0 +1,276 @@ +interactions: +- request: + body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "What is + 1 + 2?"}], "model": "claude-sonnet-4-20250514"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '110' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 0.68.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.68.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//dJBfSwMxEMS/yjGvpnB3bUUCvhR88aEg/gERCTFZ2uDd5kw2Ui333eWK + Rar4tLC/mWGYPYKHRp83pm4ublf5+uWT0/DYP9ysr+7XOx/uoCAfA00qytluCAopdtPD5hyyWBYo + 9NFTBw3X2eJpliMzyWwxa+t2WS+bBRRcZCEW6Kf9MVJoN5kPR6Opzqq2uqzmGJ8VssTBJLI5MjSI + vZGSGN8g01shdgTNpesUyqGa3iPwUMRIfCXO0M25grNuS8YlshIim1NBfeSJrP+PHb1TPg1b6inZ + ziz7v/of2mx/01EhFjlpN1fIlN6DIyOBEjSmPb1NHuP4BQAA//8DABaJlhKdAQAA + headers: + CF-RAY: + - 983cc1f7fda07e2d-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 23 Sep 2025 20:23:04 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 02af79b5-9b1a-4100-a05f-9235eb38bda4 + cf-cache-status: + - DYNAMIC + request-id: + - req_011CTRxS1WS9ia9upALgfUZK + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + via: + - 1.1 google + x-envoy-upstream-service-time: + - '1030' + status: + code: 200 + message: OK +- request: + body: '{"max_tokens":1024,"messages":[{"role":"user","content":"What is 1 + 2?"}],"model":"claude-sonnet-4-20250514"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '110' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 0.68.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.68.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//dJDdSgMxEEZfZfluTWG37YoGvLagN0VBRCSEZGhDdydrMimWsu8uWyxS + xauBOWd++I7oo6cOGq6zxdMsR2aS2XI2r+dt3TZLKAQPjT5vTN08397v1qG5WT+Ep/Z19bJfySMf + oCCHgSaLcrYbgkKK3dSwOYcslgUKLrIQC/Tb8ewLfU7kVDSa6qqaV3fVAuO7QpY4mEQ2R4YGsTdS + EuMbZPooxI6guXSdQjnd1UcEHooYiTviDN1cKzjrtmRcIishsrkU6jNPZP1/7Dw77adhSz0l25m2 + /+v/0Gb7m44KscjFdwuFTGkfHBkJlKAxheVt8hjHLwAAAP//AwBHCKHFnQEAAA== + headers: + CF-RAY: + - 99b0eabe4896b976-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:22:38 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '3000000' + anthropic-ratelimit-input-tokens-remaining: + - '3000000' + anthropic-ratelimit-input-tokens-reset: + - '2025-11-08T00:22:38Z' + anthropic-ratelimit-output-tokens-limit: + - '600000' + anthropic-ratelimit-output-tokens-remaining: + - '600000' + anthropic-ratelimit-output-tokens-reset: + - '2025-11-08T00:22:38Z' + anthropic-ratelimit-tokens-limit: + - '3600000' + anthropic-ratelimit-tokens-remaining: + - '3600000' + anthropic-ratelimit-tokens-reset: + - '2025-11-08T00:22:38Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CUuU6hWk8Jg8Bh2c4Vyty + retry-after: + - '23' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '1801' + status: + code: 200 + message: OK +- request: + body: '{"max_tokens":1024,"messages":[{"role":"user","content":"What is 1 + 2?"}],"model":"claude-sonnet-4-20250514"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '110' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 0.68.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.68.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA3SQTUvDQBCG/0p4r24gaRvRBQ9CDyJ4rBeRZbs7tNFkNu7OBkvJf5cUi1TxNDDP + Mx+8R/TBUwcN19nsqUyBmaRclYtq0VRNvYJC66HRp52p6qd1/7DZ3o8yjIe355v1o9tsm1soyGGg + 2aKU7I6gEEM3N2xKbRLLAgUXWIgF+uV49oU+Z3IqGnVxVSyKu2KJ6VUhSRhMJJsCQ4PYG8mR8Q0S + fWRiR9Ccu04hn+7qI1oeshgJ78QJur5WcNbtybhIVtrA5lKozjyS9f+x8+y8n4Y99RRtZ5r+r/9D + 6/1vOimELBffLRUSxbF1ZKSlCI05LG+jxzR9AQAA//8DAEp7u9udAQAA + headers: + CF-RAY: + - 99b0ebedd90d6897-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:23:27 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '3000000' + anthropic-ratelimit-input-tokens-remaining: + - '3000000' + anthropic-ratelimit-input-tokens-reset: + - '2025-11-08T00:23:26Z' + anthropic-ratelimit-output-tokens-limit: + - '600000' + anthropic-ratelimit-output-tokens-remaining: + - '600000' + anthropic-ratelimit-output-tokens-reset: + - '2025-11-08T00:23:26Z' + anthropic-ratelimit-tokens-limit: + - '3600000' + anthropic-ratelimit-tokens-remaining: + - '3600000' + anthropic-ratelimit-tokens-reset: + - '2025-11-08T00:23:26Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CUuUAHB8QqxGoW7TZyUaz + retry-after: + - '34' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '1851' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/wrappers/cassettes/test_langchain/test_chain_with_memory b/py/src/braintrust/wrappers/cassettes/test_langchain/test_chain_with_memory new file mode 100644 index 000000000..88cc88485 --- /dev/null +++ b/py/src/braintrust/wrappers/cassettes/test_langchain/test_chain_with_memory @@ -0,0 +1,332 @@ +interactions: +- request: + body: '{"messages": [{"content": "Assistant: Hello! How can I assist you today? + User: What''s your name?", "role": "user"}], "model": "gpt-4o-mini", "stream": + false}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '149' + content-type: + - application/json + host: + - localhost:8000 + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CJ3pSTx8NVvtJFY51xvv7gmxKCqAO\",\n \"object\": + \"chat.completion\",\n \"created\": 1758658986,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"Assistant: I don't have a personal + name, but you can call me Assistant. How can I help you today?\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 24,\n \"completion_tokens\": 23,\n \"total_tokens\": 47,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_560af6e559\"\n}\n" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-span-id,x-bt-span-export,x-bt-query-plan,x-bt-internal-trace-id + Connection: + - keep-alive + Date: + - Tue, 23 Sep 2025 20:23:06 GMT + Keep-Alive: + - timeout=5 + Transfer-Encoding: + - chunked + Vary: + - Origin + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 983cc206fc1f67ef-SJC + content-type: + - application/json + openai-organization: + - braintrust-data + openai-processing-ms: + - '755' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - _cfuvid=TIArUY3FKYo9t2vz5lADo0yFHggpjc9nkMoRBVQYfbA-1758658986949-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-bt-cached: + - MISS + x-bt-internal-trace-id: + - 899e875d60ba290b68341d027600a8fd + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '775' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999980' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3b64f4bb78c14e2ea80001681e34611d + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"Assistant: Hello! How can I assist you today? + User: What''s your name?","role":"user"}],"model":"gpt-4o-mini","stream":false}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '149' + content-type: + - application/json + cookie: + - __cf_bm=W_Ukgb.mz8e1GW7CfhzN.QQaN09_xQq1uTHm3a.dJdU-1762561359-1.0.1.1-6IrkySxpZaL.1C65iH0iOLFfere0JxHCiasT6bak.RihYFMyJgIz2OuYJqcUey8c5vicjtorNby_Z_GJX.ZMIHa6PyzVrhqgfZZmtnnn.sA; + _cfuvid=jwWMA4k30hLPwBwTSCIdIeS5.m1TkcdYLYTt4YSTZhI-1762561359243-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA4ySTW/bMAyG7/4VhC67xEW+s+YyFAO2ZNhp22HYUBiMRNvaZFGT5KRBkf8+2E5i + d22BXXTgw5fiS/IxARBaiTUIWWKUlTPp+x9fxg/u4yFuv20O9ezD98/7YvPn01fFxeFOjBoF736R + jBfVjeTKGYqabYelJ4zUVJ2sltPFcjJb3LagYkWmkRUupnNOK211Oh1P5+l4lU7entUla0lBrOFn + AgDw2L5Nn1bRg1jDeHSJVBQCFiTW1yQA4dk0EYEh6BDRRjHqoWQbybat3134Grag2L6JUOKeAMGR + D2zRgMWKRrCrIxy5BokWJBoDFcFVfAMbPrRoCyUZ12ZGVnh8N/zXU14HbLzb2pgBQGs5YjO71vH9 + mZyuHg0XzvMu/CMVubY6lJknDGwbPyGyEy09JQD37SzrJ+MRznPlYhb5N7XfTeddOdFvcABnZxg5 + ounj89XohWqZoojahMEuhERZkuqV/eKwVpoHIBl4ft7MS7U739oW/1O+B1KSi6Qy50lp+dRwn+ap + ue/X0q4zbhsWgfxeS8qiJt/sQVGOtemuToRjiFRlubYFeed1d3q5yxbLMeZLWixuRXJK/gIAAP// + AwCouO6tiAMAAA== + headers: + CF-RAY: + - 99b0eacf8822aaac-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:22:39 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '628' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '639' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999980' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_1009d84201314e5aa9ccdcbafeeac4af + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"Assistant: Hello! How can I assist you today? + User: What''s your name?","role":"user"}],"model":"gpt-4o-mini","stream":false}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '149' + content-type: + - application/json + cookie: + - __cf_bm=.AxQfRhAvElThVl_Qz9zUVdqz_GtBGXwRQ0TVPIg5pc-1762561407-1.0.1.1-klsoMaFKHjzxOrHy2Zfd8Sc76RDHsMXURLAaIzORncnm47NI1MY0BqqBGOEsVXlZb.RdqeqpxzGFhl8DlRDjy.SqRfa2B4zEYdKZqQ2kVB0; + _cfuvid=0ohSoYMS21h1NkHWl4FeeVCp5aK2KHeEjclSm1NY7yY-1762561407934-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jJLBbtswDIbvfgpO53hIgiRNcxmKAcN6GNbuMmBDYTASHauRRU2imwRF + gb3GXm9PUthp4nTLgF104Mef4k/yMQNQ1qgFKF2h6Dq4/P23L6P18vJm8+Pz1e16m8pPH25u7fbe + fJ1vlmrQKnh5T1oOqrea6+BILPs91pFQqK06upiNp7PRZDjvQM2GXCtbBcknnNfW23w8HE/y4UU+ + mr+oK7aaklrA9wwA4LF72z69oa1awHBwiNSUEq5ILY5JACqyayMKU7JJ0Isa9FCzF/Jd61cHvoBr + MOx///wlUOEDAUIKpG1pNXisaQDLRmDHDWj0oNE5qAmO8jfwkTcduoaKXOgyhQ3u3p3+HKlsErbu + fePcCUDvWbCdXuf57oU8HV06XoXIy/SHVJXW21QVkTCxbx0l4aA6+pQB3HXTbF4NSIXIdZBCeE3d + d+PJvpzqd3gGCgu6Pj6ZD85UKwwJWpdOtqE06opMr+xXh42xfAKyE89/N3Ou9t639av/Kd8DrSkI + mSJEMla/NtynRWov/F9pxxl3DatE8cFqKsRSbPdgqMTG7e9OpV0SqovS+hXFEO3++MpQTGdDLGc0 + nV6q7Cl7BgAA//8DAEMiDgGKAwAA + headers: + CF-RAY: + - 99b0ebffc94fed3b-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:23:28 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '680' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '708' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999980' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_e273cb6eb8624df78282659b4a19fffe + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/wrappers/cassettes/test_langchain/test_global_handler b/py/src/braintrust/wrappers/cassettes/test_langchain/test_global_handler new file mode 100644 index 000000000..ba9f4fa9c --- /dev/null +++ b/py/src/braintrust/wrappers/cassettes/test_langchain/test_global_handler @@ -0,0 +1,225 @@ +interactions: +- request: + body: '{"messages": [{"content": "What is 1 + 2?", "role": "user"}], "model": + "gpt-4o-mini", "frequency_penalty": 0.0, "n": 1, "presence_penalty": 0.0, "stream": + false, "temperature": 1.0, "top_p": 1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - localhost:8000 + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CJ44VUVp2sk1koSWXX64CaLEy1mWy\",\n \"object\": + \"chat.completion\",\n \"created\": 1758659919,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"1 + 2 equals 3.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 15,\n \"completion_tokens\": 8,\n \"total_tokens\": 23,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_560af6e559\"\n}\n" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-span-id,x-bt-span-export,x-bt-query-plan,x-bt-internal-trace-id + Connection: + - keep-alive + Date: + - Tue, 23 Sep 2025 20:38:40 GMT + Keep-Alive: + - timeout=5 + Transfer-Encoding: + - chunked + Vary: + - Origin + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 983cd8d01c33943a-SJC + content-type: + - application/json + openai-organization: + - braintrust-data + openai-processing-ms: + - '930' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - _cfuvid=XPwF0fhMV9JwjYuWwUMNbzPKxvSJ.HOkXEftYzjXRew-1758659920459-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-bt-cached: + - MISS + x-bt-internal-trace-id: + - 93acad0503781eb98ab6ea3412173537 + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '1026' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_181413148bbe4814a905514521d6dc34 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"What is 1 + 2?","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"top_p":1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jJJBb9swDIXv/hUCr4uL2IlTJ9et2y2HHQZ0Q2EoMm1rk0VNoocNRf77 + IDuN3a0FevGBHx/1Hs3HRAjQNRwEqE6y6p1J33/9XH7adscPRXGXnTb7+49fNse7/b2iYylhFRV0 + +o6Kn1Q3inpnkDXZCSuPkjFOzW53ebHLyiwfQU81mihrHadbSnttdZqv8226vk2z8qLuSCsMcBDf + EiGEeBy/0aet8TccxHr1VOkxBNkiHK5NQoAnEysgQ9CBpWVYzVCRZbSj9Uy8E7nAn4M0QWxull0e + myHI6NQOxiyAtJZYxqSjv4cLOV8dGWqdp1P4RwqNtjp0lUcZyMbXA5ODkZ4TIR7G5MOzMOA89Y4r + ph84PpcV0ziY9z3D8sKYWJq5nG9WLwyramSpTVgsDpRUHdazct6yHGpNC5AsIv/v5aXZU2xt27eM + n4FS6BjrynmstXqed27zGI/xtbbrikfDEND/0gor1ujjb6ixkYOZTgTCn8DYV422LXrn9XQnjauK + 3Vo2OyyKPSTn5C8AAAD//wMAcIbFgjUDAAA= + headers: + CF-RAY: + - 99b0f5db9f1cbffc-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:30:12 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=vfpKl6dvzcujjwigai_kp7UkNhR2ltT1SwFsT05VrS8-1762561812-1.0.1.1-UAyuy134RWxRUzjbClH59IJarw95du8Dl347lkXcDkbXBBx7vCmRuxRccJQB2f1T6oobZSgBj7O8hdaLY4hef6ypZ2uHUshy880EnptiWEY; + path=/; expires=Sat, 08-Nov-25 01:00:12 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=N6FAUGU_qhcPvlVWdt0kvrpbt1SzTvQ0v29fL2QCNbA-1762561812358-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '319' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '489' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3e940a310adf4d9a88c8da6b70645bb7 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/wrappers/cassettes/test_langchain/test_langchain_anthropic_integration b/py/src/braintrust/wrappers/cassettes/test_langchain/test_langchain_anthropic_integration new file mode 100644 index 000000000..6c396d025 --- /dev/null +++ b/py/src/braintrust/wrappers/cassettes/test_langchain/test_langchain_anthropic_integration @@ -0,0 +1,300 @@ +interactions: +- request: + body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "What is + 1 + 2?"}], "model": "claude-sonnet-4-20250514"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '110' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.68.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.68.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//dJBRSwMxEIT/yjGvpnDX9hQDvggegn9AEAkxWdvg3eZMNsVa7r/LFYtU + 8Wlhv5lhmAOCh8aQN6Zu7nZX3IXr+8/X9cNt213vu5fucYSC7EeaVZSz3RAUUuznh805ZLEsUBii + px4arrfF0yJHZpLFerGsl23dNmsouMhCLNBPh1Ok0MdsPh6NprqoltVNtcL0rJAljiaRzZGhQeyN + lMT4BpneC7EjaC59r1CO1fQBgcciRuIbcYZuLhWcdVsyLpGVENmcC+oTT2T9f+zknfNp3NJAyfam + Hf7qf2iz/U0nhVjkrN1KIVPaBUdGAiVozHt6mzym6QsAAP//AwD8n6CUnQEAAA== + headers: + CF-RAY: + - 983cc09c5f361679-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Tue, 23 Sep 2025 20:22:09 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 02af79b5-9b1a-4100-a05f-9235eb38bda4 + anthropic-ratelimit-input-tokens-limit: + - '30000' + anthropic-ratelimit-input-tokens-remaining: + - '30000' + anthropic-ratelimit-input-tokens-reset: + - '2025-09-23T20:22:09Z' + anthropic-ratelimit-output-tokens-limit: + - '8000' + anthropic-ratelimit-output-tokens-remaining: + - '8000' + anthropic-ratelimit-output-tokens-reset: + - '2025-09-23T20:22:09Z' + anthropic-ratelimit-requests-limit: + - '50' + anthropic-ratelimit-requests-remaining: + - '49' + anthropic-ratelimit-requests-reset: + - '2025-09-23T20:22:09Z' + anthropic-ratelimit-tokens-limit: + - '38000' + anthropic-ratelimit-tokens-remaining: + - '38000' + anthropic-ratelimit-tokens-reset: + - '2025-09-23T20:22:09Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CTRxMui53W9h6eXYGxUJb + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + via: + - 1.1 google + x-envoy-upstream-service-time: + - '1110' + status: + code: 200 + message: OK +- request: + body: '{"max_tokens":1024,"messages":[{"role":"user","content":"What is 1 + 2?"}],"model":"claude-sonnet-4-20250514"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '110' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.68.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.68.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//dJDLasMwEEV/xdxtFfAjCa2g29JVoetShJCmsRt75Eij9BH878WhpqSl + q4E5Zx7cE4bgqYeG6232tEqBmWS1XtVlvSk31RoKnYfGkHamrB5z87bPfPfwGa+Ph9f7401wL1so + yMdIs0Up2R1BIYZ+btiUuiSWBQousBAL9NNp8YXeZ3IuGlVxVdTFbdFgelZIEkYTyabA0CD2RnJk + fINEh0zsCJpz3yvk8119QsdjFiNhT5ygq62Cs64l4yJZ6QKbS6FceCTr/2PL7LyfxpYGirY3m+Gv + /0Or9jedFEKWi+8ahUTx2Dky0lGExhyWt9Fjmr4AAAD//wMARZkZqp0BAAA= + headers: + CF-RAY: + - 99b0eab2783f1758-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:22:36 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '3000000' + anthropic-ratelimit-input-tokens-remaining: + - '3000000' + anthropic-ratelimit-input-tokens-reset: + - '2025-11-08T00:22:36Z' + anthropic-ratelimit-output-tokens-limit: + - '600000' + anthropic-ratelimit-output-tokens-remaining: + - '600000' + anthropic-ratelimit-output-tokens-reset: + - '2025-11-08T00:22:36Z' + anthropic-ratelimit-tokens-limit: + - '3600000' + anthropic-ratelimit-tokens-remaining: + - '3600000' + anthropic-ratelimit-tokens-reset: + - '2025-11-08T00:22:36Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CUuU6ZRKcH4CRrH5o4j6b + retry-after: + - '24' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '1694' + status: + code: 200 + message: OK +- request: + body: '{"max_tokens":1024,"messages":[{"role":"user","content":"What is 1 + 2?"}],"model":"claude-sonnet-4-20250514"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '110' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.68.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.68.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//dJBNS8QwEIb/SnmvptDudhUCHvQkC4oieBEJIRl3y7aTmkz8Kv3v0sVF + VvE0MM8zH7wj+uCpg4brbPZUpsBMUjblolqsqlXdQKH10OjTxlT1fXPxcHXzubwOZ2vKb7fPu7vL + 9Q4K8jHQbFFKdkNQiKGbGzalNollgYILLMQC/TgefKH3meyLRl2cFIvivFhielJIEgYTyabA0CD2 + RnJkfINEL5nYETTnrlPI+7t6RMtDFiNhR5yg61MFZ92WjItkpQ1sjoXqwCNZ/x87zM77adhST9F2 + ZtX/9X9ovf1NJ4WQ5ei7pUKi+No6MtJShMYclrfRY5q+AAAA//8DAAqaanadAQAA + headers: + CF-RAY: + - 99b0ebe2db3d67ca-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:23:24 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '3000000' + anthropic-ratelimit-input-tokens-remaining: + - '3000000' + anthropic-ratelimit-input-tokens-reset: + - '2025-11-08T00:23:24Z' + anthropic-ratelimit-output-tokens-limit: + - '600000' + anthropic-ratelimit-output-tokens-remaining: + - '600000' + anthropic-ratelimit-output-tokens-reset: + - '2025-11-08T00:23:24Z' + anthropic-ratelimit-tokens-limit: + - '3600000' + anthropic-ratelimit-tokens-remaining: + - '3600000' + anthropic-ratelimit-tokens-reset: + - '2025-11-08T00:23:24Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CUuUA9cTfN1Yz5PMKHD5d + retry-after: + - '37' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '1556' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/wrappers/cassettes/test_langchain/test_langgraph_state_management b/py/src/braintrust/wrappers/cassettes/test_langchain/test_langgraph_state_management new file mode 100644 index 000000000..20ffc04b3 --- /dev/null +++ b/py/src/braintrust/wrappers/cassettes/test_langchain/test_langgraph_state_management @@ -0,0 +1,327 @@ +interactions: +- request: + body: '{"messages": [{"content": "Say hello", "role": "user"}], "model": "gpt-4o-mini", + "frequency_penalty": 0.0, "n": 1, "presence_penalty": 0.0, "stream": false, + "temperature": 1.0, "top_p": 1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '172' + content-type: + - application/json + host: + - localhost:8000 + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CJ3xSBjbTuwYXAmP3RRw0GoHz5Ooy\",\n \"object\": + \"chat.completion\",\n \"created\": 1758659482,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"Hello! How can I assist you today?\",\n + \ \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": + null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 9,\n \"completion_tokens\": 9,\n \"total_tokens\": 18,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_51db84afab\"\n}\n" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-span-id,x-bt-span-export,x-bt-query-plan,x-bt-internal-trace-id + Connection: + - keep-alive + Date: + - Tue, 23 Sep 2025 20:31:22 GMT + Keep-Alive: + - timeout=5 + Transfer-Encoding: + - chunked + Vary: + - Origin + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 983cce247c46cf2f-SJC + content-type: + - application/json + openai-organization: + - braintrust-data + openai-processing-ms: + - '381' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - _cfuvid=Y9Om0gYdHB3h9aUHhUUY9eEia6Y3wmSARFX9Xq907Ho-1758659482810-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-bt-cached: + - MISS + x-bt-internal-trace-id: + - ebcf889942216eb0b613f43f2cdb11b1 + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '397' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999995' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_75709538073646e4bd7355c91bc2ce52 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"Say hello","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"top_p":1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '172' + content-type: + - application/json + cookie: + - __cf_bm=W_Ukgb.mz8e1GW7CfhzN.QQaN09_xQq1uTHm3a.dJdU-1762561359-1.0.1.1-6IrkySxpZaL.1C65iH0iOLFfere0JxHCiasT6bak.RihYFMyJgIz2OuYJqcUey8c5vicjtorNby_Z_GJX.ZMIHa6PyzVrhqgfZZmtnnn.sA; + _cfuvid=jwWMA4k30hLPwBwTSCIdIeS5.m1TkcdYLYTt4YSTZhI-1762561359243-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA4xSwW7UMBC95ysGnzcoWbppu5eqqoSKgAu9oKIq8tqTrMHxGHuydKn235GTdpPS + InHxYd685/dm5iEDEEaLNQi1law6b/Or2y/F70p//Pz+UprNzf2uuO2uvrY/d9XN8pNYJAZtvqPi + J9ZbRZ23yIbcCKuAkjGplqfVclWV76pyADrSaBOt9ZyfUN4ZZ/JlsTzJi9O8PHtkb8kojGIN3zIA + gIfhTT6dxnuxhmLxVOkwRtmiWB+bAEQgmypCxmgiS8diMYGKHKMbrF+jtfQGrukXKOngA4wE2FMP + TFruL+bEgE0fZTLvemtngHSOWKbwg+W7R+RwNGmp9YE28S+qaIwzcVsHlJFcMhSZvBjQQwZwNwyj + f5ZP+ECd55rpBw7fnY9qYtrAS4yJpZ3K5dniFa1aI0tj42yUQkm1RT0xp7nLXhuaAdks8Usvr2mP + qY1r/0d+ApRCz6hrH1Ab9Tzv1BYwnee/2o4THgyLiGFnFNZsMKQtaGxkb8ejEXEfGbu6Ma7F4IMZ + L6fx9aoqZFPhanUuskP2BwAA//8DABw5ElFHAwAA + headers: + CF-RAY: + - 99b0eadaea14aaac-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:22:41 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '328' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '342' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999995' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_68644fc1eb1a4533b2f98192dc918822 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"Say hello","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"top_p":1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '172' + content-type: + - application/json + cookie: + - __cf_bm=.AxQfRhAvElThVl_Qz9zUVdqz_GtBGXwRQ0TVPIg5pc-1762561407-1.0.1.1-klsoMaFKHjzxOrHy2Zfd8Sc76RDHsMXURLAaIzORncnm47NI1MY0BqqBGOEsVXlZb.RdqeqpxzGFhl8DlRDjy.SqRfa2B4zEYdKZqQ2kVB0; + _cfuvid=0ohSoYMS21h1NkHWl4FeeVCp5aK2KHeEjclSm1NY7yY-1762561407934-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jFLBbtswDL37Kzid48EOkrTJZYcd1g1osRVFDy0KQ5FoW5ssChK9LSjy + 74PsNnbXDthFBz6+p/dIPmYAwmixA6FayarzNv94d13am9ubQJf79nB5W9/Z609XV1+V/BK+iUVi + 0P47Kn5mvVfUeYtsyI2wCigZk2p5tlmuN+Wq2A5ARxptojWe8xXlnXEmXxbLVV6c5eX5E7slozCK + HdxnAACPw5t8Oo2/xQ6KxXOlwxhlg2J3agIQgWyqCBmjiSwdi8UEKnKMbrB+gdbSO7igX6Ckg88w + EuBAPTBpefgwJwas+yiTeddbOwOkc8QyhR8sPzwhx5NJS40PtI9/UUVtnIltFVBGcslQZPJiQI8Z + wMMwjP5FPuEDdZ4rph84fLcd1cS0gdcYE0s7lcvzxRtalUaWxsbZKIWSqkU9Mae5y14bmgHZLPFr + L29pj6mNa/5HfgKUQs+oKx9QG/Uy79QWMJ3nv9pOEx4Mi4jhp1FYscGQtqCxlr0dj0bEQ2Tsqtq4 + BoMPZryc2lfrTSHrDa7XW5Edsz8AAAD//wMAVD8AOUcDAAA= + headers: + CF-RAY: + - 99b0ec0acb26ed3b-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:23:30 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '589' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '607' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999995' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_67359745154e404899e3fd81a37cf26a + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/wrappers/cassettes/test_langchain/test_llm_calls b/py/src/braintrust/wrappers/cassettes/test_langchain/test_llm_calls new file mode 100644 index 000000000..cea553488 --- /dev/null +++ b/py/src/braintrust/wrappers/cassettes/test_langchain/test_llm_calls @@ -0,0 +1,333 @@ +interactions: +- request: + body: '{"messages": [{"content": "What is 1 + 2?", "role": "user"}], "model": + "gpt-4o-mini", "frequency_penalty": 0.0, "n": 1, "presence_penalty": 0.0, "stream": + false, "temperature": 1.0, "top_p": 1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - localhost:8000 + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CJ3pRI2shpJIGYKUU8RFWUyB6W5O1\",\n \"object\": + \"chat.completion\",\n \"created\": 1758658985,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"1 + 2 equals 3.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 15,\n \"completion_tokens\": 8,\n \"total_tokens\": 23,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_560af6e559\"\n}\n" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-span-id,x-bt-span-export,x-bt-query-plan,x-bt-internal-trace-id + Connection: + - keep-alive + Date: + - Tue, 23 Sep 2025 20:23:06 GMT + Keep-Alive: + - timeout=5 + Transfer-Encoding: + - chunked + Vary: + - Origin + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 983cc2032f2967ef-SJC + content-type: + - application/json + openai-organization: + - braintrust-data + openai-processing-ms: + - '441' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - _cfuvid=uhF3qDlYXbYwV7mlgYhl_d7MyPH3FwQHxL6cek.ONAQ-1758658986041-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-bt-cached: + - MISS + x-bt-internal-trace-id: + - f4e0a5413e529acf383233e54ad00e99 + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '454' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999995' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_023ebefb1f6b4dec8910b8cb4d7421f5 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"What is 1 + 2?","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"top_p":1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jJI/b9wwDMV3fwqBa8+B7Yudy61dunRJtxaBoZNon1JZVCQ6/RPcdy9k + X85OmgJdPPBHPr1H8zkTAoyGvQB1lKwGb/OPX++KH7eh6XcPvz8Psm++6O1d8+kRnw5+B5s0QYcH + VPwydaVo8BbZkJuxCigZk2p501R1U27r3QQG0mjTWO85v6Z8MM7kVVFd58VNXp7F1ZGMwgh78S0T + Qojn6Zt8Oo0/YS+KzUtlwBhlj7C/NAkBgWyqgIzRRJaOYbNARY7RTdZL8UFUAh9HaaPYXq27AnZj + lMmpG61dAekcsUxJJ3/3Z3K6OLLU+0CH+GYUOuNMPLYBZSSXXo9MHiZ6yoS4n5KPr8KADzR4bpm+ + 4/RcWc9ysOx7gbszY2Jpl3K13bwj1mpkaWxcLQ6UVEfUy+SyZTlqQyuQrSL/7eU97Tm2cf3/yC9A + KfSMuvUBtVGv8y5tAdMx/qvtsuLJMEQMT0ZhywZD+g0aOzna+UQg/oqMQ9sZ12Pwwcx30vm2bgrZ + NVjXt5Cdsj8AAAD//wMAbYrr4zUDAAA= + headers: + CF-RAY: + - 99b0eacc1d35aaac-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:22:39 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=W_Ukgb.mz8e1GW7CfhzN.QQaN09_xQq1uTHm3a.dJdU-1762561359-1.0.1.1-6IrkySxpZaL.1C65iH0iOLFfere0JxHCiasT6bak.RihYFMyJgIz2OuYJqcUey8c5vicjtorNby_Z_GJX.ZMIHa6PyzVrhqgfZZmtnnn.sA; + path=/; expires=Sat, 08-Nov-25 00:52:39 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=jwWMA4k30hLPwBwTSCIdIeS5.m1TkcdYLYTt4YSTZhI-1762561359243-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '300' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '430' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999995' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_24854ba725b942179830d357f1af2add + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"What is 1 + 2?","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"top_p":1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jJJBb9swDIXv/hUEr4sL242TLtdhu+wy7BRsKAxFoh2lsqRKdLGtyH8f + ZKexu3XALj7w46Peo/mcAaBWuAOUR8Gy9yb/8O1reeq+rJ/osdvvf32sDqftA3/+NFC33+MqKdzh + RJJfVDfS9d4Qa2cnLAMJpjS13G6qelOui+0IeqfIJFnnOV+7vNdW51VRrfNim5d3F/XRaUkRd/A9 + AwB4Hr/Jp1X0A3dQrF4qPcUoOsLdtQkAgzOpgiJGHVlYxtUMpbNMdrRewjuogB4HYSLc3iy7ArVD + FMmpHYxZAGGtY5GSjv7uL+R8dWRc54M7xD+k2Gqr47EJJKKz6fXIzuNIzxnA/Zh8eBUGfXC954bd + A43PlfU0Dud9z/DuwtixMHO5ul29MaxRxEKbuFgcSiGPpGblvGUxKO0WIFtE/tvLW7On2Np2/zN+ + BlKSZ1KND6S0fJ13bguUjvFfbdcVj4YxUnjSkhrWFNJvUNSKwUwngvFnZOqbVtuOgg96upPWN/Wm + EO2G6vo9ZufsNwAAAP//AwDHwDA2NQMAAA== + headers: + CF-RAY: + - 99b0ebfc4e5ced3b-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:23:27 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=.AxQfRhAvElThVl_Qz9zUVdqz_GtBGXwRQ0TVPIg5pc-1762561407-1.0.1.1-klsoMaFKHjzxOrHy2Zfd8Sc76RDHsMXURLAaIzORncnm47NI1MY0BqqBGOEsVXlZb.RdqeqpxzGFhl8DlRDjy.SqRfa2B4zEYdKZqQ2kVB0; + path=/; expires=Sat, 08-Nov-25 00:53:27 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=0ohSoYMS21h1NkHWl4FeeVCp5aK2KHeEjclSm1NY7yY-1762561407934-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '269' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '435' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999995' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_617bc8e11f2a43a98a0658e7e91298fd + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/wrappers/cassettes/test_langchain/test_parallel_execution b/py/src/braintrust/wrappers/cassettes/test_langchain/test_parallel_execution new file mode 100644 index 000000000..aec3440ee --- /dev/null +++ b/py/src/braintrust/wrappers/cassettes/test_langchain/test_parallel_execution @@ -0,0 +1,234 @@ +interactions: +- request: + body: '{"messages": [{"content": "Tell me a joke about bear", "role": "user"}], + "model": "gpt-4o-mini", "frequency_penalty": 0.0, "n": 1, "presence_penalty": + 0.0, "stream": false, "temperature": 1.0, "top_p": 1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '188' + content-type: + - application/json + host: + - localhost:8000 + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CJ3vA6tl1z95spYoDxT9RtqqzDF8n\",\n \"object\": + \"chat.completion\",\n \"created\": 1758659340,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"Why don\u2019t bears ever get lost?\\n\\nBecause + they always take the bear necessities! \U0001F43B\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 13,\n \"completion_tokens\": 19,\n \"total_tokens\": 32,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_51db84afab\"\n}\n" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-span-id,x-bt-span-export,x-bt-query-plan,x-bt-internal-trace-id + Connection: + - keep-alive + Date: + - Tue, 23 Sep 2025 20:29:00 GMT + Keep-Alive: + - timeout=5 + Transfer-Encoding: + - chunked + Vary: + - Origin + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 983ccaa98d189e59-SJC + content-type: + - application/json + openai-organization: + - braintrust-data + openai-processing-ms: + - '742' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - _cfuvid=h4eOl14etTzzF9eOjCE9SDq4Y79ZdPOJeIYnqb.tN3E-1758659340929-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-bt-cached: + - MISS + x-bt-internal-trace-id: + - ba7859db365b14edae0dc1d75360d5cb + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '912' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999990' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_31748d3aea8d488c9f1b1b7764b3a5d7 + status: + code: 200 + message: OK +- request: + body: '{"messages": [{"content": "write a 2-line poem about bear", "role": "user"}], + "model": "gpt-4o-mini", "frequency_penalty": 0.0, "n": 1, "presence_penalty": + 0.0, "stream": false, "temperature": 1.0, "top_p": 1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '193' + content-type: + - application/json + host: + - localhost:8000 + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CJ3vAwrz88GjVnlchECG5UbilcrZG\",\n \"object\": + \"chat.completion\",\n \"created\": 1758659340,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"In forest shadows, a bear roams free, + \ \\nMajestic guardian of the ancient tree.\",\n \"refusal\": null,\n + \ \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": + \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 15,\n \"completion_tokens\": + 19,\n \"total_tokens\": 34,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_560af6e559\"\n}\n" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-span-id,x-bt-span-export,x-bt-query-plan,x-bt-internal-trace-id + Connection: + - keep-alive + Date: + - Tue, 23 Sep 2025 20:29:01 GMT + Keep-Alive: + - timeout=5 + Transfer-Encoding: + - chunked + Vary: + - Origin + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 983ccaa99f09cecd-SJC + content-type: + - application/json + openai-organization: + - braintrust-data + openai-processing-ms: + - '909' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - _cfuvid=I8TMI8qNGmqspYd_94RtBiCEVRDIffMScd.j_yw35Es-1758659341697-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-bt-cached: + - MISS + x-bt-internal-trace-id: + - 7d350d2a8b4d267107b257e3a1989c5a + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '1375' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999990' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_891af1935bbf49c39105d7299babb315 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/wrappers/cassettes/test_langchain/test_prompt_caching_tokens b/py/src/braintrust/wrappers/cassettes/test_langchain/test_prompt_caching_tokens new file mode 100644 index 000000000..441128e9d --- /dev/null +++ b/py/src/braintrust/wrappers/cassettes/test_langchain/test_prompt_caching_tokens @@ -0,0 +1,324 @@ +interactions: +- request: + body: '{"max_tokens":1024,"messages":[{"role":"user","content":"What is the first + type of testing mentioned in section 1.2?"}],"model":"claude-sonnet-4-5-20250929","system":[{"type":"text","text":"\n# + Comprehensive Guide to Software Testing Methods!\n\n## Chapter 1: Introduction + to Testing\n\nSoftware testing is a critical component of the software development + lifecycle. It ensures that applications\nfunction correctly, meet requirements, + and provide a positive user experience. This guide covers various\ntesting methodologies, + best practices, and tools used in modern software development.\n\n### 1.1 The + Importance of Testing\n\nTesting helps identify defects early in the development + process, reducing the cost of fixing issues later.\nStudies have shown that + the cost of fixing a bug increases exponentially as it progresses through the\ndevelopment + lifecycle. A bug found during requirements gathering might cost $1 to fix, while + the same bug\nfound in production could cost $100 or more.\n\n### 1.2 Types + of Testing\n\nThere are many types of testing, including:\n- Unit Testing: Testing + individual components or functions in isolation\n- Integration Testing: Testing + how components work together\n- End-to-End Testing: Testing the entire application + flow\n- Performance Testing: Testing application speed and scalability\n- Security + Testing: Testing for vulnerabilities and security issues\n- Usability Testing: + Testing user experience and interface design\n\n## Chapter 2: Unit Testing Best + Practices\n\nUnit testing focuses on testing the smallest testable parts of + an application. Here are some best practices:\n\n### 2.1 Write Tests First (TDD)\n\nTest-Driven + Development (TDD) is a methodology where tests are written before the actual + code. The process\nfollows a simple cycle: Red (write a failing test), Green + (write code to pass the test), Refactor (improve\nthe code while keeping tests + passing).\n\n### 2.2 Keep Tests Independent\n\nEach test should be independent + of others. Tests should not rely on the state created by previous tests.\nThis + ensures that tests can be run in any order and that failures are isolated and + easy to debug.\n\n### 2.3 Use Meaningful Names\n\nTest names should clearly + describe what is being tested and what the expected outcome is. A good test + name\nmight be \"test_user_registration_with_valid_email_succeeds\" rather than + just \"test_registration\".\n\n### 2.4 Test Edge Cases\n\nDon''t just test the + happy path. Consider edge cases like:\n- Empty inputs\n- Null or undefined values\n- + Very large inputs\n- Invalid formats\n- Boundary conditions\n\n## Chapter 3: + Integration Testing\n\nIntegration testing verifies that different modules or + services work together correctly.\n\n### 3.1 Database Integration\n\nWhen testing + database interactions, consider using:\n- Test databases separate from production\n- + Database transactions that roll back after each test\n- Mock data that represents + realistic scenarios\n\n### 3.2 API Integration\n\nAPI integration tests should + verify:\n- Correct HTTP status codes\n- Response format and schema\n- Error + handling\n- Authentication and authorization\n\n## Chapter 4: Performance Testing\n\nPerformance + testing ensures your application can handle expected load and scale appropriately.\n\n### + 4.1 Load Testing\n\nLoad testing simulates multiple users accessing the application + simultaneously. Key metrics include:\n- Response time under load\n- Throughput + (requests per second)\n- Error rates\n- Resource utilization (CPU, memory, network)\n\n### + 4.2 Stress Testing\n\nStress testing pushes the application beyond normal operational + capacity to find breaking points and\nunderstand how the system fails gracefully.\n\n## + Chapter 5: Continuous Integration and Testing\n\nModern development practices + integrate testing into the CI/CD pipeline.\n\n### 5.1 Automated Test Runs\n\nTests + should run automatically on every code change. This includes:\n- Running unit + tests on every commit\n- Running integration tests on pull requests\n- Running + end-to-end tests before deployment\n\n### 5.2 Test Coverage\n\nTest coverage + metrics help identify untested code. While 100% coverage isn''t always practical + or necessary,\nmaintaining good coverage helps ensure code quality. Focus on + critical paths and business logic.\n\n## Chapter 6: Testing Tools and Frameworks\n\nMany + tools exist to support testing efforts:\n\n### 6.1 Python Testing\n- pytest: + Feature-rich testing framework\n- unittest: Built-in Python testing module\n- + mock: Library for mocking objects\n\n### 6.2 JavaScript Testing\n- Jest: Popular + testing framework\n- Mocha: Flexible testing framework\n- Cypress: End-to-end + testing tool\n\n### 6.3 Other Tools\n- Selenium: Browser automation\n- JMeter: + Performance testing\n- Postman: API testing\n\n## Conclusion\n\nEffective testing + is essential for delivering high-quality software. By following best practices + and using\nappropriate tools, teams can catch bugs early, improve code quality, + and deliver better products to users.\n\nRemember: Testing is not just about + finding bugs, it''s about building confidence in your code.\n","cache_control":{"type":"ephemeral"}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '5160' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.76.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.76.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.19 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAA/22RzU7rMBCFX8WaZZWiJGqvaHYXsQIhsaBsKIqMPW0sEjt4xgVU9d0ZFyp+V4nn + fHOOZ7yDIVjsoQHT62RxSsF75OlsOp/WZT0vF/UCCnBWiIE2bVndrsO/s+1ptYzXl+fz04tN0nZ5 + JQy/jpgpJNIblEIMfS5oIkesPUvJBM8of83d7sgzvmTl8GngvzEhWuc3ioMiNOyCV9VJrVZwIzyp + sFY3SCxEsQLFHaq1i8Qqu2WR30U1SIz0olWO1GSy9I6PjZNJoZ47Z7osWSQT3YNwmnLIR7vz1m2d + TbpXJgyjGHmW8KjWyR8uRYJIf+h1Pp2sAPb3BRCHsY2oZYsyDHrbcooePgTCp4TeyNQ+9X0B6bCo + ZgfOj4lbDo/oCZq6lEVp02FrxCrbt9+BqpwtjogQ9of8qz1H4NjhgFH37Xz40+4TqLqfhvsCQuKv + pZmEEMatM9iywyjT5je2OlrY798Az+eCZFYCAAA= + headers: + CF-RAY: + - 9c1a60c71c9c67cb-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 21 Jan 2026 22:51:47 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '3000000' + anthropic-ratelimit-input-tokens-remaining: + - '3000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-01-21T22:51:46Z' + anthropic-ratelimit-output-tokens-limit: + - '600000' + anthropic-ratelimit-output-tokens-remaining: + - '600000' + anthropic-ratelimit-output-tokens-reset: + - '2026-01-21T22:51:47Z' + anthropic-ratelimit-tokens-limit: + - '3600000' + anthropic-ratelimit-tokens-remaining: + - '3600000' + anthropic-ratelimit-tokens-reset: + - '2026-01-21T22:51:46Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CXMLqXaFZ4xWZExkXJyyb + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '2088' + status: + code: 200 + message: OK +- request: + body: '{"max_tokens":1024,"messages":[{"role":"user","content":"What is the first + type of testing mentioned in section 1.2?"},{"role":"assistant","content":"According + to section 1.2 \"Types of Testing,\" the first type of testing mentioned is + **Unit Testing**, which is described as \"Testing individual components or functions + in isolation.\""},{"role":"user","content":"What testing framework is mentioned + for Python?"}],"model":"claude-sonnet-4-5-20250929","system":[{"type":"text","text":"\n# + Comprehensive Guide to Software Testing Methods!\n\n## Chapter 1: Introduction + to Testing\n\nSoftware testing is a critical component of the software development + lifecycle. It ensures that applications\nfunction correctly, meet requirements, + and provide a positive user experience. This guide covers various\ntesting methodologies, + best practices, and tools used in modern software development.\n\n### 1.1 The + Importance of Testing\n\nTesting helps identify defects early in the development + process, reducing the cost of fixing issues later.\nStudies have shown that + the cost of fixing a bug increases exponentially as it progresses through the\ndevelopment + lifecycle. A bug found during requirements gathering might cost $1 to fix, while + the same bug\nfound in production could cost $100 or more.\n\n### 1.2 Types + of Testing\n\nThere are many types of testing, including:\n- Unit Testing: Testing + individual components or functions in isolation\n- Integration Testing: Testing + how components work together\n- End-to-End Testing: Testing the entire application + flow\n- Performance Testing: Testing application speed and scalability\n- Security + Testing: Testing for vulnerabilities and security issues\n- Usability Testing: + Testing user experience and interface design\n\n## Chapter 2: Unit Testing Best + Practices\n\nUnit testing focuses on testing the smallest testable parts of + an application. Here are some best practices:\n\n### 2.1 Write Tests First (TDD)\n\nTest-Driven + Development (TDD) is a methodology where tests are written before the actual + code. The process\nfollows a simple cycle: Red (write a failing test), Green + (write code to pass the test), Refactor (improve\nthe code while keeping tests + passing).\n\n### 2.2 Keep Tests Independent\n\nEach test should be independent + of others. Tests should not rely on the state created by previous tests.\nThis + ensures that tests can be run in any order and that failures are isolated and + easy to debug.\n\n### 2.3 Use Meaningful Names\n\nTest names should clearly + describe what is being tested and what the expected outcome is. A good test + name\nmight be \"test_user_registration_with_valid_email_succeeds\" rather than + just \"test_registration\".\n\n### 2.4 Test Edge Cases\n\nDon''t just test the + happy path. Consider edge cases like:\n- Empty inputs\n- Null or undefined values\n- + Very large inputs\n- Invalid formats\n- Boundary conditions\n\n## Chapter 3: + Integration Testing\n\nIntegration testing verifies that different modules or + services work together correctly.\n\n### 3.1 Database Integration\n\nWhen testing + database interactions, consider using:\n- Test databases separate from production\n- + Database transactions that roll back after each test\n- Mock data that represents + realistic scenarios\n\n### 3.2 API Integration\n\nAPI integration tests should + verify:\n- Correct HTTP status codes\n- Response format and schema\n- Error + handling\n- Authentication and authorization\n\n## Chapter 4: Performance Testing\n\nPerformance + testing ensures your application can handle expected load and scale appropriately.\n\n### + 4.1 Load Testing\n\nLoad testing simulates multiple users accessing the application + simultaneously. Key metrics include:\n- Response time under load\n- Throughput + (requests per second)\n- Error rates\n- Resource utilization (CPU, memory, network)\n\n### + 4.2 Stress Testing\n\nStress testing pushes the application beyond normal operational + capacity to find breaking points and\nunderstand how the system fails gracefully.\n\n## + Chapter 5: Continuous Integration and Testing\n\nModern development practices + integrate testing into the CI/CD pipeline.\n\n### 5.1 Automated Test Runs\n\nTests + should run automatically on every code change. This includes:\n- Running unit + tests on every commit\n- Running integration tests on pull requests\n- Running + end-to-end tests before deployment\n\n### 5.2 Test Coverage\n\nTest coverage + metrics help identify untested code. While 100% coverage isn''t always practical + or necessary,\nmaintaining good coverage helps ensure code quality. Focus on + critical paths and business logic.\n\n## Chapter 6: Testing Tools and Frameworks\n\nMany + tools exist to support testing efforts:\n\n### 6.1 Python Testing\n- pytest: + Feature-rich testing framework\n- unittest: Built-in Python testing module\n- + mock: Library for mocking objects\n\n### 6.2 JavaScript Testing\n- Jest: Popular + testing framework\n- Mocha: Flexible testing framework\n- Cypress: End-to-end + testing tool\n\n### 6.3 Other Tools\n- Selenium: Browser automation\n- JMeter: + Performance testing\n- Postman: API testing\n\n## Conclusion\n\nEffective testing + is essential for delivering high-quality software. By following best practices + and using\nappropriate tools, teams can catch bugs early, improve code quality, + and deliver better products to users.\n\nRemember: Testing is not just about + finding bugs, it''s about building confidence in your code.\n","cache_control":{"type":"ephemeral"}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '5456' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.76.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.76.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.19 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAA/2VSzU7DMAx+lSjHqUXtYEzrDYQQB5A4IA2JoiokZg1rnZI4wDTt3XHK/zgl+X4c + +0u2sncGOllJ3aloIA8OESg/ymf5tJjOisV0ITNpDSv6sGqKsqSr5fX87OVC6RbncblY3y7PkDW0 + GSCpIAS1Aga86xKgQrCBFBJD2iEB76q77Zee4C0x41LJE62dNxZXgpwIoMk6FMcHpajl9YZaPtxA + IOazWgpqPYCgD0A8etXDq/PrIJQH0fM9bAYjHp0XH+aqxhrLAzGZDJtkm0xELs5BUfSQe6vb/8Vq + nCZ9REvfjtNoO8otflb9NnGUsYMaD5Ojd3o9qi/tg1d+M7aRwKR0D088W5C7+0wGckPjQXHyHACg + abidFOhIBHiOgJqTwth1mYxjuNVWWhwiNeTWgEFW85LD5ReBRnOpNHjzV1B88UybPa4sjhb79nQF + DC304FXXzPr/5X7Yst1nd5l0kX5Dx+wI4F+shoYseB41fQqjvJG73TumCh7LhwIAAA== + headers: + CF-RAY: + - 9c1a60d4ab5e67cb-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 21 Jan 2026 22:51:49 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '3000000' + anthropic-ratelimit-input-tokens-remaining: + - '3000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-01-21T22:51:48Z' + anthropic-ratelimit-output-tokens-limit: + - '600000' + anthropic-ratelimit-output-tokens-remaining: + - '600000' + anthropic-ratelimit-output-tokens-reset: + - '2026-01-21T22:51:49Z' + anthropic-ratelimit-tokens-limit: + - '3600000' + anthropic-ratelimit-tokens-remaining: + - '3600000' + anthropic-ratelimit-tokens-reset: + - '2026-01-21T22:51:48Z' + cf-cache-status: + - DYNAMIC + request-id: + - req_011CXMLqgrrchykwCdY7YRKM + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-envoy-upstream-service-time: + - '2016' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/wrappers/cassettes/test_langchain/test_streaming_ttft b/py/src/braintrust/wrappers/cassettes/test_langchain/test_streaming_ttft new file mode 100644 index 000000000..1ee7a8377 --- /dev/null +++ b/py/src/braintrust/wrappers/cassettes/test_langchain/test_streaming_ttft @@ -0,0 +1,298 @@ +interactions: +- request: + body: '{"messages":[{"content":"Count from 1 to 5.","role":"user"}],"model":"gpt-4o-mini","max_completion_tokens":50,"stream":true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '124' + content-type: + - application/json + cookie: + - __cf_bm=W_Ukgb.mz8e1GW7CfhzN.QQaN09_xQq1uTHm3a.dJdU-1762561359-1.0.1.1-6IrkySxpZaL.1C65iH0iOLFfere0JxHCiasT6bak.RihYFMyJgIz2OuYJqcUey8c5vicjtorNby_Z_GJX.ZMIHa6PyzVrhqgfZZmtnnn.sA; + _cfuvid=jwWMA4k30hLPwBwTSCIdIeS5.m1TkcdYLYTt4YSTZhI-1762561359243-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"obfuscation":"uoycSw"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":"1"},"logprobs":null,"finish_reason":null}],"obfuscation":"7R9sCOG"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}],"obfuscation":"jNZOnCU"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"obfuscation":"NTkR0fq"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":"2"},"logprobs":null,"finish_reason":null}],"obfuscation":"KhfgFBA"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}],"obfuscation":"u5zk4uv"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"obfuscation":"yQyBcA4"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":"3"},"logprobs":null,"finish_reason":null}],"obfuscation":"HhGcZch"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}],"obfuscation":"GNLE7Ci"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"obfuscation":"d0EKjlZ"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":"4"},"logprobs":null,"finish_reason":null}],"obfuscation":"YytmIuX"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}],"obfuscation":"Umbehc1"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"obfuscation":"3xi8C7o"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":"5"},"logprobs":null,"finish_reason":null}],"obfuscation":"N0uOsTp"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"obfuscation":"RilMN7a"} + + + data: {"id":"chatcmpl-CZR0zJXGi0lsnYkPoiga2R6HChxps","object":"chat.completion.chunk","created":1762561361,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"obfuscation":"oF"} + + + data: [DONE] + + + ' + headers: + CF-RAY: + - 99b0eaddeca8aaac-SJC + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Sat, 08 Nov 2025 00:22:42 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '275' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '519' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_05aebff8dd644228befd59a7372d3c93 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"Count from 1 to 5.","role":"user"}],"model":"gpt-4o-mini","max_completion_tokens":50,"stream":true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '124' + content-type: + - application/json + cookie: + - __cf_bm=.AxQfRhAvElThVl_Qz9zUVdqz_GtBGXwRQ0TVPIg5pc-1762561407-1.0.1.1-klsoMaFKHjzxOrHy2Zfd8Sc76RDHsMXURLAaIzORncnm47NI1MY0BqqBGOEsVXlZb.RdqeqpxzGFhl8DlRDjy.SqRfa2B4zEYdKZqQ2kVB0; + _cfuvid=0ohSoYMS21h1NkHWl4FeeVCp5aK2KHeEjclSm1NY7yY-1762561407934-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"obfuscation":"ov7JiI"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":"1"},"logprobs":null,"finish_reason":null}],"obfuscation":"eXpmCqg"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}],"obfuscation":"C8QZXu8"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"obfuscation":"xdqGFpo"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":"2"},"logprobs":null,"finish_reason":null}],"obfuscation":"O3SLgWG"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}],"obfuscation":"0aoEi42"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"obfuscation":"2oO8rJa"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":"3"},"logprobs":null,"finish_reason":null}],"obfuscation":"jOHTEGa"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}],"obfuscation":"qGeoxr1"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"obfuscation":"uvMar7j"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":"4"},"logprobs":null,"finish_reason":null}],"obfuscation":"4dFvFfq"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":","},"logprobs":null,"finish_reason":null}],"obfuscation":"GdoZztm"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":" + "},"logprobs":null,"finish_reason":null}],"obfuscation":"NHxpCPR"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":"5"},"logprobs":null,"finish_reason":null}],"obfuscation":"mfV8KdT"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"obfuscation":"EkPlssM"} + + + data: {"id":"chatcmpl-CZR1mouRDQnH9qWlT2zp6Fs0nW1Uq","object":"chat.completion.chunk","created":1762561410,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_560af6e559","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"obfuscation":"fj"} + + + data: [DONE] + + + ' + headers: + CF-RAY: + - 99b0ec0f7961ed3b-SJC + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Sat, 08 Nov 2025 00:23:30 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '149' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '171' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_8afec9e4717b433e9c6900220b2dbd93 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/wrappers/cassettes/test_langchain/test_tool_usage b/py/src/braintrust/wrappers/cassettes/test_langchain/test_tool_usage new file mode 100644 index 000000000..e21d44ccb --- /dev/null +++ b/py/src/braintrust/wrappers/cassettes/test_langchain/test_tool_usage @@ -0,0 +1,350 @@ +interactions: +- request: + body: '{"messages": [{"content": "What is 3 * 12", "role": "user"}], "model": + "gpt-4o-mini", "frequency_penalty": 0.0, "n": 1, "presence_penalty": 0.0, "stream": + false, "temperature": 1.0, "tools": [{"type": "function", "function": {"name": + "calculator", "description": "Can perform mathematical operations.", "parameters": + {"properties": {"input": {"properties": {"operation": {"description": "The type + of operation to execute.", "enum": ["add", "subtract", "multiply", "divide"], + "type": "string"}, "number1": {"description": "The first number to operate on.", + "type": "number"}, "number2": {"description": "The second number to operate + on.", "type": "number"}}, "required": ["operation", "number1", "number2"], "type": + "object"}}, "required": ["input"], "type": "object"}}}], "top_p": 1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '725' + content-type: + - application/json + host: + - localhost:8000 + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CJ3pT0xTT4C4WwCqA5bvyrihLFrbd\",\n \"object\": + \"chat.completion\",\n \"created\": 1758658987,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n + \ \"id\": \"call_faZyqlGfMGsX50e2EuExUqK0\",\n \"type\": + \"function\",\n \"function\": {\n \"name\": \"calculator\",\n + \ \"arguments\": \"{\\\"input\\\":{\\\"operation\\\":\\\"multiply\\\",\\\"number1\\\":3,\\\"number2\\\":12}}\"\n + \ }\n }\n ],\n \"refusal\": null,\n \"annotations\": + []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n + \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 97,\n \"completion_tokens\": + 26,\n \"total_tokens\": 123,\n \"prompt_tokens_details\": {\n \"cached_tokens\": + 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_51db84afab\"\n}\n" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-span-id,x-bt-span-export,x-bt-query-plan,x-bt-internal-trace-id + Connection: + - keep-alive + Date: + - Tue, 23 Sep 2025 20:23:07 GMT + Keep-Alive: + - timeout=5 + Transfer-Encoding: + - chunked + Vary: + - Origin + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 983cc20cabc267ef-SJC + content-type: + - application/json + openai-organization: + - braintrust-data + openai-processing-ms: + - '648' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - _cfuvid=inx7Y1lMFCkI1jONo8plrYH7k2d1EAvkr2WlMIyrK.s-1758658987739-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-bt-cached: + - MISS + x-bt-internal-trace-id: + - 475d214543543ac965368ac2a190850f + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '663' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_f6bcef66199c4bcaa6ad5864f7d1d9fb + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"What is 3 * 12","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"tools":[{"type":"function","function":{"name":"calculator","description":"Can + perform mathematical operations.","parameters":{"properties":{"input":{"properties":{"operation":{"description":"The + type of operation to execute.","enum":["add","subtract","multiply","divide"],"type":"string"},"number1":{"description":"The + first number to operate on.","type":"number"},"number2":{"description":"The + second number to operate on.","type":"number"}},"required":["operation","number1","number2"],"type":"object"}},"required":["input"],"type":"object"}}}],"top_p":1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '725' + content-type: + - application/json + cookie: + - __cf_bm=W_Ukgb.mz8e1GW7CfhzN.QQaN09_xQq1uTHm3a.dJdU-1762561359-1.0.1.1-6IrkySxpZaL.1C65iH0iOLFfere0JxHCiasT6bak.RihYFMyJgIz2OuYJqcUey8c5vicjtorNby_Z_GJX.ZMIHa6PyzVrhqgfZZmtnnn.sA; + _cfuvid=jwWMA4k30hLPwBwTSCIdIeS5.m1TkcdYLYTt4YSTZhI-1762561359243-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA4xT0W6bMBR95yus+xymQAppedum7SFKNXXSqmqjQo65EG/G9myzLY3y7xMmBZKm + UnlAcI/PucfH1/uAEOAlZATYljrWaBF+/P51vnsyt/dSfzCrMmFyeXu3fr96untwX2DWMdTmJzL3 + zHrHVKMFOq5kDzOD1GGnGi3TOEmjRTr3QKNKFB2t1i68UmHDJQ/jeXwVzpdhdH1kbxVnaCEjPwJC + CNn7d+dTlvgPMuK1fKVBa2mNkA2LCAGjRFcBai23jkoHsxFkSjqUnXXZCjEBnFKiYFSIsXH/7Cff + Y1hUiALZUupvWK/u/z4k5e9PuNafV+vrab9eeqe9oaqVbAhpgg/17KwZISBpg8eGrBXUKXPGJgSo + qdsGpeucwz4HLnXrcsj2OSiNhnbaOWQ5NK1wXItdDrMcZNts0EQ5ZIvhL84hi+LDAU5aHIJL34+T + 8AxWraXiZapUSuW8AR/r4xE5DCcoVK2N2tgzKlRccrstDFLrg5meT/BsxFuA9mQEQBvVaFc49Qt9 + 05tlLwrjlI5gnB5BpxwVYz2KF7MLckWJjnI/IsNUMsq2WI7UcTppW3I1AYLJ1l+6uaTdb5/L+i3y + I8AYaodloQ2WnJ3ueFxmsLvEry0bQvaGwaL5wxkWjqPpjqPEiraiH3WwO+uwKSouazTacH+/oNJF + ks5plWKS3EBwCP4DAAD//wMAguKIhm0EAAA= + headers: + CF-RAY: + - 99b0ead42b8caaac-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:22:40 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '557' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '702' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999995' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_edb893697ec245fbb710a31d27a3ed78 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"What is 3 * 12","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"tools":[{"type":"function","function":{"name":"calculator","description":"Can + perform mathematical operations.","parameters":{"properties":{"input":{"properties":{"operation":{"description":"The + type of operation to execute.","enum":["add","subtract","multiply","divide"],"type":"string"},"number1":{"description":"The + first number to operate on.","type":"number"},"number2":{"description":"The + second number to operate on.","type":"number"}},"required":["operation","number1","number2"],"type":"object"}},"required":["input"],"type":"object"}}}],"top_p":1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '725' + content-type: + - application/json + cookie: + - __cf_bm=.AxQfRhAvElThVl_Qz9zUVdqz_GtBGXwRQ0TVPIg5pc-1762561407-1.0.1.1-klsoMaFKHjzxOrHy2Zfd8Sc76RDHsMXURLAaIzORncnm47NI1MY0BqqBGOEsVXlZb.RdqeqpxzGFhl8DlRDjy.SqRfa2B4zEYdKZqQ2kVB0; + _cfuvid=0ohSoYMS21h1NkHWl4FeeVCp5aK2KHeEjclSm1NY7yY-1762561407934-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA4xTTY/aMBC951dYc4YqCV+7udEPqSdaVeqh26wiY0/AXce2/MHCIv57lQSSwFKp + OUT2PM+b5+eZY0QICA4ZAbalnlVGjj89/UjkfIlfVm+/4o9P4dW/2a/S0dfl5PAAozpDr/8g85es + D0xXRqIXWrUws0g91qzJYp7O5sk0fmyASnOUddrG+PFUjyuhxDiN0+k4XoyTMznbasHQQUZ+R4QQ + cmz+tU7FcQ8ZiUeXSIXO0Q1C1h0iBKyWdQSoc8J5qjyMepBp5VHV0lWQcgB4rWXBqJR94fY7Dta9 + WVTK4udkP9upwHY7/nm1XH2Pefi2f4n5oF5LfTCNoDIo1pk0wLt4dlOMEFC0wnNBFiT12t5kEwLU + bkKFytfK4ZiDUCb4HLJjDtqgpTV3DlkOVZBeGHnIYZSDCtUabZJDNul2aQ5Zkp5OcFXiFN1bPw/M + s1gGR+V7V6lS2jcCGlufz8ipe0GpN8bqtbtJhVIo4baFReoaY4bvE12ENBIgXLUAGKsr4wuvX7Ap + +rhoSaHv0h5M52fQa09lH0/SyegOXcHRU9G0SNeVjLIt8j61704auNADIBpc/b2ae9zt9YXa/A99 + DzCGxiMvjEUu2PWN+2MW6yH+17HO5EYwOLQ7wbDwAm39HBxLGmQ7WuAOzmNVlEJt0BormvmC0hSz + hK8fprSka4hO0V8AAAD//wMAMU2sv20EAAA= + headers: + CF-RAY: + - 99b0ec04f9abed3b-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:23:29 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '614' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '756' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999995' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_b741763f424444f38ded6343a488e723 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/wrappers/langchain.py b/py/src/braintrust/wrappers/langchain.py deleted file mode 100644 index c723d0628..000000000 --- a/py/src/braintrust/wrappers/langchain.py +++ /dev/null @@ -1,149 +0,0 @@ -import contextvars -import logging -from typing import Any -from uuid import UUID - -import braintrust - -_logger = logging.getLogger("braintrust.wrappers.langchain") - -try: - from langchain.callbacks.base import BaseCallbackHandler - from langchain.schema import Document - from langchain.schema.agent import AgentAction - from langchain.schema.messages import BaseMessage - from langchain.schema.output import LLMResult -except ImportError: - _logger.warning("Failed to import langchain, using stubs") - BaseCallbackHandler = object - Document = object - AgentAction = object - BaseMessage = object - LLMResult = object - -langchain_parent = contextvars.ContextVar("langchain_current_span", default=None) - - -class BraintrustTracer(BaseCallbackHandler): - def __init__(self, logger=None): - _logger.warning("BraintrustTracer is deprecated, use `pip install braintrust-langchain` instead") - self.logger = logger - self.spans = {} - - def _start_span(self, parent_run_id, run_id, name: str | None, **kwargs: Any) -> Any: - assert run_id not in self.spans, f"Span already exists for run_id {run_id} (this is likely a bug)" - - current_parent = langchain_parent.get() - if parent_run_id in self.spans: - parent_span = self.spans[parent_run_id] - elif current_parent is not None: - parent_span = current_parent - elif self.logger is not None: - parent_span = self.logger - else: - parent_span = braintrust - - span = parent_span.start_span(name=name, **kwargs) - langchain_parent.set(span) - self.spans[run_id] = span - return span - - def _end_span(self, run_id, **kwargs: Any) -> Any: - assert run_id in self.spans, f"No span exists for run_id {run_id} (this is likely a bug)" - span = self.spans.pop(run_id) - span.log(**kwargs) - - if langchain_parent.get() == span: - langchain_parent.set(None) - - span.end() - - def on_chain_start( - self, - serialized: dict[str, Any], - inputs: dict[str, Any], - *, - run_id: UUID, - parent_run_id: UUID | None = None, - tags: list[str] | None = None, - **kwargs: Any, - ) -> Any: - self._start_span(parent_run_id, run_id, "Chain", input=inputs, metadata={"tags": tags}) - - def on_chain_end( - self, outputs: dict[str, Any], *, run_id: UUID, parent_run_id: UUID | None = None, **kwargs: Any - ) -> Any: - self._end_span(run_id, output=outputs) - - def on_llm_start( - self, - serialized: dict[str, Any], - prompts: list[str], - *, - run_id: UUID, - parent_run_id: UUID | None = None, - tags: list[str] | None = None, - **kwargs: Any, - ) -> Any: - self._start_span( - parent_run_id, - run_id, - "LLM", - input=prompts, - metadata={"tags": tags, **kwargs["invocation_params"]}, - ) - - def on_chat_model_start( - self, - serialized: dict[str, Any], - messages: list[list[BaseMessage]], - *, - run_id: UUID, - parent_run_id: UUID | None = None, - tags: list[str] | None = None, - **kwargs: Any, - ) -> Any: - self._start_span( - parent_run_id, - run_id, - "Chat Model", - input=[[m.dict() for m in batch] for batch in messages], - metadata={"tags": tags, **kwargs["invocation_params"]}, - ) - - def on_llm_end( - self, response: LLMResult, *, run_id: UUID, parent_run_id: UUID | None = None, **kwargs: Any - ) -> Any: - metrics = {} - token_usage = response.llm_output.get("token_usage", {}) - if "total_tokens" in token_usage: - metrics["tokens"] = token_usage["total_tokens"] - if "prompt_tokens" in token_usage: - metrics["prompt_tokens"] = token_usage["prompt_tokens"] - if "completion_tokens" in token_usage: - metrics["completion_tokens"] = token_usage["completion_tokens"] - - self._end_span(run_id, output=[[m.dict() for m in batch] for batch in response.generations], metrics=metrics) - - def on_tool_start( - self, - serialized: dict[str, Any], - input_str: str, - *, - run_id: UUID, - parent_run_id: UUID | None = None, - tags: list[str] | None = None, - **kwargs: Any, - ) -> Any: - _logger.warning("Starting tool, but it will not be traced in braintrust (unsupported)") - - def on_tool_end(self, output: str, *, run_id: UUID, parent_run_id: UUID | None = None, **kwargs: Any) -> Any: - pass - - def on_retriever_start(self, query: str, *, run_id: UUID, parent_run_id: UUID | None = None, **kwargs: Any) -> Any: - _logger.warning("Starting retriever, but it will not be traced in braintrust (unsupported)") - - def on_retriever_end( - self, response: list[Document], *, run_id: UUID, parent_run_id: UUID | None = None, **kwargs: Any - ) -> Any: - pass diff --git a/py/src/braintrust/wrappers/langchain/__init__.py b/py/src/braintrust/wrappers/langchain/__init__.py new file mode 100644 index 000000000..c61ceb88b --- /dev/null +++ b/py/src/braintrust/wrappers/langchain/__init__.py @@ -0,0 +1,104 @@ +""" +Braintrust integration for LangChain. + +Provides automatic tracing of LangChain chains, agents, and LLM calls. + +Example usage with auto_instrument(): + ```python + import braintrust + braintrust.init_logger(project="my-project") + braintrust.auto_instrument() + + # All LangChain operations are now automatically traced + from langchain_openai import ChatOpenAI + model = ChatOpenAI(model="gpt-4o-mini") + model.invoke("Hello!") # Automatically traced + ``` + +Example usage with setup_langchain(): + ```python + from braintrust.wrappers.langchain import setup_langchain, BraintrustCallbackHandler + setup_langchain(project_name="my-project") + + # All LangChain operations are now automatically traced + from langchain_openai import ChatOpenAI + model = ChatOpenAI(model="gpt-4o-mini") + model.invoke("Hello!") # Automatically traced + ``` + +Example usage with manual handler: + ```python + from braintrust.wrappers.langchain import BraintrustCallbackHandler + from langchain_openai import ChatOpenAI + + handler = BraintrustCallbackHandler() + model = ChatOpenAI(model="gpt-4o-mini") + model.invoke("Hello!", config={"callbacks": [handler]}) + ``` +""" + +import logging + +from braintrust.logger import NOOP_SPAN, current_span, init_logger + +from .callbacks import BraintrustCallbackHandler +from .context import clear_global_handler, set_global_handler + +__all__ = [ + "setup_langchain", + "BraintrustCallbackHandler", + "set_global_handler", + "clear_global_handler", +] + +_logger = logging.getLogger(__name__) + + +def setup_langchain( + api_key: str | None = None, + project_id: str | None = None, + project_name: str | None = None, +) -> bool: + """ + Setup Braintrust integration with LangChain. + + Automatically registers a BraintrustCallbackHandler as the global handler, + enabling automatic tracing of all LangChain operations. + + Args: + api_key: Braintrust API key (optional, uses env var if not provided) + project_id: Braintrust project ID + project_name: Braintrust project name + + Returns: + True if setup was successful, False if langchain is not installed. + + Example: + ```python + import braintrust + from braintrust.wrappers.langchain import setup_langchain + + setup_langchain(project_name="my-langchain-app") + + # All LangChain operations are now automatically traced + from langchain_openai import ChatOpenAI + model = ChatOpenAI(model="gpt-4o-mini") + model.invoke("Hello!") # Automatically traced + ``` + """ + span = current_span() + if span == NOOP_SPAN: + init_logger(project=project_name, api_key=api_key, project_id=project_id) + + try: + # Verify langchain is installed by importing core module + from langchain_core.callbacks.base import BaseCallbackHandler as _ # noqa: F401 + + # Create and register global handler + handler = BraintrustCallbackHandler() + set_global_handler(handler) + + return True + except ImportError: + # langchain not installed + return False diff --git a/py/src/braintrust/wrappers/langchain/callbacks.py b/py/src/braintrust/wrappers/langchain/callbacks.py new file mode 100644 index 000000000..64687ba77 --- /dev/null +++ b/py/src/braintrust/wrappers/langchain/callbacks.py @@ -0,0 +1,649 @@ +import json +import logging +import re +import time +from collections.abc import Mapping, Sequence +from re import Pattern +from typing import ( + Any, + TypedDict, + Union, +) +from uuid import UUID + +import braintrust +from braintrust import NOOP_SPAN, Logger, Span, SpanAttributes, SpanTypeAttribute, current_span, init_logger +from braintrust.version import VERSION as sdk_version +from langchain_core.agents import AgentAction, AgentFinish +from langchain_core.callbacks.base import BaseCallbackHandler +from langchain_core.documents import Document +from langchain_core.messages import BaseMessage +from langchain_core.outputs.llm_result import LLMResult +from tenacity import RetryCallState +from typing_extensions import NotRequired + +_logger = logging.getLogger("braintrust.wrappers.langchain") + +# Integration version - use SDK version since this is now bundled +_integration_version = sdk_version + + +class LogEvent(TypedDict): + input: NotRequired[Any] + output: NotRequired[Any] + expected: NotRequired[Any] + error: NotRequired[str] + tags: NotRequired[Sequence[str] | None] + scores: NotRequired[Mapping[str, int | float]] + metadata: NotRequired[Mapping[str, Any]] + metrics: NotRequired[Mapping[str, int | float]] + id: NotRequired[str] + dataset_record_id: NotRequired[str] + + +class BraintrustCallbackHandler(BaseCallbackHandler): + root_run_id: UUID | None = None + + def __init__( + self, + logger: Logger | Span | None = None, + debug: bool = False, + exclude_metadata_props: Pattern[str] | None = None, + ): + self.logger = logger + self.spans: dict[UUID, Span] = {} + self.debug = debug # DEPRECATED + self.exclude_metadata_props = exclude_metadata_props or re.compile( + r"^(l[sc]_|langgraph_|__pregel_|checkpoint_ns)" + ) + self.skipped_runs: set[UUID] = set() + # Set run_inline=True to avoid thread executor in async contexts + # This ensures memory logger context is preserved + self.run_inline = True + + self._start_times: dict[UUID, float] = {} + self._first_token_times: dict[UUID, float] = {} + self._ttft_ms: dict[UUID, float] = {} + + def _start_span( + self, + parent_run_id: UUID | None, + run_id: UUID, + name: str | None = None, + type: SpanTypeAttribute | None = SpanTypeAttribute.TASK, + span_attributes: SpanAttributes | Mapping[str, Any] | None = None, + start_time: float | None = None, + set_current: bool | None = None, + parent: str | None = None, + event: LogEvent | None = None, + ) -> Any: + if run_id in self.spans: + # XXX: See graph test case of an example where this _may_ be intended. + _logger.warning(f"Span already exists for run_id {run_id} (this is likely a bug)") + return + + if not parent_run_id: + self.root_run_id = run_id + + current_parent = current_span() + parent_span = None + if parent_run_id and parent_run_id in self.spans: + parent_span = self.spans[parent_run_id] + elif current_parent != NOOP_SPAN: + parent_span = current_parent + elif self.logger is not None: + parent_span = self.logger + else: + parent_span = braintrust + + if event is None: + event = {} + + tags = event.get("tags") or [] + event = { + **event, + "tags": None, + "metadata": { + **({"tags": tags}), + **(event.get("metadata") or {}), + "run_id": run_id, + "parent_run_id": parent_run_id, + "braintrust": { + "integration_name": "langchain-py", + "integration_version": _integration_version, + "sdk_version": sdk_version, + "language": "python", + }, + }, + } + + span = parent_span.start_span( + name=name, + type=type, + span_attributes=span_attributes, + start_time=start_time, + set_current=set_current, + parent=parent, + **event, + ) + + if self.logger != NOOP_SPAN and span == NOOP_SPAN: + _logger.warning( + "Braintrust logging not configured. Pass a `logger`, call `init_logger`, or run an experiment to configure Braintrust logging. Setting up a default." + ) + span = init_logger().start_span( + name=name, + type=type, + span_attributes=span_attributes, + start_time=start_time, + set_current=set_current, + parent=parent, + **event, + ) + + span.set_current() + + self.spans[run_id] = span + return span + + def _end_span( + self, + run_id: UUID, + parent_run_id: UUID | None = None, + input: Any | None = None, + output: Any | None = None, + expected: Any | None = None, + error: str | None = None, + tags: Sequence[str] | None = None, + scores: Mapping[str, int | float] | None = None, + metadata: Mapping[str, Any] | None = None, + metrics: Mapping[str, int | float] | None = None, + dataset_record_id: str | None = None, + ) -> Any: + if run_id not in self.spans: + return + + if run_id in self.skipped_runs: + self.skipped_runs.discard(run_id) + return + + span = self.spans.pop(run_id) + + if self.root_run_id == run_id: + self.root_run_id = None + + span.log( + input=input, + output=output, + expected=expected, + error=error, + tags=None, + scores=scores, + metadata={ + **({"tags": tags} if tags else {}), + **(metadata or {}), + }, + metrics=metrics, + dataset_record_id=dataset_record_id, + ) + + # In async workflows, callbacks may execute in different async contexts. + # The span's context variable token may have been created in a different + # context, causing ValueError when trying to reset it. We catch and ignore + # this specific error since the span hierarchy is maintained via self.spans. + try: + span.unset_current() + except ValueError as e: + if "was created in a different Context" in str(e): + pass + else: + raise + + span.end() + + def on_llm_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, # TODO: response= + ) -> Any: + self._end_span(run_id, error=str(error), metadata={**kwargs}) + + self._start_times.pop(run_id, None) + self._first_token_times.pop(run_id, None) + self._ttft_ms.pop(run_id, None) + + def on_chain_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, # TODO: some metadata + ) -> Any: + self._end_span(run_id, error=str(error), metadata={**kwargs}) + + def on_tool_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> Any: + self._end_span(run_id, error=str(error), metadata={**kwargs}) + + def on_retriever_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> Any: + self._end_span(run_id, error=str(error), metadata={**kwargs}) + + # Agent Methods + def on_agent_action( + self, + action: AgentAction, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> Any: + self._start_span( + parent_run_id, + run_id, + type=SpanTypeAttribute.LLM, + name=action.tool, + event={"input": action, "metadata": {**kwargs}}, + ) + + def on_agent_finish( + self, + finish: AgentFinish, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> Any: + self._end_span(run_id, output=finish, metadata={**kwargs}) + + def on_chain_start( + self, + serialized: dict[str, Any], + inputs: dict[str, Any], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + name: str | None = None, + metadata: dict[str, Any] | None = None, + **kwargs: Any, + ) -> Any: + tags = tags or [] + + # avoids extra logs that seem not as useful esp. with langgraph + if "langsmith:hidden" in tags: + self.skipped_runs.add(run_id) + return + + metadata = metadata or {} + resolved_name = ( + name + or metadata.get("langgraph_node") + or serialized.get("name") + or last_item(serialized.get("id") or []) + or "Chain" + ) + + self._start_span( + parent_run_id, + run_id, + name=resolved_name, + event={ + "input": inputs, + "tags": tags, + "metadata": { + "serialized": serialized, + "name": name, + "metadata": metadata, + **kwargs, + }, + }, + ) + + def on_chain_end( + self, + outputs: dict[str, Any], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + **kwargs: Any, + ) -> Any: + self._end_span(run_id, output=outputs, tags=tags, metadata={**kwargs}) + + def on_llm_start( + self, + serialized: dict[str, Any], + prompts: list[str], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + name: str | None = None, + **kwargs: Any, + ) -> Any: + self._start_times[run_id] = time.perf_counter() + self._first_token_times.pop(run_id, None) + self._ttft_ms.pop(run_id, None) + + name = name or serialized.get("name") or last_item(serialized.get("id") or []) or "LLM" + self._start_span( + parent_run_id, + run_id, + name=name, + type=SpanTypeAttribute.LLM, + event={ + "input": prompts, + "tags": tags, + "metadata": { + "serialized": serialized, + "name": name, + "metadata": metadata, + **kwargs, + }, + }, + ) + + def on_chat_model_start( + self, + serialized: dict[str, Any], + messages: list[list["BaseMessage"]], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + name: str | None = None, + invocation_params: dict[str, Any] | None = None, + **kwargs: Any, + ) -> Any: + self._start_times[run_id] = time.perf_counter() + self._first_token_times.pop(run_id, None) + self._ttft_ms.pop(run_id, None) + + invocation_params = invocation_params or {} + self._start_span( + parent_run_id, + run_id, + name=name or serialized.get("name") or last_item(serialized.get("id") or []) or "Chat Model", + type=SpanTypeAttribute.LLM, + event={ + "input": messages, + "tags": tags, + "metadata": ( + { + "serialized": serialized, + "invocation_params": invocation_params, + "metadata": metadata or {}, + "name": name, + **kwargs, + } + ), + }, + ) + + def on_llm_end( + self, + response: LLMResult, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + **kwargs: Any, + ) -> Any: + if run_id not in self.spans: + return + + metrics = _get_metrics_from_response(response) + + ttft = self._ttft_ms.pop(run_id, None) + if ttft is not None: + metrics["time_to_first_token"] = ttft + + model_name = _get_model_name_from_response(response) + + self._start_times.pop(run_id, None) + self._first_token_times.pop(run_id, None) + + self._end_span( + run_id, + output=response, + metrics=metrics, + tags=tags, + metadata={ + "model": model_name, + **kwargs, + }, + ) + + def on_tool_start( + self, + serialized: dict[str, Any], + input_str: str, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + inputs: dict[str, Any] | None = None, + name: str | None = None, + **kwargs: Any, + ) -> Any: + self._start_span( + parent_run_id, + run_id, + name=name or serialized.get("name") or last_item(serialized.get("id") or []) or "Tool", + type=SpanTypeAttribute.TOOL, + event={ + "input": inputs or safe_parse_serialized_json(input_str), + "tags": tags, + "metadata": { + "metadata": metadata, + "serialized": serialized, + "input_str": input_str, + "input": safe_parse_serialized_json(input_str), + "inputs": inputs, + "name": name, + **kwargs, + }, + }, + ) + + def on_tool_end( + self, + output: Any, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> Any: + self._end_span(run_id, output=output, metadata={**kwargs}) + + def on_retriever_start( + self, + serialized: dict[str, Any], + query: str, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + name: str | None = None, + **kwargs: Any, + ) -> Any: + self._start_span( + parent_run_id, + run_id, + name=name or serialized.get("name") or last_item(serialized.get("id") or []) or "Retriever", + type=SpanTypeAttribute.FUNCTION, + event={ + "input": query, + "tags": tags, + "metadata": { + "serialized": serialized, + "metadata": metadata, + "name": name, + **kwargs, + }, + }, + ) + + def on_retriever_end( + self, + documents: Sequence[Document], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> Any: + self._end_span(run_id, output=documents, metadata={**kwargs}) + + def on_llm_new_token( + self, + token: str, + *, + chunk: Union["GenerationChunk", "ChatGenerationChunk"] | None = None, # type: ignore + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> Any: + if run_id not in self._first_token_times: + now = time.perf_counter() + self._first_token_times[run_id] = now + start = self._start_times.get(run_id) + if start is not None: + self._ttft_ms[run_id] = now - start + + def on_text( + self, + text: str, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> Any: + pass + + def on_retry( + self, + retry_state: RetryCallState, + *, + run_id: UUID, + parent_run_id: UUID | None = None, + **kwargs: Any, + ) -> Any: + pass + + def on_custom_event( + self, + name: str, + data: Any, + *, + run_id: UUID, + tags: list[str] | None = None, + metadata: dict[str, Any] | None = None, + **kwargs: Any, + ) -> Any: + pass + + +def clean_object(obj: dict[str, Any]) -> dict[str, Any]: + return { + k: v + for k, v in obj.items() + if v is not None and not (isinstance(v, list) and not v) and not (isinstance(v, dict) and not v) + } + + +def safe_parse_serialized_json(input_str: str) -> Any: + try: + return json.loads(input_str) + except: + return input_str + + +def last_item(items: list[Any]) -> Any: + return items[-1] if items else None + + +def _walk_generations(response: LLMResult): + for generations in response.generations or []: + yield from generations or [] + + +def _get_model_name_from_response(response: LLMResult) -> str | None: + model_name = None + for generation in _walk_generations(response): + message = getattr(generation, "message", None) + if not message: + continue + + response_metadata = getattr(message, "response_metadata", None) + if response_metadata and isinstance(response_metadata, dict): + model_name = response_metadata.get("model_name") + + if model_name: + break + + if not model_name: + llm_output: dict[str, Any] = response.llm_output or {} + model_name = llm_output.get("model_name") or llm_output.get("model") or "" + + return model_name + + +def _get_metrics_from_response(response: LLMResult): + metrics = {} + + for generation in _walk_generations(response): + message = getattr(generation, "message", None) + if not message: + continue + + usage_metadata = getattr(message, "usage_metadata", None) + + if usage_metadata and isinstance(usage_metadata, dict): + metrics.update( + clean_object( + { + "total_tokens": usage_metadata.get("total_tokens"), + "prompt_tokens": usage_metadata.get("input_tokens"), + "completion_tokens": usage_metadata.get("output_tokens"), + } + ) + ) + + # Extract cache tokens from nested input_token_details (LangChain format) + # Maps to Braintrust's standard cache token metric names + input_token_details = usage_metadata.get("input_token_details") + if input_token_details and isinstance(input_token_details, dict): + cache_read = input_token_details.get("cache_read") + cache_creation = input_token_details.get("cache_creation") + + if cache_read is not None: + metrics["prompt_cached_tokens"] = cache_read + if cache_creation is not None: + metrics["prompt_cache_creation_tokens"] = cache_creation + + if not metrics or not any(metrics.values()): + llm_output: dict[str, Any] = response.llm_output or {} + metrics = llm_output.get("token_usage") or llm_output.get("estimatedTokens") or {} + + return clean_object(metrics) diff --git a/py/src/braintrust/wrappers/langchain/context.py b/py/src/braintrust/wrappers/langchain/context.py new file mode 100644 index 000000000..d55eb9b30 --- /dev/null +++ b/py/src/braintrust/wrappers/langchain/context.py @@ -0,0 +1,26 @@ +from contextvars import ContextVar + +from langchain_core.tracers.context import register_configure_hook + +from .callbacks import BraintrustCallbackHandler + +__all__ = ["set_global_handler", "clear_global_handler"] + + +braintrust_callback_handler_var: ContextVar[BraintrustCallbackHandler | None] = ContextVar( + "braintrust_callback_handler", default=None +) + + +def set_global_handler(handler: BraintrustCallbackHandler): + braintrust_callback_handler_var.set(handler) + + +def clear_global_handler(): + braintrust_callback_handler_var.set(None) + + +register_configure_hook( + context_var=braintrust_callback_handler_var, + inheritable=True, +) diff --git a/py/src/braintrust/wrappers/test_langchain.py b/py/src/braintrust/wrappers/test_langchain.py new file mode 100644 index 000000000..9f6c12f9c --- /dev/null +++ b/py/src/braintrust/wrappers/test_langchain.py @@ -0,0 +1,635 @@ +# pyright: reportTypedDictNotRequiredAccess=none +""" +Tests for LangChain integration. + +Migrated from integrations/langchain-py/src/tests/ +""" + +import os +import uuid +from typing import Any, Dict, List, Optional, Sequence, TypedDict, Union, cast +from unittest.mock import ANY + +import pytest +from braintrust import flush +from braintrust.logger import ( + TEST_API_KEY, + Logger, + _internal_reset_global_state, + _internal_with_memory_background_logger, + _MemoryBackgroundLogger, +) +from braintrust.test_helpers import init_test_logger +from braintrust.wrappers.langchain import ( + BraintrustCallbackHandler, + clear_global_handler, + set_global_handler, + setup_langchain, +) + +# --- Type definitions (from types.py) --- + + +class SpanAttributes(TypedDict): + name: str + type: Optional[str] + + +class SpanMetadata(TypedDict, total=False): + tags: List[str] + model: str + temperature: float + top_p: float + frequency_penalty: float + presence_penalty: float + n: int + runId: Optional[str] + + +class SpanRequired(TypedDict): + span_id: str + + +class Span(SpanRequired, total=False): + span_attributes: SpanAttributes + input: Any + output: Any + span_parents: Optional[List[str]] + metadata: SpanMetadata + root_span_id: str + metrics: Dict[str, Any] + + +# --- Helper functions (from helpers.py) --- + +PrimitiveValue = Union[str, int, float, bool, None, Span] +RecursiveValue = Union[PrimitiveValue, Dict[str, Any], Sequence[Any]] + + +def assert_matches_object( + actual: RecursiveValue, + expected: RecursiveValue, + ignore_order: bool = False, +) -> None: + """Assert that actual contains all key-value pairs from expected.""" + if isinstance(expected, (list, tuple)): + assert isinstance(actual, (list, tuple)), f"Expected sequence but got {type(actual)}" + assert len(actual) >= len(expected), ( + f"Expected sequence of length >= {len(expected)} but got length {len(actual)}" + ) + if not ignore_order: + for i, expected_item in enumerate(expected): + assert_matches_object(actual[i], expected_item) + else: + for expected_item in expected: + matched = False + for actual_item in actual: + try: + assert_matches_object(actual_item, expected_item) + matched = True + except: + pass + assert matched, f"Expected {expected_item} in unordered sequence but couldn't find match in {actual}" + + elif isinstance(expected, dict): + assert isinstance(actual, dict), f"Expected dict but got {type(actual)}" + for k, v in expected.items(): + assert k in actual, f"Missing key {k}" + if v is ANY: + continue + if isinstance(v, (dict, list, tuple)): + assert_matches_object(cast(RecursiveValue, actual[k]), cast(RecursiveValue, v)) + else: + assert actual[k] == v, f"Key {k}: expected {v} but got {actual[k]}" + else: + assert actual == expected, f"Expected {expected} but got {actual}" + + +def find_spans_by_attributes(spans: List[Span], **attributes: Any) -> List[Span]: + """Find all spans that match the given attributes.""" + matching_spans: List[Span] = [] + for span in spans: + matches = True + if "span_attributes" not in span: + matches = False + continue + for key, value in attributes.items(): + if key not in span["span_attributes"] or span["span_attributes"][key] != value: + matches = False + break + if matches: + matching_spans.append(span) + return matching_spans + + +# --- Fixtures --- + +LoggerMemoryLogger = tuple[Logger, _MemoryBackgroundLogger] + + +@pytest.fixture(autouse=True) +def setup_braintrust(): + os.environ["BRAINTRUST_SYNC_FLUSH"] = "1" + os.environ["BRAINTRUST_API_URL"] = "http://localhost:8000" + os.environ["BRAINTRUST_APP_URL"] = "http://localhost:3000" + os.environ["BRAINTRUST_API_KEY"] = TEST_API_KEY + os.environ["ANTHROPIC_API_KEY"] = "your_anthropic_api_key_here" + os.environ["OPENAI_API_KEY"] = "your_openai_api_key_here" + os.environ["OPENAI_BASE_URL"] = "http://localhost:8000/v1/proxy" + + _internal_reset_global_state() + clear_global_handler() + yield + + +@pytest.fixture(scope="module") +def vcr_config(): + record_mode = "none" if (os.environ.get("CI") or os.environ.get("GITHUB_ACTIONS")) else "once" + + return { + "filter_headers": [ + "authorization", + "x-goog-api-key", + "x-api-key", + "api-key", + "openai-api-key", + ], + "record_mode": record_mode, + "match_on": ["uri", "method", "body"], + "cassette_library_dir": "src/braintrust/wrappers/cassettes/test_langchain", + "path_transformer": lambda path: path.replace(".yaml", ""), + } + + +@pytest.fixture +def logger_memory_logger(): + logger = init_test_logger("langchain-py") + with _internal_with_memory_background_logger() as bgl: + yield (logger, bgl) + + +# --- Tests --- + + +def test_setup_langchain(): + """Test that setup_langchain registers a global handler.""" + clear_global_handler() + result = setup_langchain() + assert result is True + + # Verify handler is registered + from langchain_core.callbacks import CallbackManager + manager = CallbackManager.configure() + assert any(isinstance(h, BraintrustCallbackHandler) for h in manager.handlers) + + +@pytest.mark.vcr +def test_llm_calls(logger_memory_logger: LoggerMemoryLogger): + from langchain_core.callbacks import BaseCallbackHandler + from langchain_core.messages import BaseMessage + from langchain_core.prompts import ChatPromptTemplate + from langchain_core.runnables import RunnableSerializable + from langchain_openai import ChatOpenAI + + logger, memory_logger = logger_memory_logger + assert not memory_logger.pop() + + handler = BraintrustCallbackHandler(logger=logger) + prompt = ChatPromptTemplate.from_template("What is 1 + {number}?") + model = ChatOpenAI( + model="gpt-4o-mini", + temperature=1, + top_p=1, + frequency_penalty=0, + presence_penalty=0, + n=1, + ) + chain: RunnableSerializable[Dict[str, str], BaseMessage] = prompt.pipe(model) + chain.invoke({"number": "2"}, config={"callbacks": [cast(BaseCallbackHandler, handler)]}) + + spans = memory_logger.pop() + assert len(spans) == 3 + + root_span_id = spans[0]["span_id"] + + assert_matches_object( + spans, + [ + { + "span_attributes": { + "name": "RunnableSequence", + "type": "task", + }, + "input": {"number": "2"}, + "metadata": {"tags": []}, + "span_id": root_span_id, + "root_span_id": root_span_id, + }, + { + "span_attributes": {"name": "ChatPromptTemplate"}, + "input": {"number": "2"}, + "metadata": {"tags": ["seq:step:1"]}, + "root_span_id": root_span_id, + "span_parents": [root_span_id], + }, + { + "span_attributes": {"name": "ChatOpenAI", "type": "llm"}, + "metadata": { + "tags": ["seq:step:2"], + "model": "gpt-4o-mini-2024-07-18", + }, + "root_span_id": root_span_id, + "span_parents": [root_span_id], + }, + ], + ) + + +@pytest.mark.vcr +def test_chain_with_memory(logger_memory_logger: LoggerMemoryLogger): + from langchain_core.callbacks import BaseCallbackHandler + from langchain_core.messages import BaseMessage + from langchain_core.prompts import ChatPromptTemplate + from langchain_core.runnables import RunnableSerializable + from langchain_openai import ChatOpenAI + + logger, memory_logger = logger_memory_logger + assert not memory_logger.pop() + + handler = BraintrustCallbackHandler(logger=logger) + prompt = ChatPromptTemplate.from_template("{history} User: {input}") + model = ChatOpenAI(model="gpt-4o-mini") + chain: RunnableSerializable[Dict[str, str], BaseMessage] = prompt.pipe(model) + + memory = {"history": "Assistant: Hello! How can I assist you today?"} + chain.invoke( + {"input": "What's your name?", **memory}, + config={"callbacks": [cast(BaseCallbackHandler, handler)], "tags": ["test"]}, + ) + + spans = memory_logger.pop() + assert len(spans) == 3 + + root_span_id = spans[0]["span_id"] + + assert_matches_object( + spans, + [ + { + "span_attributes": { + "name": "RunnableSequence", + "type": "task", + }, + "input": {"input": "What's your name?", "history": "Assistant: Hello! How can I assist you today?"}, + "metadata": {"tags": ["test"]}, + "span_id": root_span_id, + "root_span_id": root_span_id, + }, + ], + ) + + +@pytest.mark.vcr +def test_tool_usage(logger_memory_logger: LoggerMemoryLogger): + from langchain_core.callbacks import BaseCallbackHandler + from langchain_core.tools import tool + from langchain_openai import ChatOpenAI + from pydantic import BaseModel, Field + + logger, memory_logger = logger_memory_logger + assert not memory_logger.pop() + + handler = BraintrustCallbackHandler(logger=logger) + + class CalculatorInput(BaseModel): + operation: str = Field( + description="The type of operation to execute.", + json_schema_extra={"enum": ["add", "subtract", "multiply", "divide"]}, + ) + number1: float = Field(description="The first number to operate on.") + number2: float = Field(description="The second number to operate on.") + + @tool + def calculator(input: CalculatorInput) -> str: + """Can perform mathematical operations.""" + if input.operation == "add": + return str(input.number1 + input.number2) + elif input.operation == "subtract": + return str(input.number1 - input.number2) + elif input.operation == "multiply": + return str(input.number1 * input.number2) + elif input.operation == "divide": + return str(input.number1 / input.number2) + else: + raise ValueError("Invalid operation.") + + model = ChatOpenAI( + model="gpt-4o-mini", + temperature=1, + top_p=1, + frequency_penalty=0, + presence_penalty=0, + n=1, + ) + model_with_tools = model.bind_tools([calculator]) + model_with_tools.invoke("What is 3 * 12", config={"callbacks": [cast(BaseCallbackHandler, handler)]}) + + spans = memory_logger.pop() + root_span_id = spans[0]["span_id"] + + assert_matches_object( + spans, + [ + { + "span_id": root_span_id, + "root_span_id": root_span_id, + "span_attributes": { + "name": "ChatOpenAI", + "type": "llm", + }, + "metadata": { + "tags": [], + "model": "gpt-4o-mini-2024-07-18", + }, + } + ], + ) + + +@pytest.mark.vcr +def test_langgraph_state_management(logger_memory_logger: LoggerMemoryLogger): + from langchain_openai import ChatOpenAI + from langgraph.graph import END, START, StateGraph + + logger, memory_logger = logger_memory_logger + assert not memory_logger.pop() + + handler = BraintrustCallbackHandler(logger=logger) + model = ChatOpenAI( + model="gpt-4o-mini", + temperature=1, + top_p=1, + frequency_penalty=0, + presence_penalty=0, + n=1, + ) + + def say_hello(state: Dict[str, str]): + response = model.invoke("Say hello") + return cast(Union[str, List[str], Dict[str, str]], response.content) + + def say_bye(state: Dict[str, str]): + return "Bye" + + workflow = ( + StateGraph(state_schema=Dict[str, str]) + .add_node("sayHello", say_hello) + .add_node("sayBye", say_bye) + .add_edge(START, "sayHello") + .add_edge("sayHello", "sayBye") + .add_edge("sayBye", END) + ) + + graph = workflow.compile() + graph.invoke({}, config={"callbacks": [handler]}) + + spans = memory_logger.pop() + + langgraph_spans = find_spans_by_attributes(spans, name="LangGraph") + say_hello_spans = find_spans_by_attributes(spans, name="sayHello") + say_bye_spans = find_spans_by_attributes(spans, name="sayBye") + llm_spans = find_spans_by_attributes(spans, name="ChatOpenAI") + + assert len(langgraph_spans) == 1 + assert len(say_hello_spans) == 1 + assert len(say_bye_spans) == 1 + assert len(llm_spans) == 1 + + assert_matches_object( + langgraph_spans[0], + { + "span_attributes": { + "name": "LangGraph", + "type": "task", + }, + "input": {}, + "metadata": { + "tags": [], + }, + "output": "Bye", + }, + ) + + +@pytest.mark.vcr +def test_global_handler(logger_memory_logger: LoggerMemoryLogger): + from langchain_core.callbacks import CallbackManager + from langchain_core.messages import BaseMessage + from langchain_core.prompts import ChatPromptTemplate + from langchain_core.runnables import RunnableSerializable + from langchain_openai import ChatOpenAI + + logger, memory_logger = logger_memory_logger + assert not memory_logger.pop() + + handler = BraintrustCallbackHandler(logger=logger, debug=True) + set_global_handler(handler) + + manager = CallbackManager.configure() + assert next((h for h in manager.handlers if isinstance(h, BraintrustCallbackHandler)), None) == handler + + prompt = ChatPromptTemplate.from_template("What is 1 + {number}?") + model = ChatOpenAI( + model="gpt-4o-mini", + temperature=1, + top_p=1, + frequency_penalty=0, + presence_penalty=0, + n=1, + ) + chain: RunnableSerializable[Dict[str, str], BaseMessage] = prompt.pipe(model) + + message = chain.invoke({"number": "2"}) + + spans = memory_logger.pop() + assert len(spans) > 0 + + root_span_id = spans[0]["span_id"] + + assert_matches_object( + spans, + [ + { + "span_attributes": { + "name": "RunnableSequence", + "type": "task", + }, + "input": {"number": "2"}, + "metadata": {"tags": []}, + "span_id": root_span_id, + "root_span_id": root_span_id, + }, + ], + ) + + assert message.content == "1 + 2 equals 3." + + +@pytest.mark.vcr +def test_streaming_ttft(logger_memory_logger: LoggerMemoryLogger): + from langchain_core.callbacks import BaseCallbackHandler + from langchain_core.messages import BaseMessage + from langchain_core.prompts import ChatPromptTemplate + from langchain_core.runnables import RunnableSerializable + from langchain_openai import ChatOpenAI + + logger, memory_logger = logger_memory_logger + assert not memory_logger.pop() + + handler = BraintrustCallbackHandler(logger=logger) + prompt = ChatPromptTemplate.from_template("Count from 1 to 5.") + model = ChatOpenAI( + model="gpt-4o-mini", + max_completion_tokens=50, + streaming=True, + ) + chain: RunnableSerializable[Dict[str, str], BaseMessage] = prompt.pipe(model) + + chunks: List[str] = [] + for chunk in chain.stream({}, config={"callbacks": [cast(BaseCallbackHandler, handler)]}): + if chunk.content: + chunks.append(str(chunk.content)) + + assert len(chunks) > 0, "Expected to receive streaming chunks" + + spans = memory_logger.pop() + assert len(spans) == 3 + + llm_spans = find_spans_by_attributes(spans, name="ChatOpenAI", type="llm") + assert len(llm_spans) == 1 + llm_span = llm_spans[0] + + assert "metrics" in llm_span + assert "time_to_first_token" in llm_span["metrics"] + + +@pytest.mark.vcr +def test_langchain_anthropic_integration(logger_memory_logger: LoggerMemoryLogger): + from langchain_anthropic import ChatAnthropic + from langchain_core.prompts import ChatPromptTemplate + + MODEL = "claude-sonnet-4-20250514" + + logger, memory_logger = logger_memory_logger + assert not memory_logger.pop() + + handler = BraintrustCallbackHandler(logger=logger) + set_global_handler(handler) + + prompt = ChatPromptTemplate.from_template("What is 1 + {number}?") + # max_tokens must match the recorded cassette (default changed from 1024 to 64000) + model = ChatAnthropic(model_name=MODEL, max_tokens=1024) + + chain = prompt | model + + result = chain.invoke({"number": "2"}) + + flush() + + assert isinstance(result.content, str) + assert "3" in result.content.lower() + + spans = memory_logger.pop() + assert len(spans) > 0 + + llm_spans = [span for span in spans if span["span_attributes"].get("type") == "llm"] + assert len(llm_spans) > 0, "Should have at least one LLM call" + + llm_span = llm_spans[0] + assert llm_span["metadata"]["model"] == MODEL + + +@pytest.mark.vcr +@pytest.mark.asyncio +async def test_async_langchain_invoke(logger_memory_logger: LoggerMemoryLogger): + from langchain_anthropic import ChatAnthropic + from langchain_core.prompts import ChatPromptTemplate + + MODEL = "claude-sonnet-4-20250514" + + logger, memory_logger = logger_memory_logger + assert not memory_logger.pop() + + handler = BraintrustCallbackHandler(logger=logger) + set_global_handler(handler) + + prompt = ChatPromptTemplate.from_template("What is 1 + {number}?") + # max_tokens must match the recorded cassette (default changed from 1024 to 64000) + model = ChatAnthropic(model_name=MODEL, max_tokens=1024) + + chain = prompt | model + + result = await chain.ainvoke({"number": "2"}) + + flush() + + assert isinstance(result.content, str) + assert "3" in result.content.lower() + + spans = memory_logger.pop() + assert len(spans) > 0 + + +def test_chain_null_values(logger_memory_logger: LoggerMemoryLogger): + logger, memory_logger = logger_memory_logger + assert not memory_logger.pop() + + handler = BraintrustCallbackHandler(logger=logger) + + run_id = uuid.UUID("f81d4fae-7dec-11d0-a765-00a0c91e6bf6") + + handler.on_chain_start( + {"id": ["TestChain"], "lc": 1, "type": "not_implemented"}, + {"input1": "value1", "input2": None, "input3": None}, + run_id=run_id, + parent_run_id=None, + tags=["test"], + ) + + handler.on_chain_end( + {"output1": "value1", "output2": None, "output3": None}, + run_id=run_id, + parent_run_id=None, + tags=["test"], + ) + + flush() + + spans = memory_logger.pop() + root_span_id = spans[0]["span_id"] + + assert_matches_object( + spans, + [ + { + "root_span_id": root_span_id, + "span_attributes": { + "name": "TestChain", + "type": "task", + }, + "input": { + "input1": "value1", + "input2": None, + "input3": None, + }, + "metadata": { + "tags": ["test"], + }, + "output": { + "output1": "value1", + "output2": None, + "output3": None, + }, + }, + ], + )