From b9375f14fd197d426e5d11df55d4b3481a0a86a3 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Fri, 10 Jul 2026 10:44:52 -0400
Subject: [PATCH 01/26] Prototype: BYO admin-connected judge models via project
 Responses API shim

AzureOpenAIGrader routes admin-connected judge models ('connection/deployment') through the Foundry project Responses API using a chat.completions-compatible shim (_byo_judge.py), so the platform resolves the connection. Judges cannot use chat.completions for BYO (404 DeploymentNotFound).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_aoai/aoai_grader.py  |  30 ++++
 .../azure/ai/evaluation/_byo_judge.py         | 157 ++++++++++++++++++
 .../tests/unittests/test_byo_judge.py         | 111 +++++++++++++
 3 files changed, 298 insertions(+)
 create mode 100644 sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
 create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_aoai/aoai_grader.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_aoai/aoai_grader.py
index 07e771303326..79f70fbbf634 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_aoai/aoai_grader.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_aoai/aoai_grader.py
@@ -61,6 +61,27 @@ def __init__(
 
     def _validate_model_config(self) -> None:
         """Validate the model configuration that this grader wrapper is using."""
+        # Prototype: admin-connected (BYO) judge models are served via the project Responses
+        # API and require a credential + project endpoint (no api_key / azure_endpoint).
+        from azure.ai.evaluation._byo_judge import is_byo_model_config
+
+        if is_byo_model_config(self._model_config):
+            if not self._credential:
+                raise EvaluationException(
+                    message=f"{type(self).__name__}: BYO (admin-connected) judge models require a credential.",
+                    blame=ErrorBlame.USER_ERROR,
+                    category=ErrorCategory.INVALID_VALUE,
+                    target=ErrorTarget.AOAI_GRADER,
+                )
+            if not self._model_config.get("project_endpoint"):
+                raise EvaluationException(
+                    message=f"{type(self).__name__}: BYO judge model_config requires 'project_endpoint'.",
+                    blame=ErrorBlame.USER_ERROR,
+                    category=ErrorCategory.INVALID_VALUE,
+                    target=ErrorTarget.AOAI_GRADER,
+                )
+            return
+
         msg = None
         if self._is_azure_model_config(self._model_config):
             if not any(auth for auth in (self._model_config.get("api_key"), self._credential)):
@@ -104,6 +125,15 @@ def get_client(self) -> Any:
         """
         default_headers = {"User-Agent": UserAgentSingleton().value}
         model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration] = self._model_config
+
+        # Prototype: admin-connected (BYO) judge model referenced as "connection/deployment".
+        # Route judge calls through the Foundry project Responses API via a
+        # chat.completions-compatible shim, so the platform resolves the connection.
+        from azure.ai.evaluation._byo_judge import is_byo_model_config, build_byo_judge_client
+
+        if is_byo_model_config(model_config):
+            return build_byo_judge_client(model_config, self._credential)
+
         api_key: Optional[str] = model_config.get("api_key")
 
         if self._is_azure_model_config(model_config):
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
new file mode 100644
index 000000000000..99e22078988f
--- /dev/null
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -0,0 +1,157 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""Prototype: use admin-connected (BYO) models as judge / grader models.
+
+Admin-connected models (Foundry "ModelGateway" / "API Management" connections, referenced
+as ``"connection-name/deployment-name"``) are only invokable through the Foundry project
+**Responses API** — the platform resolves the connection and handles every auth type
+(API key / managed identity / OAuth), ``deploymentInPath``, api-version and custom headers.
+
+LLM-as-judge evaluators in this library call ``client.chat.completions.create(...)``. This
+module provides a small OpenAI-compatible **shim** that routes those calls to the project
+Responses API for a BYO model, so judge/grader code can use admin-connected connections
+**without any change to its calling code**.
+"""
+import time
+from typing import Any, Dict, List, Optional
+
+from typing_extensions import TypedDict
+
+from azure.core.credentials import TokenCredential
+
+
+class BYOProjectModelConfiguration(TypedDict, total=False):
+    """Model configuration for an admin-connected (BYO) judge model.
+
+    :keyword byo_model: The admin-connected model reference, ``"connection-name/deployment-name"``.
+    :keyword project_endpoint: The Foundry project endpoint,
+        e.g. ``https://<account>.services.ai.azure.com/api/projects/<project>``.
+    """
+
+    byo_model: str
+    project_endpoint: str
+
+
+def _to_responses_input(messages: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
+    """Map chat-completions messages ({role, content}) to Responses API input items."""
+    items: List[Dict[str, Any]] = []
+    for message in messages or []:
+        items.append(
+            {
+                "type": "message",
+                "role": message.get("role", "user"),
+                "content": message.get("content", ""),
+            }
+        )
+    return items
+
+
+def _map_params(kwargs: Dict[str, Any]) -> Dict[str, Any]:
+    """Map a curated set of chat-completions sampling params to Responses API params."""
+    mapped: Dict[str, Any] = {}
+    if "temperature" in kwargs:
+        mapped["temperature"] = kwargs["temperature"]
+    if "top_p" in kwargs:
+        mapped["top_p"] = kwargs["top_p"]
+    for key in ("max_output_tokens", "max_completion_tokens", "max_tokens"):
+        if key in kwargs:
+            mapped["max_output_tokens"] = kwargs[key]
+            break
+    return mapped
+
+
+class _ChatMessage:
+    def __init__(self, content: str) -> None:
+        self.role = "assistant"
+        self.content = content
+        self.tool_calls = None
+
+
+class _Choice:
+    def __init__(self, content: str) -> None:
+        self.index = 0
+        self.message = _ChatMessage(content)
+        self.finish_reason = "stop"
+
+
+class _ChatCompletion:
+    """Minimal chat.completions-shaped view over a Responses API result."""
+
+    def __init__(self, response: Any) -> None:
+        self.id = getattr(response, "id", None)
+        self.model = getattr(response, "model", None)
+        self.usage = getattr(response, "usage", None)
+        self.object = "chat.completion"
+        self.created = int(time.time())
+        self.choices = [_Choice(getattr(response, "output_text", "") or "")]
+
+
+class _ChatCompletions:
+    def __init__(self, owner: "ByoProjectResponsesClient") -> None:
+        self._owner = owner
+
+    def create(self, *, model: Optional[str] = None, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> _ChatCompletion:
+        # ``model`` from the caller is ignored — the BYO model is fixed by the shim's config.
+        response = self._owner._responses_create(messages=messages, **kwargs)
+        return _ChatCompletion(response)
+
+
+class _Chat:
+    def __init__(self, owner: "ByoProjectResponsesClient") -> None:
+        self.completions = _ChatCompletions(owner)
+
+
+class ByoProjectResponsesClient:
+    """OpenAI-compatible shim that routes ``chat.completions.create()`` to the Foundry
+    project Responses API for an admin-connected (BYO) model.
+
+    Judge/grader code that calls ``client.chat.completions.create(model=..., messages=...)``
+    works unchanged; the request is served by the platform, which resolves the connection.
+    A ``responses`` passthrough is also exposed for callers that use the Responses API directly.
+    """
+
+    def __init__(self, byo_model: str, project_endpoint: str, credential: TokenCredential) -> None:
+        self._byo_model = byo_model
+        self._project_endpoint = project_endpoint
+        self._credential = credential
+        self._client: Any = None
+        self.chat = _Chat(self)
+
+    def _openai(self) -> Any:
+        if self._client is None:
+            from azure.ai.projects import AIProjectClient
+
+            self._client = AIProjectClient(
+                endpoint=self._project_endpoint, credential=self._credential
+            ).get_openai_client()
+        return self._client
+
+    def _responses_create(self, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> Any:
+        return self._openai().responses.create(
+            model=self._byo_model,
+            input=_to_responses_input(messages),
+            **_map_params(kwargs),
+        )
+
+    @property
+    def responses(self) -> Any:
+        return self._openai().responses
+
+
+def is_byo_model_config(model_config: Dict[str, Any]) -> bool:
+    """Return True if the model configuration references an admin-connected (BYO) model."""
+    return bool(model_config) and bool(model_config.get("byo_model"))
+
+
+def build_byo_judge_client(model_config: Dict[str, Any], credential: TokenCredential) -> ByoProjectResponsesClient:
+    """Build a chat.completions-compatible client for an admin-connected (BYO) judge model."""
+    if not model_config.get("byo_model") or not model_config.get("project_endpoint"):
+        raise ValueError("BYOProjectModelConfiguration requires both 'byo_model' and 'project_endpoint'.")
+    if credential is None:
+        raise ValueError("A TokenCredential is required to call the project Responses API for BYO judge models.")
+    return ByoProjectResponsesClient(
+        byo_model=model_config["byo_model"],
+        project_endpoint=model_config["project_endpoint"],
+        credential=credential,
+    )
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
new file mode 100644
index 000000000000..f9fadd16e945
--- /dev/null
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -0,0 +1,111 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""Unit tests for the admin-connected (BYO) judge-model prototype."""
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from azure.ai.evaluation._byo_judge import (
+    ByoProjectResponsesClient,
+    build_byo_judge_client,
+    is_byo_model_config,
+    _to_responses_input,
+    _map_params,
+)
+
+
+class TestByoJudgeHelpers:
+    def test_is_byo_model_config(self):
+        assert is_byo_model_config({"byo_model": "c/d", "project_endpoint": "https://x"})
+        assert not is_byo_model_config({"azure_endpoint": "https://x"})
+        assert not is_byo_model_config({})
+
+    def test_to_responses_input(self):
+        assert _to_responses_input([{"role": "user", "content": "hi"}]) == [
+            {"type": "message", "role": "user", "content": "hi"}
+        ]
+
+    def test_map_params_curates_and_renames(self):
+        assert _map_params({"temperature": 0.0, "max_tokens": 50, "top_p": 0.9, "stream": True}) == {
+            "temperature": 0.0,
+            "top_p": 0.9,
+            "max_output_tokens": 50,
+        }
+
+
+class TestByoJudgeClient:
+    @patch("azure.ai.projects.AIProjectClient")
+    def test_chat_completions_routes_to_responses(self, mock_aipc):
+        resp = MagicMock()
+        resp.output_text = "judge says 5"
+        resp.usage = MagicMock()
+        resp.id = "resp_1"
+        resp.model = "my-conn/gpt-4o"
+        oai = MagicMock()
+        oai.responses.create.return_value = resp
+        mock_aipc.return_value.get_openai_client.return_value = oai
+
+        client = build_byo_judge_client(
+            {"byo_model": "my-conn/gpt-4o", "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p1"},
+            credential=MagicMock(),
+        )
+        # Judge/grader code calls chat.completions.create unchanged.
+        result = client.chat.completions.create(
+            model="ignored",
+            messages=[{"role": "user", "content": "score this 1-5"}],
+            temperature=0.0,
+            max_tokens=10,
+        )
+
+        # chat.completions-shaped result over the Responses output.
+        assert result.choices[0].message.content == "judge says 5"
+        assert result.choices[0].message.role == "assistant"
+
+        # Underlying call is responses.create with the BYO model + mapped input/params.
+        mock_aipc.assert_called_once()
+        _, pkwargs = mock_aipc.call_args
+        assert pkwargs["endpoint"] == "https://acct.services.ai.azure.com/api/projects/p1"
+        _, rkwargs = oai.responses.create.call_args
+        assert rkwargs["model"] == "my-conn/gpt-4o"
+        assert rkwargs["input"] == [{"type": "message", "role": "user", "content": "score this 1-5"}]
+        assert rkwargs["temperature"] == 0.0
+        assert rkwargs["max_output_tokens"] == 10
+
+    def test_build_requires_project_endpoint(self):
+        with pytest.raises(ValueError):
+            build_byo_judge_client({"byo_model": "c/d"}, credential=MagicMock())
+
+    def test_build_requires_credential(self):
+        with pytest.raises(ValueError):
+            build_byo_judge_client({"byo_model": "c/d", "project_endpoint": "https://x"}, credential=None)
+
+
+class TestAzureOpenAIGraderByo:
+    @patch("azure.ai.projects.AIProjectClient")
+    def test_grader_get_client_returns_shim_for_byo(self, _mock_aipc):
+        from azure.ai.evaluation._aoai.aoai_grader import AzureOpenAIGrader
+
+        grader = AzureOpenAIGrader(
+            model_config={
+                "byo_model": "my-conn/gpt-4o",
+                "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p1",
+            },
+            grader_config={},
+            credential=MagicMock(),
+        )
+        assert isinstance(grader.get_client(), ByoProjectResponsesClient)
+
+    def test_grader_byo_requires_credential(self):
+        from azure.ai.evaluation._aoai.aoai_grader import AzureOpenAIGrader
+        from azure.ai.evaluation._exceptions import EvaluationException
+
+        with pytest.raises(EvaluationException):
+            AzureOpenAIGrader(
+                model_config={
+                    "byo_model": "my-conn/gpt-4o",
+                    "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p1",
+                },
+                grader_config={},
+                credential=None,
+            )

From 72ae554e72fbe2e05f439f83747e2191320e2708 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Wed, 15 Jul 2026 11:27:18 -0400
Subject: [PATCH 02/26] Route admin-connected (BYO) judge models through the
 project Responses API (prompty judges)

Enables the 18 prompty-based LLM-as-a-judge builtin evaluators (coherence, relevance,
fluency, groundedness, intent_resolution, similarity, retrieval, task_adherence,
task_completion, tool_call_accuracy/success, tool_input_accuracy, tool_output_utilization,
tool_selection, response_completeness, customer_satisfaction, deflection_rate, quality_grader)
to use admin-connected models referenced as "connection/deployment".

- _byo_judge.py: AsyncByoProjectResponsesClient shims chat.completions -> project Responses
  API; _Usage adapts Responses token usage to chat shape.
- _common/utils.py: validate_model_config passes BYO configs through (they omit
  azure_endpoint/azure_deployment).
- _legacy/prompty/_prompty.py: __call__ routes BYO configs to the async shim.
- Scope: score_model/label_model (AOAI graders) are NOT supported - reverted the grader
  changes and removed the sync client.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_aoai/aoai_grader.py  |  30 -----
 .../azure/ai/evaluation/_byo_judge.py         | 101 +++++++--------
 .../azure/ai/evaluation/_common/utils.py      |   7 ++
 .../ai/evaluation/_legacy/prompty/_prompty.py |  69 +++++++----
 .../tests/unittests/test_byo_judge.py         | 116 ++++++++++--------
 5 files changed, 158 insertions(+), 165 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_aoai/aoai_grader.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_aoai/aoai_grader.py
index 79f70fbbf634..07e771303326 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_aoai/aoai_grader.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_aoai/aoai_grader.py
@@ -61,27 +61,6 @@ def __init__(
 
     def _validate_model_config(self) -> None:
         """Validate the model configuration that this grader wrapper is using."""
-        # Prototype: admin-connected (BYO) judge models are served via the project Responses
-        # API and require a credential + project endpoint (no api_key / azure_endpoint).
-        from azure.ai.evaluation._byo_judge import is_byo_model_config
-
-        if is_byo_model_config(self._model_config):
-            if not self._credential:
-                raise EvaluationException(
-                    message=f"{type(self).__name__}: BYO (admin-connected) judge models require a credential.",
-                    blame=ErrorBlame.USER_ERROR,
-                    category=ErrorCategory.INVALID_VALUE,
-                    target=ErrorTarget.AOAI_GRADER,
-                )
-            if not self._model_config.get("project_endpoint"):
-                raise EvaluationException(
-                    message=f"{type(self).__name__}: BYO judge model_config requires 'project_endpoint'.",
-                    blame=ErrorBlame.USER_ERROR,
-                    category=ErrorCategory.INVALID_VALUE,
-                    target=ErrorTarget.AOAI_GRADER,
-                )
-            return
-
         msg = None
         if self._is_azure_model_config(self._model_config):
             if not any(auth for auth in (self._model_config.get("api_key"), self._credential)):
@@ -125,15 +104,6 @@ def get_client(self) -> Any:
         """
         default_headers = {"User-Agent": UserAgentSingleton().value}
         model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration] = self._model_config
-
-        # Prototype: admin-connected (BYO) judge model referenced as "connection/deployment".
-        # Route judge calls through the Foundry project Responses API via a
-        # chat.completions-compatible shim, so the platform resolves the connection.
-        from azure.ai.evaluation._byo_judge import is_byo_model_config, build_byo_judge_client
-
-        if is_byo_model_config(model_config):
-            return build_byo_judge_client(model_config, self._credential)
-
         api_key: Optional[str] = model_config.get("api_key")
 
         if self._is_azure_model_config(model_config):
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index 99e22078988f..423ab52526f2 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -1,37 +1,21 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-"""Prototype: use admin-connected (BYO) models as judge / grader models.
+"""Use admin-connected (BYO) models as LLM-as-a-judge evaluator models.
 
 Admin-connected models (Foundry "ModelGateway" / "API Management" connections, referenced
 as ``"connection-name/deployment-name"``) are only invokable through the Foundry project
 **Responses API** — the platform resolves the connection and handles every auth type
-(API key / managed identity / OAuth), ``deploymentInPath``, api-version and custom headers.
+(API key / managed identity / OAuth2), ``deploymentInPath``, api-version and custom headers.
 
-LLM-as-judge evaluators in this library call ``client.chat.completions.create(...)``. This
-module provides a small OpenAI-compatible **shim** that routes those calls to the project
-Responses API for a BYO model, so judge/grader code can use admin-connected connections
-**without any change to its calling code**.
+Prompty-based judge evaluators (coherence, relevance, fluency, groundedness, etc.) call
+``client.chat.completions.create(...)``. This module provides a small OpenAI-compatible async
+**shim** that routes those calls to the project Responses API for a BYO model, so evaluator
+code can use admin-connected connections **without any change to its calling code**.
 """
 import time
 from typing import Any, Dict, List, Optional
 
-from typing_extensions import TypedDict
-
-from azure.core.credentials import TokenCredential
-
-
-class BYOProjectModelConfiguration(TypedDict, total=False):
-    """Model configuration for an admin-connected (BYO) judge model.
-
-    :keyword byo_model: The admin-connected model reference, ``"connection-name/deployment-name"``.
-    :keyword project_endpoint: The Foundry project endpoint,
-        e.g. ``https://<account>.services.ai.azure.com/api/projects/<project>``.
-    """
-
-    byo_model: str
-    project_endpoint: str
-
 
 def _to_responses_input(messages: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
     """Map chat-completions messages ({role, content}) to Responses API input items."""
@@ -61,6 +45,20 @@ def _map_params(kwargs: Dict[str, Any]) -> Dict[str, Any]:
     return mapped
 
 
+class _Usage:
+    """chat.completions-shaped usage view over a Responses API usage object.
+
+    The Responses API reports ``input_tokens`` / ``output_tokens`` / ``total_tokens``; judge/grader
+    code (and the prompty response formatter) expects ``prompt_tokens`` / ``completion_tokens`` /
+    ``total_tokens``. This adapts the former to the latter.
+    """
+
+    def __init__(self, response_usage: Any) -> None:
+        self.prompt_tokens = getattr(response_usage, "input_tokens", 0) or 0
+        self.completion_tokens = getattr(response_usage, "output_tokens", 0) or 0
+        self.total_tokens = getattr(response_usage, "total_tokens", 0) or 0
+
+
 class _ChatMessage:
     def __init__(self, content: str) -> None:
         self.role = "assistant"
@@ -81,54 +79,62 @@ class _ChatCompletion:
     def __init__(self, response: Any) -> None:
         self.id = getattr(response, "id", None)
         self.model = getattr(response, "model", None)
-        self.usage = getattr(response, "usage", None)
+        _usage = getattr(response, "usage", None)
+        self.usage = _Usage(_usage) if _usage is not None else None
         self.object = "chat.completion"
         self.created = int(time.time())
         self.choices = [_Choice(getattr(response, "output_text", "") or "")]
 
 
-class _ChatCompletions:
-    def __init__(self, owner: "ByoProjectResponsesClient") -> None:
+class _AsyncChatCompletions:
+    def __init__(self, owner: "AsyncByoProjectResponsesClient") -> None:
         self._owner = owner
 
-    def create(self, *, model: Optional[str] = None, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> _ChatCompletion:
+    async def create(
+        self, *, model: Optional[str] = None, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any
+    ) -> _ChatCompletion:
         # ``model`` from the caller is ignored — the BYO model is fixed by the shim's config.
-        response = self._owner._responses_create(messages=messages, **kwargs)
+        response = await self._owner._responses_create(messages=messages, **kwargs)
         return _ChatCompletion(response)
 
 
-class _Chat:
-    def __init__(self, owner: "ByoProjectResponsesClient") -> None:
-        self.completions = _ChatCompletions(owner)
+class _AsyncChat:
+    def __init__(self, owner: "AsyncByoProjectResponsesClient") -> None:
+        self.completions = _AsyncChatCompletions(owner)
 
 
-class ByoProjectResponsesClient:
-    """OpenAI-compatible shim that routes ``chat.completions.create()`` to the Foundry
-    project Responses API for an admin-connected (BYO) model.
+class AsyncByoProjectResponsesClient:
+    """Async OpenAI-compatible shim that routes ``chat.completions.create()`` to the Foundry project
+    Responses API for an admin-connected (BYO) model.
 
-    Judge/grader code that calls ``client.chat.completions.create(model=..., messages=...)``
-    works unchanged; the request is served by the platform, which resolves the connection.
-    A ``responses`` passthrough is also exposed for callers that use the Responses API directly.
+    Used by the prompty judge path (LLM-as-a-judge evaluators such as coherence, relevance, fluency,
+    groundedness, etc.), whose async runner calls ``client.with_options(...).chat.completions.create(...)``.
+    Those calls are served by the platform, which resolves the connection and every auth type
+    (API key / managed identity / OAuth2), so evaluator code is unchanged.
     """
 
-    def __init__(self, byo_model: str, project_endpoint: str, credential: TokenCredential) -> None:
+    def __init__(self, byo_model: str, project_endpoint: str, credential: Any) -> None:
         self._byo_model = byo_model
         self._project_endpoint = project_endpoint
         self._credential = credential
         self._client: Any = None
-        self.chat = _Chat(self)
+        self.chat = _AsyncChat(self)
+
+    def with_options(self, **_kwargs: Any) -> "AsyncByoProjectResponsesClient":
+        # The prompty runner drives its own retry/timeout loop, so there is nothing to reconfigure here.
+        return self
 
     def _openai(self) -> Any:
         if self._client is None:
-            from azure.ai.projects import AIProjectClient
+            from azure.ai.projects.aio import AIProjectClient
 
             self._client = AIProjectClient(
                 endpoint=self._project_endpoint, credential=self._credential
             ).get_openai_client()
         return self._client
 
-    def _responses_create(self, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> Any:
-        return self._openai().responses.create(
+    async def _responses_create(self, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> Any:
+        return await self._openai().responses.create(
             model=self._byo_model,
             input=_to_responses_input(messages),
             **_map_params(kwargs),
@@ -142,16 +148,3 @@ def responses(self) -> Any:
 def is_byo_model_config(model_config: Dict[str, Any]) -> bool:
     """Return True if the model configuration references an admin-connected (BYO) model."""
     return bool(model_config) and bool(model_config.get("byo_model"))
-
-
-def build_byo_judge_client(model_config: Dict[str, Any], credential: TokenCredential) -> ByoProjectResponsesClient:
-    """Build a chat.completions-compatible client for an admin-connected (BYO) judge model."""
-    if not model_config.get("byo_model") or not model_config.get("project_endpoint"):
-        raise ValueError("BYOProjectModelConfiguration requires both 'byo_model' and 'project_endpoint'.")
-    if credential is None:
-        raise ValueError("A TokenCredential is required to call the project Responses API for BYO judge models.")
-    return ByoProjectResponsesClient(
-        byo_model=model_config["byo_model"],
-        project_endpoint=model_config["project_endpoint"],
-        credential=credential,
-    )
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
index b54677cf1b13..14e02beebc2e 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
@@ -310,6 +310,13 @@ def validate_azure_ai_project(o: object) -> AzureAIProject:
 
 
 def validate_model_config(config: dict) -> Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration]:
+    # Admin-connected (BYO) judge models are referenced as "connection/deployment" and served via the
+    # Foundry project Responses API. Their config intentionally omits azure_endpoint/azure_deployment,
+    # so bypass the AzureOpenAI/OpenAI TypedDict validation and let the prompty layer route them.
+    from azure.ai.evaluation._byo_judge import is_byo_model_config
+
+    if is_byo_model_config(config):
+        return cast(AzureOpenAIModelConfiguration, config)
     try:
         return _validate_typed_dict(config, AzureOpenAIModelConfiguration)
     except TypeError:
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
index 217514bf5f2e..8c2780d0ee43 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
@@ -25,6 +25,7 @@
     WrappedOpenAIError,
 )
 from azure.ai.evaluation._legacy.prompty._connection import AzureOpenAIConnection, Connection, OpenAIConnection
+from azure.ai.evaluation._byo_judge import AsyncByoProjectResponsesClient, is_byo_model_config
 from azure.ai.evaluation._legacy.prompty._yaml_utils import load_yaml_string
 from azure.ai.evaluation._legacy.prompty._utils import (
     dataclass_from_dict,
@@ -279,7 +280,8 @@ async def __call__(  # pylint: disable=docstring-keyword-should-match-keyword-on
         """
 
         inputs = self._resolve_inputs(kwargs)
-        connection = Connection.parse_from_config(self._model.configuration)
+        configuration = self._model.configuration
+        is_byo = is_byo_model_config(configuration)
         messages = build_messages(prompt=self._template, working_dir=self.path.parent, **inputs)
         params = prepare_open_ai_request_params(self._model, messages)
 
@@ -293,31 +295,42 @@ async def __call__(  # pylint: disable=docstring-keyword-should-match-keyword-on
 
         default_headers = {"User-Agent": UserAgentSingleton().value}
 
-        api_client: Union[AsyncAzureOpenAI, AsyncOpenAI]
-        if isinstance(connection, AzureOpenAIConnection):
-            api_client = AsyncAzureOpenAI(
-                azure_endpoint=connection.azure_endpoint,
-                api_key=connection.api_key,
-                azure_deployment=connection.azure_deployment,
-                api_version=connection.api_version,
-                max_retries=max_retries,
-                azure_ad_token_provider=(
-                    self.get_token_provider(self._token_credential) if not connection.api_key else None
-                ),
-                default_headers=default_headers,
-            )
-        elif isinstance(connection, OpenAIConnection):
-            api_client = AsyncOpenAI(
-                base_url=connection.base_url,
-                api_key=connection.api_key,
-                organization=connection.organization,
-                max_retries=max_retries,
-                default_headers=default_headers,
+        api_client: Union[AsyncAzureOpenAI, AsyncOpenAI, AsyncByoProjectResponsesClient]
+        if is_byo:
+            # Admin-connected (BYO) judge model referenced as "connection/deployment": route the
+            # chat.completions call through the Foundry project Responses API. The platform resolves
+            # the connection and every auth type (API key / managed identity / OAuth2).
+            api_client = AsyncByoProjectResponsesClient(
+                byo_model=configuration["byo_model"],
+                project_endpoint=configuration["project_endpoint"],
+                credential=self._token_credential,
             )
         else:
-            raise NotSupportedError(
-                f"'{type(connection).__name__}' is not a supported connection type.", target=ErrorTarget.EVAL_RUN
-            )
+            connection = Connection.parse_from_config(configuration)
+            if isinstance(connection, AzureOpenAIConnection):
+                api_client = AsyncAzureOpenAI(
+                    azure_endpoint=connection.azure_endpoint,
+                    api_key=connection.api_key,
+                    azure_deployment=connection.azure_deployment,
+                    api_version=connection.api_version,
+                    max_retries=max_retries,
+                    azure_ad_token_provider=(
+                        self.get_token_provider(self._token_credential) if not connection.api_key else None
+                    ),
+                    default_headers=default_headers,
+                )
+            elif isinstance(connection, OpenAIConnection):
+                api_client = AsyncOpenAI(
+                    base_url=connection.base_url,
+                    api_key=connection.api_key,
+                    organization=connection.organization,
+                    max_retries=max_retries,
+                    default_headers=default_headers,
+                )
+            else:
+                raise NotSupportedError(
+                    f"'{type(connection).__name__}' is not a supported connection type.", target=ErrorTarget.EVAL_RUN
+                )
 
         response: OpenAIChatResponseType = await self._send_with_retries(
             api_client=api_client,
@@ -350,7 +363,7 @@ def render(  # pylint: disable=docstring-keyword-should-match-keyword-only
 
     async def _send_with_retries(
         self,
-        api_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
+        api_client: Union[AsyncAzureOpenAI, AsyncOpenAI, AsyncByoProjectResponsesClient],
         params: Mapping[str, Any],
         timeout: Optional[float],
         max_retries: int = 10,
@@ -358,7 +371,7 @@ async def _send_with_retries(
     ) -> OpenAIChatResponseType:
         """Send the request with retries.
 
-        :param Union[AsyncAzureOpenAI, AsyncOpenAI] api_client: The OpenAI client.
+        :param Union[AsyncAzureOpenAI, AsyncOpenAI, AsyncByoProjectResponsesClient] api_client: The OpenAI client.
         :param Mapping[str, Any] params: The request parameters.
         :param Optional[float] timeout: The timeout for the request.
         :param int max_retries: The maximum number of retries.
@@ -368,7 +381,9 @@ async def _send_with_retries(
         """
 
         client_name: str = api_client.__class__.__name__
-        client: Union[AsyncAzureOpenAI, AsyncOpenAI] = api_client.with_options(timeout=timeout or NotGiven())
+        client: Union[AsyncAzureOpenAI, AsyncOpenAI, AsyncByoProjectResponsesClient] = api_client.with_options(
+            timeout=timeout or NotGiven()
+        )
 
         entity_retries: List[int] = [0]
         should_retry: bool = True
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index f9fadd16e945..a143572d3944 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -1,17 +1,18 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-"""Unit tests for the admin-connected (BYO) judge-model prototype."""
-from unittest.mock import MagicMock, patch
+"""Unit tests for admin-connected (BYO) judge-model support in the prompty judge path."""
+import asyncio
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
 from azure.ai.evaluation._byo_judge import (
-    ByoProjectResponsesClient,
-    build_byo_judge_client,
+    AsyncByoProjectResponsesClient,
     is_byo_model_config,
     _to_responses_input,
     _map_params,
+    _Usage,
 )
 
 
@@ -34,78 +35,85 @@ def test_map_params_curates_and_renames(self):
         }
 
 
-class TestByoJudgeClient:
-    @patch("azure.ai.projects.AIProjectClient")
-    def test_chat_completions_routes_to_responses(self, mock_aipc):
+class TestUsageAdapter:
+    def test_responses_usage_mapped_to_chat_shape(self):
+        usage = _Usage(MagicMock(input_tokens=5, output_tokens=7, total_tokens=12))
+        assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (5, 7, 12)
+
+    def test_missing_fields_default_to_zero(self):
+        usage = _Usage(object())
+        assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (0, 0, 0)
+
+
+class TestAsyncByoProjectResponsesClient:
+    """The prompty judge path (coherence/relevance/fluency/... LLM-as-a-judge evaluators)."""
+
+    @patch("azure.ai.projects.aio.AIProjectClient")
+    def test_async_chat_completions_routes_to_responses(self, mock_aipc):
         resp = MagicMock()
-        resp.output_text = "judge says 5"
-        resp.usage = MagicMock()
-        resp.id = "resp_1"
+        resp.output_text = "coherent: 5"
+        resp.usage = MagicMock(input_tokens=12, output_tokens=3, total_tokens=15)
+        resp.id = "resp_2"
         resp.model = "my-conn/gpt-4o"
         oai = MagicMock()
-        oai.responses.create.return_value = resp
+        oai.responses.create = AsyncMock(return_value=resp)
         mock_aipc.return_value.get_openai_client.return_value = oai
 
-        client = build_byo_judge_client(
-            {"byo_model": "my-conn/gpt-4o", "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p1"},
+        client = AsyncByoProjectResponsesClient(
+            byo_model="my-conn/gpt-4o",
+            project_endpoint="https://acct.services.ai.azure.com/api/projects/p1",
             credential=MagicMock(),
         )
-        # Judge/grader code calls chat.completions.create unchanged.
-        result = client.chat.completions.create(
-            model="ignored",
-            messages=[{"role": "user", "content": "score this 1-5"}],
-            temperature=0.0,
-            max_tokens=10,
+        # The prompty runner calls with_options(...).chat.completions.create(...).
+        result = asyncio.run(
+            client.with_options(timeout=30).chat.completions.create(
+                model="ignored",
+                messages=[{"role": "user", "content": "rate coherence"}],
+                temperature=0.0,
+                max_tokens=800,
+            )
         )
 
-        # chat.completions-shaped result over the Responses output.
-        assert result.choices[0].message.content == "judge says 5"
+        # chat.completions-shaped result the prompty response formatter expects.
+        assert result.choices[0].message.content == "coherent: 5"
         assert result.choices[0].message.role == "assistant"
+        assert result.choices[0].finish_reason == "stop"
+        assert result.usage.prompt_tokens == 12
+        assert result.usage.completion_tokens == 3
+        assert result.usage.total_tokens == 15
+        assert result.model == "my-conn/gpt-4o"
 
-        # Underlying call is responses.create with the BYO model + mapped input/params.
+        # Underlying call is the project Responses API with the BYO model + mapped input/params.
         mock_aipc.assert_called_once()
         _, pkwargs = mock_aipc.call_args
         assert pkwargs["endpoint"] == "https://acct.services.ai.azure.com/api/projects/p1"
         _, rkwargs = oai.responses.create.call_args
         assert rkwargs["model"] == "my-conn/gpt-4o"
-        assert rkwargs["input"] == [{"type": "message", "role": "user", "content": "score this 1-5"}]
+        assert rkwargs["input"] == [{"type": "message", "role": "user", "content": "rate coherence"}]
         assert rkwargs["temperature"] == 0.0
-        assert rkwargs["max_output_tokens"] == 10
-
-    def test_build_requires_project_endpoint(self):
-        with pytest.raises(ValueError):
-            build_byo_judge_client({"byo_model": "c/d"}, credential=MagicMock())
+        assert rkwargs["max_output_tokens"] == 800
 
-    def test_build_requires_credential(self):
-        with pytest.raises(ValueError):
-            build_byo_judge_client({"byo_model": "c/d", "project_endpoint": "https://x"}, credential=None)
+    def test_with_options_returns_self(self):
+        client = AsyncByoProjectResponsesClient(
+            "c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock()
+        )
+        assert client.with_options(timeout=5) is client
 
 
-class TestAzureOpenAIGraderByo:
-    @patch("azure.ai.projects.AIProjectClient")
-    def test_grader_get_client_returns_shim_for_byo(self, _mock_aipc):
-        from azure.ai.evaluation._aoai.aoai_grader import AzureOpenAIGrader
+class TestValidateModelConfigByo:
+    def test_byo_config_passes_through_validation(self):
+        from azure.ai.evaluation._common.utils import validate_model_config
 
-        grader = AzureOpenAIGrader(
-            model_config={
-                "byo_model": "my-conn/gpt-4o",
-                "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p1",
-            },
-            grader_config={},
-            credential=MagicMock(),
-        )
-        assert isinstance(grader.get_client(), ByoProjectResponsesClient)
+        cfg = {
+            "byo_model": "my-conn/gpt-4o",
+            "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p1",
+        }
+        # BYO configs intentionally omit azure_endpoint/azure_deployment and must not be rejected.
+        assert validate_model_config(cfg) is cfg
 
-    def test_grader_byo_requires_credential(self):
-        from azure.ai.evaluation._aoai.aoai_grader import AzureOpenAIGrader
+    def test_non_byo_invalid_config_still_raises(self):
+        from azure.ai.evaluation._common.utils import validate_model_config
         from azure.ai.evaluation._exceptions import EvaluationException
 
         with pytest.raises(EvaluationException):
-            AzureOpenAIGrader(
-                model_config={
-                    "byo_model": "my-conn/gpt-4o",
-                    "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p1",
-                },
-                grader_config={},
-                credential=None,
-            )
+            validate_model_config({"not": "a valid model config"})

From c84d4c2721bc9e6e761d53f58b80ead7c0e82d6d Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Wed, 15 Jul 2026 12:06:28 -0400
Subject: [PATCH 03/26] Forward response_format as text.format in BYO judge
 Responses shim

The BYO prompty-judge shim dropped the chat.completions `response_format`
param, so admin-connected judge models were not held to JSON output mode
(coherence/relevance/etc. prompts request json_object). The Responses API
exposes the same capability under `text.format`, so translate it:

  response_format={'type':'json_object'}  -> text={'format':{'type':'json_object'}}
  response_format json_schema (nested)    -> text.format json_schema (flattened)

This restores enforced-JSON parity with the AzureOpenAI path and prevents
json.loads failures when a BYO model would otherwise drift to prose output.

Added 6 unit tests for _map_response_format and the _map_params wiring.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_byo_judge.py         | 39 +++++++++++++++++
 .../tests/unittests/test_byo_judge.py         | 42 +++++++++++++++++++
 2 files changed, 81 insertions(+)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index 423ab52526f2..780c8820390b 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -31,6 +31,41 @@ def _to_responses_input(messages: Optional[List[Dict[str, Any]]]) -> List[Dict[s
     return items
 
 
+def _map_response_format(response_format: Any) -> Optional[Dict[str, Any]]:
+    """Translate a chat-completions ``response_format`` into a Responses API ``text.format`` value.
+
+    The Responses API exposes the same JSON-output capability as chat.completions, but under
+    ``text.format`` instead of ``response_format``:
+
+    ================================================  ===========================================
+    chat.completions ``response_format``              Responses API ``text.format``
+    ================================================  ===========================================
+    ``{"type": "text"}``                              ``{"type": "text"}``
+    ``{"type": "json_object"}``                       ``{"type": "json_object"}``
+    ``{"type": "json_schema",``                       ``{"type": "json_schema", "name": ...,``
+    ``  "json_schema": {"name", "schema", ...}}``     ``  "schema": ..., "strict": ...}``
+    ================================================  ===========================================
+
+    Note the json_schema shape difference: chat.completions **nests** ``name``/``schema``/``strict``
+    under a ``json_schema`` key, whereas the Responses API **flattens** them directly under
+    ``format``. Returns ``None`` for anything unrecognized so the param is simply omitted.
+    """
+    if not isinstance(response_format, dict):
+        return None
+    rf_type = response_format.get("type")
+    if rf_type in ("text", "json_object"):
+        return {"type": rf_type}
+    if rf_type == "json_schema":
+        # chat.completions nests the schema spec under "json_schema"; the Responses API flattens it.
+        schema_spec = response_format.get("json_schema", response_format)
+        fmt: Dict[str, Any] = {"type": "json_schema"}
+        for key in ("name", "schema", "strict", "description"):
+            if key in schema_spec:
+                fmt[key] = schema_spec[key]
+        return fmt
+    return None
+
+
 def _map_params(kwargs: Dict[str, Any]) -> Dict[str, Any]:
     """Map a curated set of chat-completions sampling params to Responses API params."""
     mapped: Dict[str, Any] = {}
@@ -42,6 +77,10 @@ def _map_params(kwargs: Dict[str, Any]) -> Dict[str, Any]:
         if key in kwargs:
             mapped["max_output_tokens"] = kwargs[key]
             break
+    if "response_format" in kwargs:
+        text_format = _map_response_format(kwargs["response_format"])
+        if text_format is not None:
+            mapped["text"] = {"format": text_format}
     return mapped
 
 
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index a143572d3944..d17c609faf81 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -12,6 +12,7 @@
     is_byo_model_config,
     _to_responses_input,
     _map_params,
+    _map_response_format,
     _Usage,
 )
 
@@ -35,6 +36,47 @@ def test_map_params_curates_and_renames(self):
         }
 
 
+class TestResponseFormatMapping:
+    """chat.completions ``response_format`` -> Responses API ``text.format`` (JSON output parity)."""
+
+    def test_json_object_passthrough(self):
+        assert _map_response_format({"type": "json_object"}) == {"type": "json_object"}
+
+    def test_text_passthrough(self):
+        assert _map_response_format({"type": "text"}) == {"type": "text"}
+
+    def test_json_schema_is_flattened(self):
+        # chat.completions nests name/schema/strict under "json_schema"; Responses flattens them.
+        rf = {
+            "type": "json_schema",
+            "json_schema": {
+                "name": "coherence",
+                "schema": {"type": "object", "properties": {"score": {"type": "integer"}}},
+                "strict": True,
+            },
+        }
+        assert _map_response_format(rf) == {
+            "type": "json_schema",
+            "name": "coherence",
+            "schema": {"type": "object", "properties": {"score": {"type": "integer"}}},
+            "strict": True,
+        }
+
+    def test_unknown_or_non_dict_returns_none(self):
+        assert _map_response_format({"type": "weird"}) is None
+        assert _map_response_format(None) is None
+        assert _map_response_format("json_object") is None
+
+    def test_map_params_wires_response_format_into_text(self):
+        # The prompty runner passes response_format={"type": "json_object"}; it must reach text.format.
+        mapped = _map_params({"temperature": 0.0, "response_format": {"type": "json_object"}})
+        assert mapped["text"] == {"format": {"type": "json_object"}}
+        assert mapped["temperature"] == 0.0
+
+    def test_map_params_omits_text_for_unrecognized_response_format(self):
+        assert "text" not in _map_params({"response_format": {"type": "weird"}})
+
+
 class TestUsageAdapter:
     def test_responses_usage_mapped_to_chat_shape(self):
         usage = _Usage(MagicMock(input_tokens=5, output_tokens=7, total_tokens=12))

From ba92b2856c26dc5ee5bffff64133eeb1275194b1 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Wed, 15 Jul 2026 17:34:08 -0400
Subject: [PATCH 04/26] Fix black formatting in test_byo_judge.py

Collapse a multi-line AsyncByoProjectResponsesClient constructor call onto one
line (fits within the 120-char limit) to satisfy the azpysdk black check.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure-ai-evaluation/tests/unittests/test_byo_judge.py     | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index d17c609faf81..639f10483449 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -136,9 +136,7 @@ def test_async_chat_completions_routes_to_responses(self, mock_aipc):
         assert rkwargs["max_output_tokens"] == 800
 
     def test_with_options_returns_self(self):
-        client = AsyncByoProjectResponsesClient(
-            "c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock()
-        )
+        client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
         assert client.with_options(timeout=5) is client
 
 

From 3f914aba170138ce188150c79956ac9d3486e119 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Thu, 16 Jul 2026 11:41:49 -0400
Subject: [PATCH 05/26] Fix: require both BYO markers (byo_model +
 project_endpoint)

is_byo_model_config returned True on byo_model alone, but the prompty branch
reads configuration['project_endpoint'] -> a partial config raised KeyError
instead of taking the normal (non-BYO) path. Require both markers, matching the
control-plane contract (BYO active iff both present). Adds coverage for the
byo_model-only and project_endpoint-only cases.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_byo_judge.py                  | 10 ++++++++--
 .../tests/unittests/test_byo_judge.py                  |  4 ++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index 780c8820390b..b896ed19b128 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -185,5 +185,11 @@ def responses(self) -> Any:
 
 
 def is_byo_model_config(model_config: Dict[str, Any]) -> bool:
-    """Return True if the model configuration references an admin-connected (BYO) model."""
-    return bool(model_config) and bool(model_config.get("byo_model"))
+    """Return True if the model configuration references an admin-connected (BYO) model.
+
+    Requires **both** markers — ``byo_model`` (``"connection/deployment"``) and ``project_endpoint``
+    — because the prompty branch needs the project endpoint to route the call. Requiring both matches
+    the control-plane contract (BYO is active iff both markers are present) and avoids a ``KeyError``
+    when only a partial config is supplied.
+    """
+    return bool(model_config) and bool(model_config.get("byo_model")) and bool(model_config.get("project_endpoint"))
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index 639f10483449..16d29771b120 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -22,6 +22,10 @@ def test_is_byo_model_config(self):
         assert is_byo_model_config({"byo_model": "c/d", "project_endpoint": "https://x"})
         assert not is_byo_model_config({"azure_endpoint": "https://x"})
         assert not is_byo_model_config({})
+        # Both markers are required — byo_model alone must not activate the BYO path
+        # (the prompty branch needs project_endpoint to route, else it would KeyError).
+        assert not is_byo_model_config({"byo_model": "c/d"})
+        assert not is_byo_model_config({"project_endpoint": "https://x"})
 
     def test_to_responses_input(self):
         assert _to_responses_input([{"role": "user", "content": "hi"}]) == [

From 6a11938b0da19f5ae453c7cdea2415de1df431bb Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Thu, 16 Jul 2026 11:44:09 -0400
Subject: [PATCH 06/26] Fix: surface real finish_reason from the Responses
 result

The shim hardcoded finish_reason='stop', so a judge output truncated at
max_output_tokens was reported as complete and then failed json.loads downstream
with no signal. Map the Responses status/incomplete_details.reason to a
chat.completions finish_reason (max_output_tokens -> 'length', else 'stop').

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_byo_judge.py         | 24 ++++++++++++++--
 .../tests/unittests/test_byo_judge.py         | 28 +++++++++++++++++++
 2 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index b896ed19b128..b28120c78946 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -105,11 +105,29 @@ def __init__(self, content: str) -> None:
         self.tool_calls = None
 
 
+def _finish_reason(response: Any) -> str:
+    """Map a Responses result's status to a chat-completions ``finish_reason``.
+
+    A Responses call that stops early reports ``status="incomplete"`` with an
+    ``incomplete_details.reason`` (e.g. ``"max_output_tokens"`` / ``"content_filter"``).
+    Chat.completions callers expect ``"length"`` for truncation; surfacing it lets the prompty
+    formatter distinguish a truncated (invalid-JSON-prone) judge output from a complete one instead
+    of always seeing ``"stop"``.
+    """
+    if getattr(response, "status", None) != "incomplete":
+        return "stop"
+    details = getattr(response, "incomplete_details", None)
+    reason = getattr(details, "reason", None) if details is not None else None
+    if reason == "max_output_tokens":
+        return "length"
+    return reason or "stop"
+
+
 class _Choice:
-    def __init__(self, content: str) -> None:
+    def __init__(self, content: str, finish_reason: str = "stop") -> None:
         self.index = 0
         self.message = _ChatMessage(content)
-        self.finish_reason = "stop"
+        self.finish_reason = finish_reason
 
 
 class _ChatCompletion:
@@ -122,7 +140,7 @@ def __init__(self, response: Any) -> None:
         self.usage = _Usage(_usage) if _usage is not None else None
         self.object = "chat.completion"
         self.created = int(time.time())
-        self.choices = [_Choice(getattr(response, "output_text", "") or "")]
+        self.choices = [_Choice(getattr(response, "output_text", "") or "", _finish_reason(response))]
 
 
 class _AsyncChatCompletions:
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index 16d29771b120..942b9a3a00b0 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -3,6 +3,7 @@
 # ---------------------------------------------------------
 """Unit tests for admin-connected (BYO) judge-model support in the prompty judge path."""
 import asyncio
+from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
@@ -13,6 +14,7 @@
     _to_responses_input,
     _map_params,
     _map_response_format,
+    _ChatCompletion,
     _Usage,
 )
 
@@ -91,6 +93,32 @@ def test_missing_fields_default_to_zero(self):
         assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (0, 0, 0)
 
 
+class TestFinishReason:
+    """A Responses result maps status -> chat.completions finish_reason (truncation detection)."""
+
+    @staticmethod
+    def _resp(**kwargs):
+        kwargs.setdefault("output_text", "ok")
+        kwargs.setdefault("usage", None)
+        kwargs.setdefault("id", "r")
+        kwargs.setdefault("model", "m")
+        return SimpleNamespace(**kwargs)
+
+    def test_completed_is_stop(self):
+        assert _ChatCompletion(self._resp(status="completed")).choices[0].finish_reason == "stop"
+
+    def test_missing_status_defaults_to_stop(self):
+        assert _ChatCompletion(self._resp()).choices[0].finish_reason == "stop"
+
+    def test_truncation_is_length(self):
+        resp = self._resp(status="incomplete", incomplete_details=SimpleNamespace(reason="max_output_tokens"))
+        assert _ChatCompletion(resp).choices[0].finish_reason == "length"
+
+    def test_other_incomplete_reason_passes_through(self):
+        resp = self._resp(status="incomplete", incomplete_details=SimpleNamespace(reason="content_filter"))
+        assert _ChatCompletion(resp).choices[0].finish_reason == "content_filter"
+
+
 class TestAsyncByoProjectResponsesClient:
     """The prompty judge path (coherence/relevance/fluency/... LLM-as-a-judge evaluators)."""
 

From 8ae470adcf8fcd519b7000ef0b61f9fade980b1d Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Thu, 16 Jul 2026 11:46:07 -0400
Subject: [PATCH 07/26] Fix: honor the per-request timeout in the BYO shim

with_options(timeout=...) was a no-op, so the prompty runner's per-request
timeout was dropped and a hung Responses call could block on openai's default
(~600s). Capture a concrete numeric timeout and forward it to responses.create
(openai's NotGiven sentinel, used when no timeout is set, is ignored).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_byo_judge.py                 | 11 +++++++++--
 .../tests/unittests/test_byo_judge.py                 |  9 +++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index b28120c78946..845ac8c6af50 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -175,10 +175,15 @@ def __init__(self, byo_model: str, project_endpoint: str, credential: Any) -> No
         self._project_endpoint = project_endpoint
         self._credential = credential
         self._client: Any = None
+        self._timeout: Optional[float] = None
         self.chat = _AsyncChat(self)
 
-    def with_options(self, **_kwargs: Any) -> "AsyncByoProjectResponsesClient":
-        # The prompty runner drives its own retry/timeout loop, so there is nothing to reconfigure here.
+    def with_options(self, *, timeout: Any = None, **_kwargs: Any) -> "AsyncByoProjectResponsesClient":
+        # The prompty runner sets a per-request timeout via with_options(timeout=...). Capture a
+        # concrete numeric value so it reaches responses.create; openai's ``NotGiven`` sentinel (used
+        # when no timeout is configured) is not numeric and is therefore ignored.
+        if isinstance(timeout, (int, float)) and not isinstance(timeout, bool):
+            self._timeout = float(timeout)
         return self
 
     def _openai(self) -> Any:
@@ -191,10 +196,12 @@ def _openai(self) -> Any:
         return self._client
 
     async def _responses_create(self, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> Any:
+        extra: Dict[str, Any] = {"timeout": self._timeout} if self._timeout is not None else {}
         return await self._openai().responses.create(
             model=self._byo_model,
             input=_to_responses_input(messages),
             **_map_params(kwargs),
+            **extra,
         )
 
     @property
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index 942b9a3a00b0..7f663c46e6e9 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -166,6 +166,15 @@ def test_async_chat_completions_routes_to_responses(self, mock_aipc):
         assert rkwargs["input"] == [{"type": "message", "role": "user", "content": "rate coherence"}]
         assert rkwargs["temperature"] == 0.0
         assert rkwargs["max_output_tokens"] == 800
+        # The per-request timeout set via with_options(timeout=30) reaches responses.create.
+        assert rkwargs["timeout"] == 30
+
+    def test_non_numeric_timeout_is_not_forwarded(self):
+        # openai passes a NotGiven() sentinel when no timeout is configured; the shim must ignore it
+        # (only a concrete numeric timeout is forwarded to responses.create).
+        client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
+        client.with_options(timeout=object())
+        assert client._timeout is None
 
     def test_with_options_returns_self(self):
         client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())

From 1c664e5d846e7c69736540a98b455ab35701c8a0 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Thu, 16 Jul 2026 11:47:42 -0400
Subject: [PATCH 08/26] Fix: compute total_tokens fallback in the usage adapter

_Usage reported total_tokens=0 when the Responses usage omits it. Fall back to
prompt_tokens + completion_tokens so token telemetry stays correct.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py    | 5 ++++-
 .../azure-ai-evaluation/tests/unittests/test_byo_judge.py    | 5 +++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index 845ac8c6af50..6eaa0207a326 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -95,7 +95,10 @@ class _Usage:
     def __init__(self, response_usage: Any) -> None:
         self.prompt_tokens = getattr(response_usage, "input_tokens", 0) or 0
         self.completion_tokens = getattr(response_usage, "output_tokens", 0) or 0
-        self.total_tokens = getattr(response_usage, "total_tokens", 0) or 0
+        # Fall back to prompt + completion when the Responses usage omits total_tokens.
+        self.total_tokens = (getattr(response_usage, "total_tokens", 0) or 0) or (
+            self.prompt_tokens + self.completion_tokens
+        )
 
 
 class _ChatMessage:
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index 7f663c46e6e9..e10ea81f41a8 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -92,6 +92,11 @@ def test_missing_fields_default_to_zero(self):
         usage = _Usage(object())
         assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (0, 0, 0)
 
+    def test_total_tokens_falls_back_to_prompt_plus_completion(self):
+        # Responses usage without total_tokens -> compute it from prompt + completion.
+        usage = _Usage(SimpleNamespace(input_tokens=5, output_tokens=7))
+        assert usage.total_tokens == 12
+
 
 class TestFinishReason:
     """A Responses result maps status -> chat.completions finish_reason (truncation detection)."""

From 9ce64ff1391a81e083090a589703788f80a99f6e Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Thu, 16 Jul 2026 11:50:04 -0400
Subject: [PATCH 09/26] Fix: await-safe project client creation; drop dead
 responses property

get_openai_client() is synchronous in some azure-ai-projects versions and a
coroutine in others; the shim assumed sync, so a version bump would assign a
coroutine to self._client and break responses.create. Create the client in an
async helper that awaits it when needed. Also removes the unused 'responses'
property (leftover from the removed grader path).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_byo_judge.py         | 19 ++++++++++---------
 .../tests/unittests/test_byo_judge.py         | 17 +++++++++++++++++
 2 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index 6eaa0207a326..0185e5b4d6ed 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -13,6 +13,7 @@
 **shim** that routes those calls to the project Responses API for a BYO model, so evaluator
 code can use admin-connected connections **without any change to its calling code**.
 """
+import inspect
 import time
 from typing import Any, Dict, List, Optional
 
@@ -189,28 +190,28 @@ def with_options(self, *, timeout: Any = None, **_kwargs: Any) -> "AsyncByoProje
             self._timeout = float(timeout)
         return self
 
-    def _openai(self) -> Any:
+    async def _ensure_client(self) -> Any:
         if self._client is None:
             from azure.ai.projects.aio import AIProjectClient
 
-            self._client = AIProjectClient(
-                endpoint=self._project_endpoint, credential=self._credential
-            ).get_openai_client()
+            client = AIProjectClient(endpoint=self._project_endpoint, credential=self._credential).get_openai_client()
+            # ``get_openai_client()`` is synchronous in some azure-ai-projects versions and a coroutine
+            # in others; await it only when needed so the shim works across both.
+            if inspect.isawaitable(client):
+                client = await client
+            self._client = client
         return self._client
 
     async def _responses_create(self, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> Any:
+        client = await self._ensure_client()
         extra: Dict[str, Any] = {"timeout": self._timeout} if self._timeout is not None else {}
-        return await self._openai().responses.create(
+        return await client.responses.create(
             model=self._byo_model,
             input=_to_responses_input(messages),
             **_map_params(kwargs),
             **extra,
         )
 
-    @property
-    def responses(self) -> Any:
-        return self._openai().responses
-
 
 def is_byo_model_config(model_config: Dict[str, Any]) -> bool:
     """Return True if the model configuration references an admin-connected (BYO) model.
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index e10ea81f41a8..01ca7310912a 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -181,6 +181,23 @@ def test_non_numeric_timeout_is_not_forwarded(self):
         client.with_options(timeout=object())
         assert client._timeout is None
 
+    @patch("azure.ai.projects.aio.AIProjectClient")
+    def test_awaits_coroutine_get_openai_client(self, mock_aipc):
+        # In some azure-ai-projects versions get_openai_client() is a coroutine; the shim must await it.
+        resp = MagicMock(output_text="ok", usage=None, id="r", model="m", status="completed")
+        oai = MagicMock()
+        oai.responses.create = AsyncMock(return_value=resp)
+
+        async def _coro_get_openai_client(*args, **kwargs):
+            return oai
+
+        mock_aipc.return_value.get_openai_client = _coro_get_openai_client
+
+        client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
+        result = asyncio.run(client.chat.completions.create(messages=[{"role": "user", "content": "hi"}]))
+        assert result.choices[0].message.content == "ok"
+        oai.responses.create.assert_awaited_once()
+
     def test_with_options_returns_self(self):
         client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
         assert client.with_options(timeout=5) is client

From e38edbad4d49888463047f609a86158c41816409 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Thu, 16 Jul 2026 13:12:43 -0400
Subject: [PATCH 10/26] Surface clear MissingRequiredPackage error when
 azure-ai-projects is absent

azure-ai-projects is an optional dependency (not in install_requires/extras),
so the BYO judge path's lazy `from azure.ai.projects.aio import AIProjectClient`
raised a raw ModuleNotFoundError when it was not installed. Wrap the import in
try/except and raise the SDK's MissingRequiredPackage with an actionable install
message, matching the existing optional-dependency idiom (azure-ai-inference,
promptflow, openai). Adds a unit test that simulates the missing package.

Addresses PR #48002 reviewer comment on _byo_judge.py.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_byo_judge.py              | 10 +++++++++-
 .../tests/unittests/test_byo_judge.py              | 14 ++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index 0185e5b4d6ed..7bff59b69264 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -192,7 +192,15 @@ def with_options(self, *, timeout: Any = None, **_kwargs: Any) -> "AsyncByoProje
 
     async def _ensure_client(self) -> Any:
         if self._client is None:
-            from azure.ai.projects.aio import AIProjectClient
+            try:
+                from azure.ai.projects.aio import AIProjectClient
+            except ImportError as ex:
+                from azure.ai.evaluation._legacy._adapters._errors import MissingRequiredPackage
+
+                raise MissingRequiredPackage(
+                    message="Please install the 'azure-ai-projects' package to use admin-connected "
+                    "(BYO) judge models."
+                ) from ex
 
             client = AIProjectClient(endpoint=self._project_endpoint, credential=self._credential).get_openai_client()
             # ``get_openai_client()`` is synchronous in some azure-ai-projects versions and a coroutine
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index 01ca7310912a..501bb6e70650 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -202,6 +202,20 @@ def test_with_options_returns_self(self):
         client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
         assert client.with_options(timeout=5) is client
 
+    def test_missing_azure_ai_projects_raises_clear_error(self, monkeypatch):
+        # azure-ai-projects is an optional dependency (not in install_requires). When the BYO path
+        # is used without it installed, the shim must surface a clear MissingRequiredPackage error
+        # instead of a raw ModuleNotFoundError. A None entry in sys.modules makes
+        # ``from azure.ai.projects.aio import AIProjectClient`` raise ImportError.
+        import sys
+
+        from azure.ai.evaluation._legacy._adapters._errors import MissingRequiredPackage
+
+        monkeypatch.setitem(sys.modules, "azure.ai.projects.aio", None)
+        client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
+        with pytest.raises(MissingRequiredPackage):
+            asyncio.run(client.chat.completions.create(messages=[{"role": "user", "content": "hi"}]))
+
 
 class TestValidateModelConfigByo:
     def test_byo_config_passes_through_validation(self):

From 428bd8d77f95187a8ad3f8ed1996e7f734580d2a Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Thu, 16 Jul 2026 13:14:24 -0400
Subject: [PATCH 11/26] Require BYO markers to be non-empty strings in
 is_byo_model_config

Truthiness alone accepted invalid configs such as {"byo_model": 1,
"project_endpoint": 2}: those markers are truthy, so the config was treated as
BYO, bypassed validate_model_config, and failed deep inside the project/OpenAI
client instead of producing the evaluator's normal validation error. Require
both markers to be non-empty strings before activating the BYO path. Extends the
is_byo_model_config unit test with non-string and empty-string cases.

Addresses PR #48002 reviewer comment on _byo_judge.py.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_byo_judge.py            | 16 ++++++++++++----
 .../tests/unittests/test_byo_judge.py            |  5 +++++
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index 7bff59b69264..26aa1f68325e 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -225,8 +225,16 @@ def is_byo_model_config(model_config: Dict[str, Any]) -> bool:
     """Return True if the model configuration references an admin-connected (BYO) model.
 
     Requires **both** markers — ``byo_model`` (``"connection/deployment"``) and ``project_endpoint``
-    — because the prompty branch needs the project endpoint to route the call. Requiring both matches
-    the control-plane contract (BYO is active iff both markers are present) and avoids a ``KeyError``
-    when only a partial config is supplied.
+    — to be present **non-empty strings**, because the prompty branch needs the project endpoint to
+    route the call. Requiring both matches the control-plane contract (BYO is active iff both markers
+    are present) and avoids a ``KeyError`` when only a partial config is supplied. Enforcing the string
+    type prevents truthy-but-invalid values (e.g. ``{"byo_model": 1, "project_endpoint": 2}``) from
+    bypassing the evaluator's normal model-config validation and failing deep inside the client instead.
     """
-    return bool(model_config) and bool(model_config.get("byo_model")) and bool(model_config.get("project_endpoint"))
+    if not model_config:
+        return False
+    byo_model = model_config.get("byo_model")
+    project_endpoint = model_config.get("project_endpoint")
+    return (
+        isinstance(byo_model, str) and bool(byo_model) and isinstance(project_endpoint, str) and bool(project_endpoint)
+    )
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index 501bb6e70650..b8611320df43 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -28,6 +28,11 @@ def test_is_byo_model_config(self):
         # (the prompty branch needs project_endpoint to route, else it would KeyError).
         assert not is_byo_model_config({"byo_model": "c/d"})
         assert not is_byo_model_config({"project_endpoint": "https://x"})
+        # Markers must be non-empty strings — truthy-but-invalid values (e.g. ints) must not
+        # activate BYO, else they bypass validate_model_config and fail deep inside the client.
+        assert not is_byo_model_config({"byo_model": 1, "project_endpoint": 2})
+        assert not is_byo_model_config({"byo_model": "", "project_endpoint": "https://x"})
+        assert not is_byo_model_config({"byo_model": "c/d", "project_endpoint": ""})
 
     def test_to_responses_input(self):
         assert _to_responses_input([{"role": "user", "content": "hi"}]) == [

From c1c6369acfb73c8dee5ea9f54ec93dcb151c5406 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Thu, 16 Jul 2026 18:13:08 -0400
Subject: [PATCH 12/26] Support optional additional headers in
 AsyncByoProjectResponsesClient

ACA may forward additional request headers (e.g. correlation / telemetry
headers) when running continuous evaluations. Add an optional extra_headers
parameter to AsyncByoProjectResponsesClient that is forwarded on every Responses
API call via responses.create(extra_headers=...). The headers are copied on
construction so later caller mutations do not leak into the client. AsyncPrompty
threads them from the BYO model config (configuration["extra_headers"]) so the
end-to-end path stays unchanged for callers that do not supply headers.

Adds unit tests covering header forwarding and the no-headers default.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_byo_judge.py         | 16 +++++++-
 .../ai/evaluation/_legacy/prompty/_prompty.py |  1 +
 .../tests/unittests/test_byo_judge.py         | 37 +++++++++++++++++++
 3 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index 26aa1f68325e..3cdaf2f10ecf 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -174,10 +174,18 @@ class AsyncByoProjectResponsesClient:
     (API key / managed identity / OAuth2), so evaluator code is unchanged.
     """
 
-    def __init__(self, byo_model: str, project_endpoint: str, credential: Any) -> None:
+    def __init__(
+        self,
+        byo_model: str,
+        project_endpoint: str,
+        credential: Any,
+        extra_headers: Optional[Dict[str, str]] = None,
+    ) -> None:
         self._byo_model = byo_model
         self._project_endpoint = project_endpoint
         self._credential = credential
+        # Optional headers forwarded on every Responses API call, needed when ACA runs continuous evaluations.
+        self._extra_headers: Optional[Dict[str, str]] = dict(extra_headers) if extra_headers else None
         self._client: Any = None
         self._timeout: Optional[float] = None
         self.chat = _AsyncChat(self)
@@ -212,7 +220,11 @@ async def _ensure_client(self) -> Any:
 
     async def _responses_create(self, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> Any:
         client = await self._ensure_client()
-        extra: Dict[str, Any] = {"timeout": self._timeout} if self._timeout is not None else {}
+        extra: Dict[str, Any] = {}
+        if self._timeout is not None:
+            extra["timeout"] = self._timeout
+        if self._extra_headers:
+            extra["extra_headers"] = self._extra_headers
         return await client.responses.create(
             model=self._byo_model,
             input=_to_responses_input(messages),
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
index 8c2780d0ee43..012835574841 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
@@ -304,6 +304,7 @@ async def __call__(  # pylint: disable=docstring-keyword-should-match-keyword-on
                 byo_model=configuration["byo_model"],
                 project_endpoint=configuration["project_endpoint"],
                 credential=self._token_credential,
+                extra_headers=configuration.get("extra_headers"),
             )
         else:
             connection = Connection.parse_from_config(configuration)
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index b8611320df43..cd4b35cf7e9a 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -221,6 +221,43 @@ def test_missing_azure_ai_projects_raises_clear_error(self, monkeypatch):
         with pytest.raises(MissingRequiredPackage):
             asyncio.run(client.chat.completions.create(messages=[{"role": "user", "content": "hi"}]))
 
+    @patch("azure.ai.projects.aio.AIProjectClient")
+    def test_extra_headers_forwarded_to_responses(self, mock_aipc):
+        # ACA may supply additional headers (e.g. correlation/telemetry headers) when running
+        # continuous evaluations; the shim must forward them to responses.create.
+        resp = MagicMock(output_text="ok", usage=None, id="r", model="m", status="completed")
+        oai = MagicMock()
+        oai.responses.create = AsyncMock(return_value=resp)
+        mock_aipc.return_value.get_openai_client.return_value = oai
+
+        headers = {"x-ms-client-request-id": "abc-123", "x-correlation-id": "def-456"}
+        client = AsyncByoProjectResponsesClient(
+            byo_model="c/d",
+            project_endpoint="https://acct.services.ai.azure.com/api/projects/p",
+            credential=MagicMock(),
+            extra_headers=headers,
+        )
+        asyncio.run(client.chat.completions.create(messages=[{"role": "user", "content": "hi"}]))
+
+        _, rkwargs = oai.responses.create.call_args
+        assert rkwargs["extra_headers"] == headers
+        # The shim copies the headers so later caller mutations do not leak into the client.
+        assert rkwargs["extra_headers"] is not headers
+
+    @patch("azure.ai.projects.aio.AIProjectClient")
+    def test_no_extra_headers_by_default(self, mock_aipc):
+        # When no extra headers are supplied, none are forwarded to responses.create.
+        resp = MagicMock(output_text="ok", usage=None, id="r", model="m", status="completed")
+        oai = MagicMock()
+        oai.responses.create = AsyncMock(return_value=resp)
+        mock_aipc.return_value.get_openai_client.return_value = oai
+
+        client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
+        asyncio.run(client.chat.completions.create(messages=[{"role": "user", "content": "hi"}]))
+
+        _, rkwargs = oai.responses.create.call_args
+        assert "extra_headers" not in rkwargs
+
 
 class TestValidateModelConfigByo:
     def test_byo_config_passes_through_validation(self):

From c7f6aaea72cf732ec203cac8854e456e9a81dddc Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Thu, 16 Jul 2026 18:26:48 -0400
Subject: [PATCH 13/26] Preserve server-provided created timestamp in BYO
 chat-completion shim

_ChatCompletion always set `created` to the local wall-clock time, discarding
any server `created` timestamp on the Responses result. Preserve the server
value when it is a real number (isinstance-guarded so it falls back to now for
missing/non-numeric values), keeping the chat-completions shape faithful and
avoiding surprising timestamp drift in logs/traces. Adds a unit test.

Addresses PR #48002 reviewer comment on _byo_judge.py.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_byo_judge.py               |  6 +++++-
 .../tests/unittests/test_byo_judge.py               | 13 +++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index 3cdaf2f10ecf..b1524c100a3c 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -143,7 +143,11 @@ def __init__(self, response: Any) -> None:
         _usage = getattr(response, "usage", None)
         self.usage = _Usage(_usage) if _usage is not None else None
         self.object = "chat.completion"
-        self.created = int(time.time())
+        # Preserve the server-provided created timestamp when present; fall back to now otherwise.
+        _created = getattr(response, "created", None)
+        self.created = (
+            int(_created) if isinstance(_created, (int, float)) and not isinstance(_created, bool) else int(time.time())
+        )
         self.choices = [_Choice(getattr(response, "output_text", "") or "", _finish_reason(response))]
 
 
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index cd4b35cf7e9a..bf86ba71e456 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -258,6 +258,19 @@ def test_no_extra_headers_by_default(self, mock_aipc):
         _, rkwargs = oai.responses.create.call_args
         assert "extra_headers" not in rkwargs
 
+    @patch("azure.ai.projects.aio.AIProjectClient")
+    def test_preserves_server_created_timestamp(self, mock_aipc):
+        # When the Responses result carries a server ``created`` timestamp, the shim preserves it
+        # instead of overwriting with local wall-clock time.
+        resp = MagicMock(output_text="ok", usage=None, id="r", model="m", status="completed", created=1752694800)
+        oai = MagicMock()
+        oai.responses.create = AsyncMock(return_value=resp)
+        mock_aipc.return_value.get_openai_client.return_value = oai
+
+        client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
+        result = asyncio.run(client.chat.completions.create(messages=[{"role": "user", "content": "hi"}]))
+        assert result.created == 1752694800
+
 
 class TestValidateModelConfigByo:
     def test_byo_config_passes_through_validation(self):

From dbd11aaee6d6fc16c64d6d6d14d93e6f72362d6c Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Thu, 16 Jul 2026 18:26:49 -0400
Subject: [PATCH 14/26] Preserve SDK default headers on the BYO prompty path

The BYO prompty branch forwarded only caller-supplied extra_headers to the
Responses API, dropping the SDK default headers (e.g. User-Agent) that the
non-BYO OpenAI/AzureOpenAI paths always set via default_headers. Merge
default_headers with the caller-supplied extra_headers (caller headers take
precedence) so BYO judge calls keep telemetry parity with the other paths.

Addresses PR #48002 reviewer comment on _prompty.py.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_legacy/prompty/_prompty.py          | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
index 012835574841..a80bfc3fb502 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
@@ -300,11 +300,14 @@ async def __call__(  # pylint: disable=docstring-keyword-should-match-keyword-on
             # Admin-connected (BYO) judge model referenced as "connection/deployment": route the
             # chat.completions call through the Foundry project Responses API. The platform resolves
             # the connection and every auth type (API key / managed identity / OAuth2).
+            # Merge the SDK default headers (e.g. User-Agent) with any caller-supplied extra headers,
+            # keeping parity with the non-BYO paths' default_headers; caller headers take precedence.
+            byo_headers = {**default_headers, **(configuration.get("extra_headers") or {})}
             api_client = AsyncByoProjectResponsesClient(
                 byo_model=configuration["byo_model"],
                 project_endpoint=configuration["project_endpoint"],
                 credential=self._token_credential,
-                extra_headers=configuration.get("extra_headers"),
+                extra_headers=byo_headers,
             )
         else:
             connection = Connection.parse_from_config(configuration)

From e858e2b80a64b954fa5077075c5af0fafbae676b Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Thu, 16 Jul 2026 18:26:50 -0400
Subject: [PATCH 15/26] Cast BYO model config to Any in validate_model_config

The BYO branch cast the config to AzureOpenAIModelConfiguration, a differently
shaped TypedDict (no azure_endpoint/azure_deployment), which is misleading to
type checkers and readers. Cast to Any instead, accurately reflecting that BYO
configs bypass TypedDict validation.

Addresses PR #48002 reviewer comment on _common/utils.py.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure-ai-evaluation/azure/ai/evaluation/_common/utils.py    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
index 14e02beebc2e..ca31624cb001 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
@@ -316,7 +316,7 @@ def validate_model_config(config: dict) -> Union[AzureOpenAIModelConfiguration,
     from azure.ai.evaluation._byo_judge import is_byo_model_config
 
     if is_byo_model_config(config):
-        return cast(AzureOpenAIModelConfiguration, config)
+        return cast(Any, config)
     try:
         return _validate_typed_dict(config, AzureOpenAIModelConfiguration)
     except TypeError:

From 5012db0e48b85913ef0f07e01aa47ee19ee54647 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Fri, 17 Jul 2026 11:08:46 -0400
Subject: [PATCH 16/26] Forward per-request extra_headers in the BYO
 chat-completions shim

Match the native OpenAI/AzureOpenAI client contract: headers passed per-request
to chat.completions.create(extra_headers=...) are now merged over the
client-level headers (per-request wins) and forwarded to responses.create as a
fresh copy, instead of being dropped by _map_params. Previously only the
client-level extra_headers were forwarded. Adds a unit test covering merge,
per-request precedence, and copy semantics (no mutation leak).

Addresses PR #48002 reviewer comments on _byo_judge.py and test_byo_judge.py.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_byo_judge.py         | 11 ++++++--
 .../tests/unittests/test_byo_judge.py         | 26 +++++++++++++++++++
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index b1524c100a3c..c26fcca5ff50 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -224,11 +224,18 @@ async def _ensure_client(self) -> Any:
 
     async def _responses_create(self, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> Any:
         client = await self._ensure_client()
+        # Per-request headers (those an OpenAI-compatible caller passes to
+        # chat.completions.create(extra_headers=...)) are merged over the client-level headers
+        # (per-request wins) and forwarded as a fresh copy so neither source dict is mutated.
+        request_headers = kwargs.pop("extra_headers", None)
+        merged_headers: Optional[Dict[str, str]] = None
+        if self._extra_headers or request_headers:
+            merged_headers = {**(self._extra_headers or {}), **(request_headers or {})}
         extra: Dict[str, Any] = {}
         if self._timeout is not None:
             extra["timeout"] = self._timeout
-        if self._extra_headers:
-            extra["extra_headers"] = self._extra_headers
+        if merged_headers:
+            extra["extra_headers"] = merged_headers
         return await client.responses.create(
             model=self._byo_model,
             input=_to_responses_input(messages),
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index bf86ba71e456..98a6163f0c1c 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -258,6 +258,32 @@ def test_no_extra_headers_by_default(self, mock_aipc):
         _, rkwargs = oai.responses.create.call_args
         assert "extra_headers" not in rkwargs
 
+    @patch("azure.ai.projects.aio.AIProjectClient")
+    def test_per_request_extra_headers_merged_and_forwarded(self, mock_aipc):
+        # Headers passed per-request to chat.completions.create(extra_headers=...) must be forwarded
+        # to responses.create and merged over client-level headers (per-request wins), as a copy.
+        resp = MagicMock(output_text="ok", usage=None, id="r", model="m", status="completed")
+        oai = MagicMock()
+        oai.responses.create = AsyncMock(return_value=resp)
+        mock_aipc.return_value.get_openai_client.return_value = oai
+
+        client_headers = {"x-ms-client-request-id": "client-1", "x-shared": "from-client"}
+        request_headers = {"Connection": "close", "x-shared": "from-request"}
+        client = AsyncByoProjectResponsesClient(
+            "c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock(), extra_headers=client_headers
+        )
+        asyncio.run(
+            client.chat.completions.create(messages=[{"role": "user", "content": "hi"}], extra_headers=request_headers)
+        )
+
+        _, rkwargs = oai.responses.create.call_args
+        fwd = rkwargs["extra_headers"]
+        assert fwd == {"x-ms-client-request-id": "client-1", "Connection": "close", "x-shared": "from-request"}
+        # Per-request value wins on conflict.
+        assert fwd["x-shared"] == "from-request"
+        # Forwarded as a copy — neither source dict is the forwarded object (no mutation leak).
+        assert fwd is not client_headers and fwd is not request_headers
+
     @patch("azure.ai.projects.aio.AIProjectClient")
     def test_preserves_server_created_timestamp(self, mock_aipc):
         # When the Responses result carries a server ``created`` timestamp, the shim preserves it

From 10a2949ebfa35e6603dc762fa815a23ff003673a Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Mon, 20 Jul 2026 11:32:18 -0400
Subject: [PATCH 17/26] Make test_evaluate_output_path xdist-safe (fixes
 sdist/whl CI legs)

test_evaluate_output_path[False] used a bare relative output_path
("eval_test_results.jsonl"), written and os.remove'd relative to the process
CWD. Under pytest-xdist (used by the whl/sdist test legs, unlike the serial
coverage leg) the CWD is shared across worker processes, so the write/exists/
remove of a relative file races with other workers and intermittently fails with
FileNotFoundError on cleanup. Adding the new BYO unit tests shifted the xdist
test distribution and deterministically exposed this pre-existing race.

chdir into the per-test tmpdir for the relative-path case so the bare filename
resolves to an isolated per-test directory, preserving the relative-path
behavior under test while removing the cross-worker race. The absolute-path
case is unchanged.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure-ai-evaluation/tests/unittests/test_evaluate.py   | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
index 27e35bd88ab6..53c99661c304 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
@@ -551,11 +551,16 @@ def test_evaluate_output_dir_not_exist(self, mock_model_config, questions_file):
         assert "The output directory './not_exist_dir' does not exist." in exc_info.value.args[0]
 
     @pytest.mark.parametrize("use_relative_path", [True, False])
-    def test_evaluate_output_path(self, evaluate_test_data_jsonl_file, tmpdir, use_relative_path):
+    def test_evaluate_output_path(self, evaluate_test_data_jsonl_file, tmpdir, monkeypatch, use_relative_path):
         # output_path is a file
         if use_relative_path:
             output_path = os.path.join(tmpdir, "eval_test_results.jsonl")
         else:
+            # A bare relative filename resolves against the process CWD. Under pytest-xdist the CWD is
+            # shared across worker processes, so writing/removing a relative file in the CWD races with
+            # other workers (intermittent FileNotFoundError on cleanup). chdir into the per-test tmpdir
+            # to keep the relative-path behavior under test while isolating the file per test.
+            monkeypatch.chdir(tmpdir)
             output_path = "eval_test_results.jsonl"
 
         result = evaluate(

From 2b14b1aa41c5506f4aa4217e4b26529d0ccf1920 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Mon, 20 Jul 2026 11:49:06 -0400
Subject: [PATCH 18/26] Fix BYO shim: read Responses created_at and disable
 client-side retries

- created timestamp: OpenAI Responses objects expose the server timestamp as
  created_at, not created. The shim read created, so real responses always took
  the local-time fallback (and the test mocked the wrong field). Read created_at
  first, keep created as a fallback for older/mocked shapes.
- retries: AsyncPrompty disables the OpenAI client's built-in retries and runs
  its own observable retry loop (max_retries=0). The BYO shim built the project
  client without that, so each outer attempt could trigger hidden per-request
  retries and exceed the intended budget. Pass max_retries=0 to get_openai_client
  (kwargs are forwarded to the underlying AsyncOpenAI client).

Tests: cover created_at + created fallback via _ChatCompletion directly, and
assert get_openai_client is called with max_retries=0. Also add an AsyncPrompty
integration test that loads a prompty with a BYO configuration and verifies the
BYO branch routes through the project Responses API end-to-end (not just the shim
in isolation).

Addresses PR #48002 reviewer comments on _byo_judge.py and test_byo_judge.py.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_byo_judge.py         | 15 +++-
 .../tests/unittests/test_byo_judge.py         | 73 +++++++++++++++++--
 2 files changed, 77 insertions(+), 11 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index c26fcca5ff50..3f70963eb87b 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -143,8 +143,11 @@ def __init__(self, response: Any) -> None:
         _usage = getattr(response, "usage", None)
         self.usage = _Usage(_usage) if _usage is not None else None
         self.object = "chat.completion"
-        # Preserve the server-provided created timestamp when present; fall back to now otherwise.
-        _created = getattr(response, "created", None)
+        # Preserve the server-provided created timestamp when present. OpenAI Responses objects expose
+        # it as ``created_at`` (older/mocked shapes may use ``created``); fall back to now otherwise.
+        _created = getattr(response, "created_at", None)
+        if _created is None:
+            _created = getattr(response, "created", None)
         self.created = (
             int(_created) if isinstance(_created, (int, float)) and not isinstance(_created, bool) else int(time.time())
         )
@@ -214,7 +217,13 @@ async def _ensure_client(self) -> Any:
                     "(BYO) judge models."
                 ) from ex
 
-            client = AIProjectClient(endpoint=self._project_endpoint, credential=self._credential).get_openai_client()
+            # AsyncPrompty disables the OpenAI client's built-in retries and runs its own observable
+            # retry loop (see _prompty.py), so build the project client with max_retries=0 to avoid
+            # hidden per-request retries multiplying the intended retry budget. get_openai_client
+            # forwards kwargs to the underlying AsyncOpenAI client.
+            client = AIProjectClient(endpoint=self._project_endpoint, credential=self._credential).get_openai_client(
+                max_retries=0
+            )
             # ``get_openai_client()`` is synchronous in some azure-ai-projects versions and a coroutine
             # in others; await it only when needed so the shim works across both.
             if inspect.isawaitable(client):
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index 98a6163f0c1c..c0dfbae5329f 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -171,6 +171,10 @@ def test_async_chat_completions_routes_to_responses(self, mock_aipc):
         mock_aipc.assert_called_once()
         _, pkwargs = mock_aipc.call_args
         assert pkwargs["endpoint"] == "https://acct.services.ai.azure.com/api/projects/p1"
+        # The OpenAI client is built with max_retries=0 so AsyncPrompty's own retry loop is the only
+        # retry layer (no hidden per-request retries multiplying the budget).
+        _, gkwargs = mock_aipc.return_value.get_openai_client.call_args
+        assert gkwargs["max_retries"] == 0
         _, rkwargs = oai.responses.create.call_args
         assert rkwargs["model"] == "my-conn/gpt-4o"
         assert rkwargs["input"] == [{"type": "message", "role": "user", "content": "rate coherence"}]
@@ -286,16 +290,16 @@ def test_per_request_extra_headers_merged_and_forwarded(self, mock_aipc):
 
     @patch("azure.ai.projects.aio.AIProjectClient")
     def test_preserves_server_created_timestamp(self, mock_aipc):
-        # When the Responses result carries a server ``created`` timestamp, the shim preserves it
+        # OpenAI Responses objects expose the server timestamp as ``created_at``; the shim preserves it
         # instead of overwriting with local wall-clock time.
-        resp = MagicMock(output_text="ok", usage=None, id="r", model="m", status="completed", created=1752694800)
-        oai = MagicMock()
-        oai.responses.create = AsyncMock(return_value=resp)
-        mock_aipc.return_value.get_openai_client.return_value = oai
+        cc = _ChatCompletion(SimpleNamespace(output_text="ok", usage=None, id="r", model="m", created_at=1752694800))
+        assert cc.created == 1752694800
 
-        client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
-        result = asyncio.run(client.chat.completions.create(messages=[{"role": "user", "content": "hi"}]))
-        assert result.created == 1752694800
+    def test_created_timestamp_falls_back_to_created_then_now(self):
+        # Older/mocked shapes may only carry ``created``; that is honored as a fallback.
+        assert _ChatCompletion(SimpleNamespace(output_text="ok", created=1700000000)).created == 1700000000
+        # When neither is a real number, fall back to the local wall-clock time (a positive int).
+        assert _ChatCompletion(SimpleNamespace(output_text="ok")).created > 0
 
 
 class TestValidateModelConfigByo:
@@ -315,3 +319,56 @@ def test_non_byo_invalid_config_still_raises(self):
 
         with pytest.raises(EvaluationException):
             validate_model_config({"not": "a valid model config"})
+
+
+class TestAsyncPromptyByoIntegration:
+    """Exercise the BYO branch through AsyncPrompty end-to-end (not just the shim in isolation)."""
+
+    _PROMPTY = (
+        "---\n"
+        "name: byo-judge\n"
+        "model:\n"
+        "  api: chat\n"
+        "  parameters:\n"
+        "    temperature: 0.0\n"
+        "inputs:\n"
+        "  question:\n"
+        "    type: string\n"
+        "---\n"
+        "system:\n"
+        "You are a judge.\n"
+        "\n"
+        "user:\n"
+        "{{question}}\n"
+    )
+
+    @patch("azure.ai.projects.aio.AIProjectClient")
+    def test_async_prompty_routes_byo_config_through_shim(self, mock_aipc, tmp_path):
+        from azure.ai.evaluation._legacy.prompty._prompty import AsyncPrompty
+
+        prompty_path = tmp_path / "byo_judge.prompty"
+        prompty_path.write_text(self._PROMPTY, encoding="utf-8")
+
+        resp = SimpleNamespace(output_text="score: 5", usage=None, id="r", model="conn/dep", created_at=1)
+        oai = MagicMock()
+        oai.responses.create = AsyncMock(return_value=resp)
+        mock_aipc.return_value.get_openai_client.return_value = oai
+
+        model = {
+            "configuration": {
+                "byo_model": "conn/dep",
+                "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p",
+            },
+            "parameters": {"temperature": 0.0},
+        }
+        flow = AsyncPrompty.load(source=str(prompty_path), model=model, token_credential=MagicMock())
+        result = asyncio.run(flow(question="rate this"))
+
+        # The BYO branch built the project client with the project endpoint and routed to responses.
+        mock_aipc.assert_called_once()
+        _, pkwargs = mock_aipc.call_args
+        assert pkwargs["endpoint"] == "https://acct.services.ai.azure.com/api/projects/p"
+        _, rkwargs = oai.responses.create.call_args
+        assert rkwargs["model"] == "conn/dep"
+        # The formatted judge output flows back through AsyncPrompty's response formatting.
+        assert "score: 5" in str(result)

From 54039206ba3e960a40278015f82ffb6bc8e25e19 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Mon, 20 Jul 2026 11:49:07 -0400
Subject: [PATCH 19/26] Represent the BYO caller contract with a
 BYOModelConfiguration TypedDict

validate_model_config returned a BYO config cast to Any, which hid the newly
supported caller shape from type checkers. Add a BYOModelConfiguration TypedDict
(byo_model + project_endpoint, optional extra_headers) and include it in the
validate_model_config return union and the construct_prompty_model_config
parameter union, casting the BYO branch to it. Runtime behavior is unchanged
(TypedDicts are dicts; the cast is identity).

Addresses PR #48002 reviewer comment on _common/utils.py.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_common/utils.py      |  9 +++++---
 .../ai/evaluation/_model_configurations.py    | 21 +++++++++++++++++++
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
index ca31624cb001..a4b6d597d75a 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
@@ -18,6 +18,7 @@
 from azure.ai.evaluation._model_configurations import (
     AzureAIProject,
     AzureOpenAIModelConfiguration,
+    BYOModelConfiguration,
     OpenAIModelConfiguration,
 )
 
@@ -222,7 +223,7 @@ def parse_model_config_type(
 
 
 def construct_prompty_model_config(
-    model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+    model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration, BYOModelConfiguration],
     default_api_version: str,
     user_agent: str,
 ) -> dict:
@@ -309,14 +310,16 @@ def validate_azure_ai_project(o: object) -> AzureAIProject:
     return cast(AzureAIProject, o)
 
 
-def validate_model_config(config: dict) -> Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration]:
+def validate_model_config(
+    config: dict,
+) -> Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration, BYOModelConfiguration]:
     # Admin-connected (BYO) judge models are referenced as "connection/deployment" and served via the
     # Foundry project Responses API. Their config intentionally omits azure_endpoint/azure_deployment,
     # so bypass the AzureOpenAI/OpenAI TypedDict validation and let the prompty layer route them.
     from azure.ai.evaluation._byo_judge import is_byo_model_config
 
     if is_byo_model_config(config):
-        return cast(Any, config)
+        return cast(BYOModelConfiguration, config)
     try:
         return _validate_typed_dict(config, AzureOpenAIModelConfiguration)
     except TypeError:
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py
index 26ec432d04f1..51bab9214e4b 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py
@@ -83,6 +83,27 @@ class OpenAIModelConfiguration(TypedDict):
     extra_headers: NotRequired[Dict[str, str]]
 
 
+class BYOModelConfiguration(TypedDict):
+    """Model configuration for an admin-connected (bring-your-own / BYO) model.
+
+    An admin-connected model (a Foundry ``ModelGateway`` / ``ApiManagement`` connection) is
+    referenced as ``"connection/deployment"`` and served through the Foundry project Responses API,
+    where the platform resolves the connection and its auth. Both markers are required.
+
+    :param byo_model: The admin-connected model reference, ``"connection-name/deployment-name"``.
+    :type byo_model: str
+    :param project_endpoint: The Foundry project endpoint,
+        ``https://<account>.services.ai.azure.com/api/projects/<project>``.
+    :type project_endpoint: str
+    :param extra_headers: Additional HTTP headers to include in every request. Optional.
+    :type extra_headers: NotRequired[Dict[str, str]]
+    """
+
+    byo_model: str
+    project_endpoint: str
+    extra_headers: NotRequired[Dict[str, str]]
+
+
 class AzureAIProject(TypedDict):
     """Information about the Azure AI project
 

From 9c2c6ae71fa2ad71ca95a4ab3c9a86b2a233d0b0 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Mon, 20 Jul 2026 11:49:07 -0400
Subject: [PATCH 20/26] Add CHANGELOG entry for admin-connected (BYO) judge
 models

Document the new BYO model configuration (byo_model + project_endpoint), its
routing through the Foundry project Responses API, and its optional
azure-ai-projects requirement, under the upcoming 1.18.1 Features Added section.

Addresses PR #48002 reviewer comment.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 sdk/evaluation/azure-ai-evaluation/CHANGELOG.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md
index 64b34c92ddf7..a92107e3a4bb 100644
--- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md
+++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md
@@ -2,6 +2,16 @@
 
 ## 1.18.1 (2026-07-09)
 
+### Features Added
+
+- Added support for admin-connected (bring-your-own / BYO) models as
+  LLM-as-a-judge evaluator models. A model configuration carrying
+  `byo_model` (`"connection/deployment"`) and `project_endpoint` routes
+  prompty-based judges through the Foundry project Responses API, where the
+  platform resolves the connection and its auth (API key / managed identity /
+  OAuth2). This path requires the optional `azure-ai-projects` package to be
+  installed; a clear error is raised if it is missing.
+
 ### Bugs Fixed
 
 - Enabled `azure_ai_search`, `azure_fabric`, and `sharepoint_grounding` tool

From d5185a7c03720be9ed1a507e3386a7f4940f3390 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Mon, 20 Jul 2026 13:59:18 -0400
Subject: [PATCH 21/26] Keep BYOModelConfiguration internal
 (_BYOModelConfiguration)

Admin-connected (BYO) model configs are constructed server-side by the
Foundry platform, not hand-authored by SDK users, and the feature is still
preview/gated. Rename the TypedDict to _BYOModelConfiguration and reframe its
docstring as internal plumbing so it is not advertised as a public, supported
type. It stays out of azure.ai.evaluation / __all__; the public model-config
union will be widened to advertise BYO once the feature reaches GA.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_common/utils.py          |  8 ++++----
 .../azure/ai/evaluation/_model_configurations.py  | 15 +++++++++------
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
index a4b6d597d75a..c466b771d378 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
@@ -18,7 +18,7 @@
 from azure.ai.evaluation._model_configurations import (
     AzureAIProject,
     AzureOpenAIModelConfiguration,
-    BYOModelConfiguration,
+    _BYOModelConfiguration,
     OpenAIModelConfiguration,
 )
 
@@ -223,7 +223,7 @@ def parse_model_config_type(
 
 
 def construct_prompty_model_config(
-    model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration, BYOModelConfiguration],
+    model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration, _BYOModelConfiguration],
     default_api_version: str,
     user_agent: str,
 ) -> dict:
@@ -312,14 +312,14 @@ def validate_azure_ai_project(o: object) -> AzureAIProject:
 
 def validate_model_config(
     config: dict,
-) -> Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration, BYOModelConfiguration]:
+) -> Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration, _BYOModelConfiguration]:
     # Admin-connected (BYO) judge models are referenced as "connection/deployment" and served via the
     # Foundry project Responses API. Their config intentionally omits azure_endpoint/azure_deployment,
     # so bypass the AzureOpenAI/OpenAI TypedDict validation and let the prompty layer route them.
     from azure.ai.evaluation._byo_judge import is_byo_model_config
 
     if is_byo_model_config(config):
-        return cast(BYOModelConfiguration, config)
+        return cast(_BYOModelConfiguration, config)
     try:
         return _validate_typed_dict(config, AzureOpenAIModelConfiguration)
     except TypeError:
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py
index 51bab9214e4b..d922d597fb93 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py
@@ -83,12 +83,15 @@ class OpenAIModelConfiguration(TypedDict):
     extra_headers: NotRequired[Dict[str, str]]
 
 
-class BYOModelConfiguration(TypedDict):
-    """Model configuration for an admin-connected (bring-your-own / BYO) model.
-
-    An admin-connected model (a Foundry ``ModelGateway`` / ``ApiManagement`` connection) is
-    referenced as ``"connection/deployment"`` and served through the Foundry project Responses API,
-    where the platform resolves the connection and its auth. Both markers are required.
+class _BYOModelConfiguration(TypedDict):
+    """Internal shape of an admin-connected (bring-your-own / BYO) model configuration.
+
+    This is service-internal plumbing, not a caller-facing type. BYO configs are constructed
+    server-side by the Foundry platform (the evaluation data plane resolves an admin-connected
+    ``ModelGateway`` / ``ApiManagement`` connection and stamps these markers), then routed through
+    the Foundry project Responses API where the platform owns connection resolution and auth. It is
+    intentionally not exported from ``azure.ai.evaluation``; the public model-config union will be
+    widened to advertise BYO once the feature reaches GA.
 
     :param byo_model: The admin-connected model reference, ``"connection-name/deployment-name"``.
     :type byo_model: str

From 2736ccaada3586dac3929cbce6b8da4e71f67785 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Mon, 20 Jul 2026 14:10:13 -0400
Subject: [PATCH 22/26] Drop BYO CHANGELOG entry; require
 azure-ai-projects>=2.0.0b1 for tests

Remove the admin-connected (BYO) Features Added entry from the 1.18.1 section
(the release entry will be added when the feature ships).

Bump the dev/test pin from azure-ai-projects<=1.0.0b10 to >=2.0.0b1: the BYO
judge path uses AIProjectClient.get_openai_client().responses (the Foundry
project Responses API), which the ACA data plane runs on 2.0.0b1, and Red Team
already documents azure-ai-projects>=2.0.0b1 as the minimum in TROUBLESHOOTING.md.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 sdk/evaluation/azure-ai-evaluation/CHANGELOG.md        | 10 ----------
 .../azure-ai-evaluation/dev_requirements.txt           |  2 +-
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md
index a92107e3a4bb..64b34c92ddf7 100644
--- a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md
+++ b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md
@@ -2,16 +2,6 @@
 
 ## 1.18.1 (2026-07-09)
 
-### Features Added
-
-- Added support for admin-connected (bring-your-own / BYO) models as
-  LLM-as-a-judge evaluator models. A model configuration carrying
-  `byo_model` (`"connection/deployment"`) and `project_endpoint` routes
-  prompty-based judges through the Foundry project Responses API, where the
-  platform resolves the connection and its auth (API key / managed identity /
-  OAuth2). This path requires the optional `azure-ai-projects` package to be
-  installed; a clear error is raised if it is missing.
-
 ### Bugs Fixed
 
 - Enabled `azure_ai_search`, `azure_fabric`, and `sharepoint_grounding` tool
diff --git a/sdk/evaluation/azure-ai-evaluation/dev_requirements.txt b/sdk/evaluation/azure-ai-evaluation/dev_requirements.txt
index 527d87a0d912..2b74af19ef78 100644
--- a/sdk/evaluation/azure-ai-evaluation/dev_requirements.txt
+++ b/sdk/evaluation/azure-ai-evaluation/dev_requirements.txt
@@ -7,7 +7,7 @@ pytest-cov
 pytest-mock
 pytest-xdist
 azure-ai-inference>=1.0.0b4
-azure-ai-projects<=1.0.0b10
+azure-ai-projects>=2.0.0b1
 aiohttp
 filelock
 promptflow-core>=1.17.1

From 31e93d68ab64834ac0cbf2488df5ea1440ded02f Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Mon, 20 Jul 2026 14:20:37 -0400
Subject: [PATCH 23/26] Make BYO inline comments concise

Tighten verbose inline comments across the BYO judge shim, prompty branch,
model-config validation, and their tests to one or two lines each, keeping the
essential rationale. Docstrings/function explanations are unchanged.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_byo_judge.py         | 24 +++++++----------
 .../azure/ai/evaluation/_common/utils.py      |  5 ++--
 .../ai/evaluation/_legacy/prompty/_prompty.py |  9 +++----
 .../tests/unittests/test_byo_judge.py         | 26 +++++++------------
 .../tests/unittests/test_evaluate.py          |  6 ++---
 5 files changed, 26 insertions(+), 44 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index 3f70963eb87b..fa42e06307b3 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -143,8 +143,7 @@ def __init__(self, response: Any) -> None:
         _usage = getattr(response, "usage", None)
         self.usage = _Usage(_usage) if _usage is not None else None
         self.object = "chat.completion"
-        # Preserve the server-provided created timestamp when present. OpenAI Responses objects expose
-        # it as ``created_at`` (older/mocked shapes may use ``created``); fall back to now otherwise.
+        # Server timestamp: Responses uses created_at (older/mocked shapes may use created); else now.
         _created = getattr(response, "created_at", None)
         if _created is None:
             _created = getattr(response, "created", None)
@@ -191,16 +190,15 @@ def __init__(
         self._byo_model = byo_model
         self._project_endpoint = project_endpoint
         self._credential = credential
-        # Optional headers forwarded on every Responses API call, needed when ACA runs continuous evaluations.
+        # Headers forwarded on every Responses call; used when ACA runs continuous evaluations.
         self._extra_headers: Optional[Dict[str, str]] = dict(extra_headers) if extra_headers else None
         self._client: Any = None
         self._timeout: Optional[float] = None
         self.chat = _AsyncChat(self)
 
     def with_options(self, *, timeout: Any = None, **_kwargs: Any) -> "AsyncByoProjectResponsesClient":
-        # The prompty runner sets a per-request timeout via with_options(timeout=...). Capture a
-        # concrete numeric value so it reaches responses.create; openai's ``NotGiven`` sentinel (used
-        # when no timeout is configured) is not numeric and is therefore ignored.
+        # Capture a numeric per-request timeout so it reaches responses.create; openai's NotGiven
+        # sentinel (no timeout configured) is non-numeric and is ignored.
         if isinstance(timeout, (int, float)) and not isinstance(timeout, bool):
             self._timeout = float(timeout)
         return self
@@ -217,15 +215,12 @@ async def _ensure_client(self) -> Any:
                     "(BYO) judge models."
                 ) from ex
 
-            # AsyncPrompty disables the OpenAI client's built-in retries and runs its own observable
-            # retry loop (see _prompty.py), so build the project client with max_retries=0 to avoid
-            # hidden per-request retries multiplying the intended retry budget. get_openai_client
-            # forwards kwargs to the underlying AsyncOpenAI client.
+            # AsyncPrompty runs its own retry loop, so use max_retries=0 to avoid compounding retries.
+            # get_openai_client forwards kwargs to the underlying AsyncOpenAI client.
             client = AIProjectClient(endpoint=self._project_endpoint, credential=self._credential).get_openai_client(
                 max_retries=0
             )
-            # ``get_openai_client()`` is synchronous in some azure-ai-projects versions and a coroutine
-            # in others; await it only when needed so the shim works across both.
+            # get_openai_client() is sync in some azure-ai-projects versions, async in others; await if needed.
             if inspect.isawaitable(client):
                 client = await client
             self._client = client
@@ -233,9 +228,8 @@ async def _ensure_client(self) -> Any:
 
     async def _responses_create(self, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> Any:
         client = await self._ensure_client()
-        # Per-request headers (those an OpenAI-compatible caller passes to
-        # chat.completions.create(extra_headers=...)) are merged over the client-level headers
-        # (per-request wins) and forwarded as a fresh copy so neither source dict is mutated.
+        # Merge per-request extra_headers over the client-level headers (per-request wins),
+        # forwarding a fresh copy so neither source dict is mutated.
         request_headers = kwargs.pop("extra_headers", None)
         merged_headers: Optional[Dict[str, str]] = None
         if self._extra_headers or request_headers:
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
index c466b771d378..35deefedb3fb 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
@@ -313,9 +313,8 @@ def validate_azure_ai_project(o: object) -> AzureAIProject:
 def validate_model_config(
     config: dict,
 ) -> Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration, _BYOModelConfiguration]:
-    # Admin-connected (BYO) judge models are referenced as "connection/deployment" and served via the
-    # Foundry project Responses API. Their config intentionally omits azure_endpoint/azure_deployment,
-    # so bypass the AzureOpenAI/OpenAI TypedDict validation and let the prompty layer route them.
+    # BYO judge configs (connection/deployment) omit azure_endpoint/azure_deployment and route via
+    # the Foundry Responses API, so skip the AzureOpenAI/OpenAI TypedDict validation.
     from azure.ai.evaluation._byo_judge import is_byo_model_config
 
     if is_byo_model_config(config):
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
index a80bfc3fb502..87876d997321 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
@@ -297,11 +297,10 @@ async def __call__(  # pylint: disable=docstring-keyword-should-match-keyword-on
 
         api_client: Union[AsyncAzureOpenAI, AsyncOpenAI, AsyncByoProjectResponsesClient]
         if is_byo:
-            # Admin-connected (BYO) judge model referenced as "connection/deployment": route the
-            # chat.completions call through the Foundry project Responses API. The platform resolves
-            # the connection and every auth type (API key / managed identity / OAuth2).
-            # Merge the SDK default headers (e.g. User-Agent) with any caller-supplied extra headers,
-            # keeping parity with the non-BYO paths' default_headers; caller headers take precedence.
+            # BYO judge model (connection/deployment): route chat.completions through the Foundry
+            # Responses API, which resolves the connection and auth (API key / managed identity / OAuth2).
+            # Merge SDK default headers (e.g. User-Agent) with caller extra_headers (caller wins),
+            # matching the non-BYO paths' default_headers.
             byo_headers = {**default_headers, **(configuration.get("extra_headers") or {})}
             api_client = AsyncByoProjectResponsesClient(
                 byo_model=configuration["byo_model"],
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
index c0dfbae5329f..e35eca66592e 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -24,12 +24,10 @@ def test_is_byo_model_config(self):
         assert is_byo_model_config({"byo_model": "c/d", "project_endpoint": "https://x"})
         assert not is_byo_model_config({"azure_endpoint": "https://x"})
         assert not is_byo_model_config({})
-        # Both markers are required — byo_model alone must not activate the BYO path
-        # (the prompty branch needs project_endpoint to route, else it would KeyError).
+        # Both markers required — byo_model alone must not activate BYO (prompty needs project_endpoint).
         assert not is_byo_model_config({"byo_model": "c/d"})
         assert not is_byo_model_config({"project_endpoint": "https://x"})
-        # Markers must be non-empty strings — truthy-but-invalid values (e.g. ints) must not
-        # activate BYO, else they bypass validate_model_config and fail deep inside the client.
+        # Markers must be non-empty strings; truthy-but-invalid values (e.g. ints) must not activate BYO.
         assert not is_byo_model_config({"byo_model": 1, "project_endpoint": 2})
         assert not is_byo_model_config({"byo_model": "", "project_endpoint": "https://x"})
         assert not is_byo_model_config({"byo_model": "c/d", "project_endpoint": ""})
@@ -171,8 +169,7 @@ def test_async_chat_completions_routes_to_responses(self, mock_aipc):
         mock_aipc.assert_called_once()
         _, pkwargs = mock_aipc.call_args
         assert pkwargs["endpoint"] == "https://acct.services.ai.azure.com/api/projects/p1"
-        # The OpenAI client is built with max_retries=0 so AsyncPrompty's own retry loop is the only
-        # retry layer (no hidden per-request retries multiplying the budget).
+        # max_retries=0 so AsyncPrompty's own retry loop is the only retry layer.
         _, gkwargs = mock_aipc.return_value.get_openai_client.call_args
         assert gkwargs["max_retries"] == 0
         _, rkwargs = oai.responses.create.call_args
@@ -184,8 +181,7 @@ def test_async_chat_completions_routes_to_responses(self, mock_aipc):
         assert rkwargs["timeout"] == 30
 
     def test_non_numeric_timeout_is_not_forwarded(self):
-        # openai passes a NotGiven() sentinel when no timeout is configured; the shim must ignore it
-        # (only a concrete numeric timeout is forwarded to responses.create).
+        # openai passes NotGiven() when no timeout is set; the shim forwards only a numeric timeout.
         client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
         client.with_options(timeout=object())
         assert client._timeout is None
@@ -212,10 +208,8 @@ def test_with_options_returns_self(self):
         assert client.with_options(timeout=5) is client
 
     def test_missing_azure_ai_projects_raises_clear_error(self, monkeypatch):
-        # azure-ai-projects is an optional dependency (not in install_requires). When the BYO path
-        # is used without it installed, the shim must surface a clear MissingRequiredPackage error
-        # instead of a raw ModuleNotFoundError. A None entry in sys.modules makes
-        # ``from azure.ai.projects.aio import AIProjectClient`` raise ImportError.
+        # azure-ai-projects is optional (not in install_requires); the BYO path must surface a clear
+        # MissingRequiredPackage error. A None entry in sys.modules makes the import raise ImportError.
         import sys
 
         from azure.ai.evaluation._legacy._adapters._errors import MissingRequiredPackage
@@ -227,8 +221,7 @@ def test_missing_azure_ai_projects_raises_clear_error(self, monkeypatch):
 
     @patch("azure.ai.projects.aio.AIProjectClient")
     def test_extra_headers_forwarded_to_responses(self, mock_aipc):
-        # ACA may supply additional headers (e.g. correlation/telemetry headers) when running
-        # continuous evaluations; the shim must forward them to responses.create.
+        # ACA may supply extra headers for continuous evals; the shim must forward them to responses.create.
         resp = MagicMock(output_text="ok", usage=None, id="r", model="m", status="completed")
         oai = MagicMock()
         oai.responses.create = AsyncMock(return_value=resp)
@@ -265,7 +258,7 @@ def test_no_extra_headers_by_default(self, mock_aipc):
     @patch("azure.ai.projects.aio.AIProjectClient")
     def test_per_request_extra_headers_merged_and_forwarded(self, mock_aipc):
         # Headers passed per-request to chat.completions.create(extra_headers=...) must be forwarded
-        # to responses.create and merged over client-level headers (per-request wins), as a copy.
+        # to responses.create, merged over client-level headers (per-request wins), as a copy.
         resp = MagicMock(output_text="ok", usage=None, id="r", model="m", status="completed")
         oai = MagicMock()
         oai.responses.create = AsyncMock(return_value=resp)
@@ -290,8 +283,7 @@ def test_per_request_extra_headers_merged_and_forwarded(self, mock_aipc):
 
     @patch("azure.ai.projects.aio.AIProjectClient")
     def test_preserves_server_created_timestamp(self, mock_aipc):
-        # OpenAI Responses objects expose the server timestamp as ``created_at``; the shim preserves it
-        # instead of overwriting with local wall-clock time.
+        # Responses exposes the server timestamp as created_at; the shim preserves it (not local time).
         cc = _ChatCompletion(SimpleNamespace(output_text="ok", usage=None, id="r", model="m", created_at=1752694800))
         assert cc.created == 1752694800
 
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
index 53c99661c304..08c78b555b4b 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
@@ -556,10 +556,8 @@ def test_evaluate_output_path(self, evaluate_test_data_jsonl_file, tmpdir, monke
         if use_relative_path:
             output_path = os.path.join(tmpdir, "eval_test_results.jsonl")
         else:
-            # A bare relative filename resolves against the process CWD. Under pytest-xdist the CWD is
-            # shared across worker processes, so writing/removing a relative file in the CWD races with
-            # other workers (intermittent FileNotFoundError on cleanup). chdir into the per-test tmpdir
-            # to keep the relative-path behavior under test while isolating the file per test.
+            # A bare relative filename races across pytest-xdist workers sharing the CWD (intermittent
+            # FileNotFoundError on cleanup); chdir into the per-test tmpdir to isolate it.
             monkeypatch.chdir(tmpdir)
             output_path = "eval_test_results.jsonl"
 

From 770ff824931b4ec3d8b179e2ddb6674b4351e850 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Mon, 20 Jul 2026 14:42:18 -0400
Subject: [PATCH 24/26] Revert azure-ai-projects test pin to <=1.0.0b10

The bump to >=2.0.0b1 broke the whl leg: (1) tests/e2etests/test_remote_evaluation.py
imports EvaluatorConfiguration/Evaluation/Dataset from azure.ai.projects.models,
which exist only in the 1.x API (removed in 2.x) -> collection ImportError; and
(2) azure-ai-projects 2.x requires openai>=2.8.0 and azure-core>=1.35.0, which
conflict with this SDK's pinned openai==1.108.0 / azure-core==1.31.0, making the
resolve unsatisfiable.

The test environment must stay on azure-ai-projects 1.x. BYO's 2.x runtime
requirement is a data-plane/ACA concern (already documented in TROUBLESHOOTING.md);
BYO unit tests mock azure-ai-projects, so they pass on 1.0.0b10.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 sdk/evaluation/azure-ai-evaluation/dev_requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/dev_requirements.txt b/sdk/evaluation/azure-ai-evaluation/dev_requirements.txt
index 2b74af19ef78..527d87a0d912 100644
--- a/sdk/evaluation/azure-ai-evaluation/dev_requirements.txt
+++ b/sdk/evaluation/azure-ai-evaluation/dev_requirements.txt
@@ -7,7 +7,7 @@ pytest-cov
 pytest-mock
 pytest-xdist
 azure-ai-inference>=1.0.0b4
-azure-ai-projects>=2.0.0b1
+azure-ai-projects<=1.0.0b10
 aiohttp
 filelock
 promptflow-core>=1.17.1

From 4a9782ed7930929f7abf8484f4c005be848bbd32 Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Mon, 20 Jul 2026 15:53:22 -0400
Subject: [PATCH 25/26] Fix OTel event-logging test guard to detect the Events
 API submodule

The 22 TestLogEvents*/TestEmitEvalResultShutdown tests failed on the sk_ CI leg
with ModuleNotFoundError: No module named 'opentelemetry._events'. The App
Insights event-logging code they cover imports opentelemetry._events /
opentelemetry.sdk._events (the OpenTelemetry Events API, added in opentelemetry
1.26), but the MISSING_OPENTELEMETRY guard only checked the top-level
opentelemetry package. On the sk_ leg semantic-kernel pins an older opentelemetry
(<1.26) that lacks the Events API, so the guard reported present, the tests ran,
and the code crashed on the missing submodule.

Guard on the Events API submodules the code actually imports so the tests skip
cleanly where the API is unavailable (sk_ leg) and still run where it is present
(all other legs). Verified locally: 22 pass with the API present, 22 skip when it
is simulated absent.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure-ai-evaluation/tests/unittests/test_evaluate.py    | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
index 08c78b555b4b..2ac7d282f237 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
@@ -2302,7 +2302,11 @@ def test_nan_threshold_gets_injected(self):
 
 
 try:
-    import opentelemetry  # noqa: F401
+    # These tests exercise the OpenTelemetry Events API (opentelemetry._events /
+    # opentelemetry.sdk._events, added in 1.26). Guard on those submodules, not just the top-level
+    # package: the sk_ CI leg pins an older opentelemetry (via semantic-kernel) that lacks them.
+    import opentelemetry._events  # noqa: F401
+    from opentelemetry.sdk._events import EventLoggerProvider  # noqa: F401
 
     MISSING_OPENTELEMETRY = False
 except ImportError:

From 9c0a8c70821d8ac42c8b1a75f6f26375ec5097ea Mon Sep 17 00:00:00 2001
From: Shruti Iyer <you@example.com>
Date: Mon, 20 Jul 2026 17:05:36 -0400
Subject: [PATCH 26/26] Build Responses input via openai SDK
 EasyInputMessageParam

Address PR review (posaninagendra): construct the Responses API input with the
openai SDK's typed entity (openai.types.responses.EasyInputMessageParam) and
return the SDK's ResponseInputParam, instead of a hand-written dict. This keeps
the shim in sync with the SDK's input schema (type-checked against it) rather
than a hand-authored shape that drifts silently on SDK upgrades.

Runtime output is identical (EasyInputMessageParam is a TypedDict -> plain dict),
so existing input-shape assertions hold. The output side already reads the SDK's
typed Response via its output_text convenience aggregator. openai is a hard
dependency (openai>=1.108.0), so the import is safe. 29 byo tests pass.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: b11202e6-6a52-4777-bed1-9c48730a63c3
---
 .../azure/ai/evaluation/_byo_judge.py            | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
index fa42e06307b3..d3cbebe2d82d 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -17,17 +17,19 @@
 import time
 from typing import Any, Dict, List, Optional
 
+from openai.types.responses import EasyInputMessageParam, ResponseInputParam
 
-def _to_responses_input(messages: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
+
+def _to_responses_input(messages: Optional[List[Dict[str, Any]]]) -> ResponseInputParam:
     """Map chat-completions messages ({role, content}) to Responses API input items."""
-    items: List[Dict[str, Any]] = []
+    items: ResponseInputParam = []
     for message in messages or []:
         items.append(
-            {
-                "type": "message",
-                "role": message.get("role", "user"),
-                "content": message.get("content", ""),
-            }
+            EasyInputMessageParam(
+                type="message",
+                role=message.get("role", "user"),
+                content=message.get("content", ""),
+            )
         )
     return items