diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
new file mode 100644
index 000000000000..d3cbebe2d82d
--- /dev/null
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_byo_judge.py
@@ -0,0 +1,268 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""Use admin-connected (BYO) models as LLM-as-a-judge evaluator models.
+
+Admin-connected models (Foundry "ModelGateway" / "API Management" connections, referenced
+as ``"connection-name/deployment-name"``) are only invokable through the Foundry project
+**Responses API** — the platform resolves the connection and handles every auth type
+(API key / managed identity / OAuth2), ``deploymentInPath``, api-version and custom headers.
+
+Prompty-based judge evaluators (coherence, relevance, fluency, groundedness, etc.) call
+``client.chat.completions.create(...)``. This module provides a small OpenAI-compatible async
+**shim** that routes those calls to the project Responses API for a BYO model, so evaluator
+code can use admin-connected connections **without any change to its calling code**.
+"""
+import inspect
+import time
+from typing import Any, Dict, List, Optional
+
+from openai.types.responses import EasyInputMessageParam, ResponseInputParam
+
+
+def _to_responses_input(messages: Optional[List[Dict[str, Any]]]) -> ResponseInputParam:
+    """Map chat-completions messages ({role, content}) to Responses API input items."""
+    items: ResponseInputParam = []
+    for message in messages or []:
+        items.append(
+            EasyInputMessageParam(
+                type="message",
+                role=message.get("role", "user"),
+                content=message.get("content", ""),
+            )
+        )
+    return items
+
+
+def _map_response_format(response_format: Any) -> Optional[Dict[str, Any]]:
+    """Translate a chat-completions ``response_format`` into a Responses API ``text.format`` value.
+
+    The Responses API exposes the same JSON-output capability as chat.completions, but under
+    ``text.format`` instead of ``response_format``:
+
+    ================================================  ===========================================
+    chat.completions ``response_format``              Responses API ``text.format``
+    ================================================  ===========================================
+    ``{"type": "text"}``                              ``{"type": "text"}``
+    ``{"type": "json_object"}``                       ``{"type": "json_object"}``
+    ``{"type": "json_schema",``                       ``{"type": "json_schema", "name": ...,``
+    ``  "json_schema": {"name", "schema", ...}}``     ``  "schema": ..., "strict": ...}``
+    ================================================  ===========================================
+
+    Note the json_schema shape difference: chat.completions **nests** ``name``/``schema``/``strict``
+    under a ``json_schema`` key, whereas the Responses API **flattens** them directly under
+    ``format``. Returns ``None`` for anything unrecognized so the param is simply omitted.
+    """
+    if not isinstance(response_format, dict):
+        return None
+    rf_type = response_format.get("type")
+    if rf_type in ("text", "json_object"):
+        return {"type": rf_type}
+    if rf_type == "json_schema":
+        # chat.completions nests the schema spec under "json_schema"; the Responses API flattens it.
+        schema_spec = response_format.get("json_schema", response_format)
+        fmt: Dict[str, Any] = {"type": "json_schema"}
+        for key in ("name", "schema", "strict", "description"):
+            if key in schema_spec:
+                fmt[key] = schema_spec[key]
+        return fmt
+    return None
+
+
+def _map_params(kwargs: Dict[str, Any]) -> Dict[str, Any]:
+    """Map a curated set of chat-completions sampling params to Responses API params."""
+    mapped: Dict[str, Any] = {}
+    if "temperature" in kwargs:
+        mapped["temperature"] = kwargs["temperature"]
+    if "top_p" in kwargs:
+        mapped["top_p"] = kwargs["top_p"]
+    for key in ("max_output_tokens", "max_completion_tokens", "max_tokens"):
+        if key in kwargs:
+            mapped["max_output_tokens"] = kwargs[key]
+            break
+    if "response_format" in kwargs:
+        text_format = _map_response_format(kwargs["response_format"])
+        if text_format is not None:
+            mapped["text"] = {"format": text_format}
+    return mapped
+
+
+class _Usage:
+    """chat.completions-shaped usage view over a Responses API usage object.
+
+    The Responses API reports ``input_tokens`` / ``output_tokens`` / ``total_tokens``; judge/grader
+    code (and the prompty response formatter) expects ``prompt_tokens`` / ``completion_tokens`` /
+    ``total_tokens``. This adapts the former to the latter.
+    """
+
+    def __init__(self, response_usage: Any) -> None:
+        self.prompt_tokens = getattr(response_usage, "input_tokens", 0) or 0
+        self.completion_tokens = getattr(response_usage, "output_tokens", 0) or 0
+        # Fall back to prompt + completion when the Responses usage omits total_tokens.
+        self.total_tokens = (getattr(response_usage, "total_tokens", 0) or 0) or (
+            self.prompt_tokens + self.completion_tokens
+        )
+
+
+class _ChatMessage:
+    def __init__(self, content: str) -> None:
+        self.role = "assistant"
+        self.content = content
+        self.tool_calls = None
+
+
+def _finish_reason(response: Any) -> str:
+    """Map a Responses result's status to a chat-completions ``finish_reason``.
+
+    A Responses call that stops early reports ``status="incomplete"`` with an
+    ``incomplete_details.reason`` (e.g. ``"max_output_tokens"`` / ``"content_filter"``).
+    Chat.completions callers expect ``"length"`` for truncation; surfacing it lets the prompty
+    formatter distinguish a truncated (invalid-JSON-prone) judge output from a complete one instead
+    of always seeing ``"stop"``.
+    """
+    if getattr(response, "status", None) != "incomplete":
+        return "stop"
+    details = getattr(response, "incomplete_details", None)
+    reason = getattr(details, "reason", None) if details is not None else None
+    if reason == "max_output_tokens":
+        return "length"
+    return reason or "stop"
+
+
+class _Choice:
+    def __init__(self, content: str, finish_reason: str = "stop") -> None:
+        self.index = 0
+        self.message = _ChatMessage(content)
+        self.finish_reason = finish_reason
+
+
+class _ChatCompletion:
+    """Minimal chat.completions-shaped view over a Responses API result."""
+
+    def __init__(self, response: Any) -> None:
+        self.id = getattr(response, "id", None)
+        self.model = getattr(response, "model", None)
+        _usage = getattr(response, "usage", None)
+        self.usage = _Usage(_usage) if _usage is not None else None
+        self.object = "chat.completion"
+        # Server timestamp: Responses uses created_at (older/mocked shapes may use created); else now.
+        _created = getattr(response, "created_at", None)
+        if _created is None:
+            _created = getattr(response, "created", None)
+        self.created = (
+            int(_created) if isinstance(_created, (int, float)) and not isinstance(_created, bool) else int(time.time())
+        )
+        self.choices = [_Choice(getattr(response, "output_text", "") or "", _finish_reason(response))]
+
+
+class _AsyncChatCompletions:
+    def __init__(self, owner: "AsyncByoProjectResponsesClient") -> None:
+        self._owner = owner
+
+    async def create(
+        self, *, model: Optional[str] = None, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any
+    ) -> _ChatCompletion:
+        # ``model`` from the caller is ignored — the BYO model is fixed by the shim's config.
+        response = await self._owner._responses_create(messages=messages, **kwargs)
+        return _ChatCompletion(response)
+
+
+class _AsyncChat:
+    def __init__(self, owner: "AsyncByoProjectResponsesClient") -> None:
+        self.completions = _AsyncChatCompletions(owner)
+
+
+class AsyncByoProjectResponsesClient:
+    """Async OpenAI-compatible shim that routes ``chat.completions.create()`` to the Foundry project
+    Responses API for an admin-connected (BYO) model.
+
+    Used by the prompty judge path (LLM-as-a-judge evaluators such as coherence, relevance, fluency,
+    groundedness, etc.), whose async runner calls ``client.with_options(...).chat.completions.create(...)``.
+    Those calls are served by the platform, which resolves the connection and every auth type
+    (API key / managed identity / OAuth2), so evaluator code is unchanged.
+    """
+
+    def __init__(
+        self,
+        byo_model: str,
+        project_endpoint: str,
+        credential: Any,
+        extra_headers: Optional[Dict[str, str]] = None,
+    ) -> None:
+        self._byo_model = byo_model
+        self._project_endpoint = project_endpoint
+        self._credential = credential
+        # Headers forwarded on every Responses call; used when ACA runs continuous evaluations.
+        self._extra_headers: Optional[Dict[str, str]] = dict(extra_headers) if extra_headers else None
+        self._client: Any = None
+        self._timeout: Optional[float] = None
+        self.chat = _AsyncChat(self)
+
+    def with_options(self, *, timeout: Any = None, **_kwargs: Any) -> "AsyncByoProjectResponsesClient":
+        # Capture a numeric per-request timeout so it reaches responses.create; openai's NotGiven
+        # sentinel (no timeout configured) is non-numeric and is ignored.
+        if isinstance(timeout, (int, float)) and not isinstance(timeout, bool):
+            self._timeout = float(timeout)
+        return self
+
+    async def _ensure_client(self) -> Any:
+        if self._client is None:
+            try:
+                from azure.ai.projects.aio import AIProjectClient
+            except ImportError as ex:
+                from azure.ai.evaluation._legacy._adapters._errors import MissingRequiredPackage
+
+                raise MissingRequiredPackage(
+                    message="Please install the 'azure-ai-projects' package to use admin-connected "
+                    "(BYO) judge models."
+                ) from ex
+
+            # AsyncPrompty runs its own retry loop, so use max_retries=0 to avoid compounding retries.
+            # get_openai_client forwards kwargs to the underlying AsyncOpenAI client.
+            client = AIProjectClient(endpoint=self._project_endpoint, credential=self._credential).get_openai_client(
+                max_retries=0
+            )
+            # get_openai_client() is sync in some azure-ai-projects versions, async in others; await if needed.
+            if inspect.isawaitable(client):
+                client = await client
+            self._client = client
+        return self._client
+
+    async def _responses_create(self, messages: Optional[List[Dict[str, Any]]] = None, **kwargs: Any) -> Any:
+        client = await self._ensure_client()
+        # Merge per-request extra_headers over the client-level headers (per-request wins),
+        # forwarding a fresh copy so neither source dict is mutated.
+        request_headers = kwargs.pop("extra_headers", None)
+        merged_headers: Optional[Dict[str, str]] = None
+        if self._extra_headers or request_headers:
+            merged_headers = {**(self._extra_headers or {}), **(request_headers or {})}
+        extra: Dict[str, Any] = {}
+        if self._timeout is not None:
+            extra["timeout"] = self._timeout
+        if merged_headers:
+            extra["extra_headers"] = merged_headers
+        return await client.responses.create(
+            model=self._byo_model,
+            input=_to_responses_input(messages),
+            **_map_params(kwargs),
+            **extra,
+        )
+
+
+def is_byo_model_config(model_config: Dict[str, Any]) -> bool:
+    """Return True if the model configuration references an admin-connected (BYO) model.
+
+    Requires **both** markers — ``byo_model`` (``"connection/deployment"``) and ``project_endpoint``
+    — to be present **non-empty strings**, because the prompty branch needs the project endpoint to
+    route the call. Requiring both matches the control-plane contract (BYO is active iff both markers
+    are present) and avoids a ``KeyError`` when only a partial config is supplied. Enforcing the string
+    type prevents truthy-but-invalid values (e.g. ``{"byo_model": 1, "project_endpoint": 2}``) from
+    bypassing the evaluator's normal model-config validation and failing deep inside the client instead.
+    """
+    if not model_config:
+        return False
+    byo_model = model_config.get("byo_model")
+    project_endpoint = model_config.get("project_endpoint")
+    return (
+        isinstance(byo_model, str) and bool(byo_model) and isinstance(project_endpoint, str) and bool(project_endpoint)
+    )
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
index b54677cf1b13..35deefedb3fb 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
@@ -18,6 +18,7 @@
 from azure.ai.evaluation._model_configurations import (
     AzureAIProject,
     AzureOpenAIModelConfiguration,
+    _BYOModelConfiguration,
     OpenAIModelConfiguration,
 )
 
@@ -222,7 +223,7 @@ def parse_model_config_type(
 
 
 def construct_prompty_model_config(
-    model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+    model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration, _BYOModelConfiguration],
     default_api_version: str,
     user_agent: str,
 ) -> dict:
@@ -309,7 +310,15 @@ def validate_azure_ai_project(o: object) -> AzureAIProject:
     return cast(AzureAIProject, o)
 
 
-def validate_model_config(config: dict) -> Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration]:
+def validate_model_config(
+    config: dict,
+) -> Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration, _BYOModelConfiguration]:
+    # BYO judge configs (connection/deployment) omit azure_endpoint/azure_deployment and route via
+    # the Foundry Responses API, so skip the AzureOpenAI/OpenAI TypedDict validation.
+    from azure.ai.evaluation._byo_judge import is_byo_model_config
+
+    if is_byo_model_config(config):
+        return cast(_BYOModelConfiguration, config)
     try:
         return _validate_typed_dict(config, AzureOpenAIModelConfiguration)
     except TypeError:
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
index 217514bf5f2e..87876d997321 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_legacy/prompty/_prompty.py
@@ -25,6 +25,7 @@
     WrappedOpenAIError,
 )
 from azure.ai.evaluation._legacy.prompty._connection import AzureOpenAIConnection, Connection, OpenAIConnection
+from azure.ai.evaluation._byo_judge import AsyncByoProjectResponsesClient, is_byo_model_config
 from azure.ai.evaluation._legacy.prompty._yaml_utils import load_yaml_string
 from azure.ai.evaluation._legacy.prompty._utils import (
     dataclass_from_dict,
@@ -279,7 +280,8 @@ async def __call__(  # pylint: disable=docstring-keyword-should-match-keyword-on
         """
 
         inputs = self._resolve_inputs(kwargs)
-        connection = Connection.parse_from_config(self._model.configuration)
+        configuration = self._model.configuration
+        is_byo = is_byo_model_config(configuration)
         messages = build_messages(prompt=self._template, working_dir=self.path.parent, **inputs)
         params = prepare_open_ai_request_params(self._model, messages)
 
@@ -293,31 +295,45 @@ async def __call__(  # pylint: disable=docstring-keyword-should-match-keyword-on
 
         default_headers = {"User-Agent": UserAgentSingleton().value}
 
-        api_client: Union[AsyncAzureOpenAI, AsyncOpenAI]
-        if isinstance(connection, AzureOpenAIConnection):
-            api_client = AsyncAzureOpenAI(
-                azure_endpoint=connection.azure_endpoint,
-                api_key=connection.api_key,
-                azure_deployment=connection.azure_deployment,
-                api_version=connection.api_version,
-                max_retries=max_retries,
-                azure_ad_token_provider=(
-                    self.get_token_provider(self._token_credential) if not connection.api_key else None
-                ),
-                default_headers=default_headers,
-            )
-        elif isinstance(connection, OpenAIConnection):
-            api_client = AsyncOpenAI(
-                base_url=connection.base_url,
-                api_key=connection.api_key,
-                organization=connection.organization,
-                max_retries=max_retries,
-                default_headers=default_headers,
+        api_client: Union[AsyncAzureOpenAI, AsyncOpenAI, AsyncByoProjectResponsesClient]
+        if is_byo:
+            # BYO judge model (connection/deployment): route chat.completions through the Foundry
+            # Responses API, which resolves the connection and auth (API key / managed identity / OAuth2).
+            # Merge SDK default headers (e.g. User-Agent) with caller extra_headers (caller wins),
+            # matching the non-BYO paths' default_headers.
+            byo_headers = {**default_headers, **(configuration.get("extra_headers") or {})}
+            api_client = AsyncByoProjectResponsesClient(
+                byo_model=configuration["byo_model"],
+                project_endpoint=configuration["project_endpoint"],
+                credential=self._token_credential,
+                extra_headers=byo_headers,
             )
         else:
-            raise NotSupportedError(
-                f"'{type(connection).__name__}' is not a supported connection type.", target=ErrorTarget.EVAL_RUN
-            )
+            connection = Connection.parse_from_config(configuration)
+            if isinstance(connection, AzureOpenAIConnection):
+                api_client = AsyncAzureOpenAI(
+                    azure_endpoint=connection.azure_endpoint,
+                    api_key=connection.api_key,
+                    azure_deployment=connection.azure_deployment,
+                    api_version=connection.api_version,
+                    max_retries=max_retries,
+                    azure_ad_token_provider=(
+                        self.get_token_provider(self._token_credential) if not connection.api_key else None
+                    ),
+                    default_headers=default_headers,
+                )
+            elif isinstance(connection, OpenAIConnection):
+                api_client = AsyncOpenAI(
+                    base_url=connection.base_url,
+                    api_key=connection.api_key,
+                    organization=connection.organization,
+                    max_retries=max_retries,
+                    default_headers=default_headers,
+                )
+            else:
+                raise NotSupportedError(
+                    f"'{type(connection).__name__}' is not a supported connection type.", target=ErrorTarget.EVAL_RUN
+                )
 
         response: OpenAIChatResponseType = await self._send_with_retries(
             api_client=api_client,
@@ -350,7 +366,7 @@ def render(  # pylint: disable=docstring-keyword-should-match-keyword-only
 
     async def _send_with_retries(
         self,
-        api_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
+        api_client: Union[AsyncAzureOpenAI, AsyncOpenAI, AsyncByoProjectResponsesClient],
         params: Mapping[str, Any],
         timeout: Optional[float],
         max_retries: int = 10,
@@ -358,7 +374,7 @@ async def _send_with_retries(
     ) -> OpenAIChatResponseType:
         """Send the request with retries.
 
-        :param Union[AsyncAzureOpenAI, AsyncOpenAI] api_client: The OpenAI client.
+        :param Union[AsyncAzureOpenAI, AsyncOpenAI, AsyncByoProjectResponsesClient] api_client: The OpenAI client.
         :param Mapping[str, Any] params: The request parameters.
         :param Optional[float] timeout: The timeout for the request.
         :param int max_retries: The maximum number of retries.
@@ -368,7 +384,9 @@ async def _send_with_retries(
         """
 
         client_name: str = api_client.__class__.__name__
-        client: Union[AsyncAzureOpenAI, AsyncOpenAI] = api_client.with_options(timeout=timeout or NotGiven())
+        client: Union[AsyncAzureOpenAI, AsyncOpenAI, AsyncByoProjectResponsesClient] = api_client.with_options(
+            timeout=timeout or NotGiven()
+        )
 
         entity_retries: List[int] = [0]
         should_retry: bool = True
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py
index 26ec432d04f1..d922d597fb93 100644
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py
@@ -83,6 +83,30 @@ class OpenAIModelConfiguration(TypedDict):
     extra_headers: NotRequired[Dict[str, str]]
 
 
+class _BYOModelConfiguration(TypedDict):
+    """Internal shape of an admin-connected (bring-your-own / BYO) model configuration.
+
+    This is service-internal plumbing, not a caller-facing type. BYO configs are constructed
+    server-side by the Foundry platform (the evaluation data plane resolves an admin-connected
+    ``ModelGateway`` / ``ApiManagement`` connection and stamps these markers), then routed through
+    the Foundry project Responses API where the platform owns connection resolution and auth. It is
+    intentionally not exported from ``azure.ai.evaluation``; the public model-config union will be
+    widened to advertise BYO once the feature reaches GA.
+
+    :param byo_model: The admin-connected model reference, ``"connection-name/deployment-name"``.
+    :type byo_model: str
+    :param project_endpoint: The Foundry project endpoint,
+        ``https://<account>.services.ai.azure.com/api/projects/<project>``.
+    :type project_endpoint: str
+    :param extra_headers: Additional HTTP headers to include in every request. Optional.
+    :type extra_headers: NotRequired[Dict[str, str]]
+    """
+
+    byo_model: str
+    project_endpoint: str
+    extra_headers: NotRequired[Dict[str, str]]
+
+
 class AzureAIProject(TypedDict):
     """Information about the Azure AI project
 
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
new file mode 100644
index 000000000000..e35eca66592e
--- /dev/null
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_byo_judge.py
@@ -0,0 +1,366 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+"""Unit tests for admin-connected (BYO) judge-model support in the prompty judge path."""
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from azure.ai.evaluation._byo_judge import (
+    AsyncByoProjectResponsesClient,
+    is_byo_model_config,
+    _to_responses_input,
+    _map_params,
+    _map_response_format,
+    _ChatCompletion,
+    _Usage,
+)
+
+
+class TestByoJudgeHelpers:
+    def test_is_byo_model_config(self):
+        assert is_byo_model_config({"byo_model": "c/d", "project_endpoint": "https://x"})
+        assert not is_byo_model_config({"azure_endpoint": "https://x"})
+        assert not is_byo_model_config({})
+        # Both markers required — byo_model alone must not activate BYO (prompty needs project_endpoint).
+        assert not is_byo_model_config({"byo_model": "c/d"})
+        assert not is_byo_model_config({"project_endpoint": "https://x"})
+        # Markers must be non-empty strings; truthy-but-invalid values (e.g. ints) must not activate BYO.
+        assert not is_byo_model_config({"byo_model": 1, "project_endpoint": 2})
+        assert not is_byo_model_config({"byo_model": "", "project_endpoint": "https://x"})
+        assert not is_byo_model_config({"byo_model": "c/d", "project_endpoint": ""})
+
+    def test_to_responses_input(self):
+        assert _to_responses_input([{"role": "user", "content": "hi"}]) == [
+            {"type": "message", "role": "user", "content": "hi"}
+        ]
+
+    def test_map_params_curates_and_renames(self):
+        assert _map_params({"temperature": 0.0, "max_tokens": 50, "top_p": 0.9, "stream": True}) == {
+            "temperature": 0.0,
+            "top_p": 0.9,
+            "max_output_tokens": 50,
+        }
+
+
+class TestResponseFormatMapping:
+    """chat.completions ``response_format`` -> Responses API ``text.format`` (JSON output parity)."""
+
+    def test_json_object_passthrough(self):
+        assert _map_response_format({"type": "json_object"}) == {"type": "json_object"}
+
+    def test_text_passthrough(self):
+        assert _map_response_format({"type": "text"}) == {"type": "text"}
+
+    def test_json_schema_is_flattened(self):
+        # chat.completions nests name/schema/strict under "json_schema"; Responses flattens them.
+        rf = {
+            "type": "json_schema",
+            "json_schema": {
+                "name": "coherence",
+                "schema": {"type": "object", "properties": {"score": {"type": "integer"}}},
+                "strict": True,
+            },
+        }
+        assert _map_response_format(rf) == {
+            "type": "json_schema",
+            "name": "coherence",
+            "schema": {"type": "object", "properties": {"score": {"type": "integer"}}},
+            "strict": True,
+        }
+
+    def test_unknown_or_non_dict_returns_none(self):
+        assert _map_response_format({"type": "weird"}) is None
+        assert _map_response_format(None) is None
+        assert _map_response_format("json_object") is None
+
+    def test_map_params_wires_response_format_into_text(self):
+        # The prompty runner passes response_format={"type": "json_object"}; it must reach text.format.
+        mapped = _map_params({"temperature": 0.0, "response_format": {"type": "json_object"}})
+        assert mapped["text"] == {"format": {"type": "json_object"}}
+        assert mapped["temperature"] == 0.0
+
+    def test_map_params_omits_text_for_unrecognized_response_format(self):
+        assert "text" not in _map_params({"response_format": {"type": "weird"}})
+
+
+class TestUsageAdapter:
+    def test_responses_usage_mapped_to_chat_shape(self):
+        usage = _Usage(MagicMock(input_tokens=5, output_tokens=7, total_tokens=12))
+        assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (5, 7, 12)
+
+    def test_missing_fields_default_to_zero(self):
+        usage = _Usage(object())
+        assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (0, 0, 0)
+
+    def test_total_tokens_falls_back_to_prompt_plus_completion(self):
+        # Responses usage without total_tokens -> compute it from prompt + completion.
+        usage = _Usage(SimpleNamespace(input_tokens=5, output_tokens=7))
+        assert usage.total_tokens == 12
+
+
+class TestFinishReason:
+    """A Responses result maps status -> chat.completions finish_reason (truncation detection)."""
+
+    @staticmethod
+    def _resp(**kwargs):
+        kwargs.setdefault("output_text", "ok")
+        kwargs.setdefault("usage", None)
+        kwargs.setdefault("id", "r")
+        kwargs.setdefault("model", "m")
+        return SimpleNamespace(**kwargs)
+
+    def test_completed_is_stop(self):
+        assert _ChatCompletion(self._resp(status="completed")).choices[0].finish_reason == "stop"
+
+    def test_missing_status_defaults_to_stop(self):
+        assert _ChatCompletion(self._resp()).choices[0].finish_reason == "stop"
+
+    def test_truncation_is_length(self):
+        resp = self._resp(status="incomplete", incomplete_details=SimpleNamespace(reason="max_output_tokens"))
+        assert _ChatCompletion(resp).choices[0].finish_reason == "length"
+
+    def test_other_incomplete_reason_passes_through(self):
+        resp = self._resp(status="incomplete", incomplete_details=SimpleNamespace(reason="content_filter"))
+        assert _ChatCompletion(resp).choices[0].finish_reason == "content_filter"
+
+
+class TestAsyncByoProjectResponsesClient:
+    """The prompty judge path (coherence/relevance/fluency/... LLM-as-a-judge evaluators)."""
+
+    @patch("azure.ai.projects.aio.AIProjectClient")
+    def test_async_chat_completions_routes_to_responses(self, mock_aipc):
+        resp = MagicMock()
+        resp.output_text = "coherent: 5"
+        resp.usage = MagicMock(input_tokens=12, output_tokens=3, total_tokens=15)
+        resp.id = "resp_2"
+        resp.model = "my-conn/gpt-4o"
+        oai = MagicMock()
+        oai.responses.create = AsyncMock(return_value=resp)
+        mock_aipc.return_value.get_openai_client.return_value = oai
+
+        client = AsyncByoProjectResponsesClient(
+            byo_model="my-conn/gpt-4o",
+            project_endpoint="https://acct.services.ai.azure.com/api/projects/p1",
+            credential=MagicMock(),
+        )
+        # The prompty runner calls with_options(...).chat.completions.create(...).
+        result = asyncio.run(
+            client.with_options(timeout=30).chat.completions.create(
+                model="ignored",
+                messages=[{"role": "user", "content": "rate coherence"}],
+                temperature=0.0,
+                max_tokens=800,
+            )
+        )
+
+        # chat.completions-shaped result the prompty response formatter expects.
+        assert result.choices[0].message.content == "coherent: 5"
+        assert result.choices[0].message.role == "assistant"
+        assert result.choices[0].finish_reason == "stop"
+        assert result.usage.prompt_tokens == 12
+        assert result.usage.completion_tokens == 3
+        assert result.usage.total_tokens == 15
+        assert result.model == "my-conn/gpt-4o"
+
+        # Underlying call is the project Responses API with the BYO model + mapped input/params.
+        mock_aipc.assert_called_once()
+        _, pkwargs = mock_aipc.call_args
+        assert pkwargs["endpoint"] == "https://acct.services.ai.azure.com/api/projects/p1"
+        # max_retries=0 so AsyncPrompty's own retry loop is the only retry layer.
+        _, gkwargs = mock_aipc.return_value.get_openai_client.call_args
+        assert gkwargs["max_retries"] == 0
+        _, rkwargs = oai.responses.create.call_args
+        assert rkwargs["model"] == "my-conn/gpt-4o"
+        assert rkwargs["input"] == [{"type": "message", "role": "user", "content": "rate coherence"}]
+        assert rkwargs["temperature"] == 0.0
+        assert rkwargs["max_output_tokens"] == 800
+        # The per-request timeout set via with_options(timeout=30) reaches responses.create.
+        assert rkwargs["timeout"] == 30
+
+    def test_non_numeric_timeout_is_not_forwarded(self):
+        # openai passes NotGiven() when no timeout is set; the shim forwards only a numeric timeout.
+        client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
+        client.with_options(timeout=object())
+        assert client._timeout is None
+
+    @patch("azure.ai.projects.aio.AIProjectClient")
+    def test_awaits_coroutine_get_openai_client(self, mock_aipc):
+        # In some azure-ai-projects versions get_openai_client() is a coroutine; the shim must await it.
+        resp = MagicMock(output_text="ok", usage=None, id="r", model="m", status="completed")
+        oai = MagicMock()
+        oai.responses.create = AsyncMock(return_value=resp)
+
+        async def _coro_get_openai_client(*args, **kwargs):
+            return oai
+
+        mock_aipc.return_value.get_openai_client = _coro_get_openai_client
+
+        client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
+        result = asyncio.run(client.chat.completions.create(messages=[{"role": "user", "content": "hi"}]))
+        assert result.choices[0].message.content == "ok"
+        oai.responses.create.assert_awaited_once()
+
+    def test_with_options_returns_self(self):
+        client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
+        assert client.with_options(timeout=5) is client
+
+    def test_missing_azure_ai_projects_raises_clear_error(self, monkeypatch):
+        # azure-ai-projects is optional (not in install_requires); the BYO path must surface a clear
+        # MissingRequiredPackage error. A None entry in sys.modules makes the import raise ImportError.
+        import sys
+
+        from azure.ai.evaluation._legacy._adapters._errors import MissingRequiredPackage
+
+        monkeypatch.setitem(sys.modules, "azure.ai.projects.aio", None)
+        client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
+        with pytest.raises(MissingRequiredPackage):
+            asyncio.run(client.chat.completions.create(messages=[{"role": "user", "content": "hi"}]))
+
+    @patch("azure.ai.projects.aio.AIProjectClient")
+    def test_extra_headers_forwarded_to_responses(self, mock_aipc):
+        # ACA may supply extra headers for continuous evals; the shim must forward them to responses.create.
+        resp = MagicMock(output_text="ok", usage=None, id="r", model="m", status="completed")
+        oai = MagicMock()
+        oai.responses.create = AsyncMock(return_value=resp)
+        mock_aipc.return_value.get_openai_client.return_value = oai
+
+        headers = {"x-ms-client-request-id": "abc-123", "x-correlation-id": "def-456"}
+        client = AsyncByoProjectResponsesClient(
+            byo_model="c/d",
+            project_endpoint="https://acct.services.ai.azure.com/api/projects/p",
+            credential=MagicMock(),
+            extra_headers=headers,
+        )
+        asyncio.run(client.chat.completions.create(messages=[{"role": "user", "content": "hi"}]))
+
+        _, rkwargs = oai.responses.create.call_args
+        assert rkwargs["extra_headers"] == headers
+        # The shim copies the headers so later caller mutations do not leak into the client.
+        assert rkwargs["extra_headers"] is not headers
+
+    @patch("azure.ai.projects.aio.AIProjectClient")
+    def test_no_extra_headers_by_default(self, mock_aipc):
+        # When no extra headers are supplied, none are forwarded to responses.create.
+        resp = MagicMock(output_text="ok", usage=None, id="r", model="m", status="completed")
+        oai = MagicMock()
+        oai.responses.create = AsyncMock(return_value=resp)
+        mock_aipc.return_value.get_openai_client.return_value = oai
+
+        client = AsyncByoProjectResponsesClient("c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock())
+        asyncio.run(client.chat.completions.create(messages=[{"role": "user", "content": "hi"}]))
+
+        _, rkwargs = oai.responses.create.call_args
+        assert "extra_headers" not in rkwargs
+
+    @patch("azure.ai.projects.aio.AIProjectClient")
+    def test_per_request_extra_headers_merged_and_forwarded(self, mock_aipc):
+        # Headers passed per-request to chat.completions.create(extra_headers=...) must be forwarded
+        # to responses.create, merged over client-level headers (per-request wins), as a copy.
+        resp = MagicMock(output_text="ok", usage=None, id="r", model="m", status="completed")
+        oai = MagicMock()
+        oai.responses.create = AsyncMock(return_value=resp)
+        mock_aipc.return_value.get_openai_client.return_value = oai
+
+        client_headers = {"x-ms-client-request-id": "client-1", "x-shared": "from-client"}
+        request_headers = {"Connection": "close", "x-shared": "from-request"}
+        client = AsyncByoProjectResponsesClient(
+            "c/d", "https://acct.services.ai.azure.com/api/projects/p", MagicMock(), extra_headers=client_headers
+        )
+        asyncio.run(
+            client.chat.completions.create(messages=[{"role": "user", "content": "hi"}], extra_headers=request_headers)
+        )
+
+        _, rkwargs = oai.responses.create.call_args
+        fwd = rkwargs["extra_headers"]
+        assert fwd == {"x-ms-client-request-id": "client-1", "Connection": "close", "x-shared": "from-request"}
+        # Per-request value wins on conflict.
+        assert fwd["x-shared"] == "from-request"
+        # Forwarded as a copy — neither source dict is the forwarded object (no mutation leak).
+        assert fwd is not client_headers and fwd is not request_headers
+
+    @patch("azure.ai.projects.aio.AIProjectClient")
+    def test_preserves_server_created_timestamp(self, mock_aipc):
+        # Responses exposes the server timestamp as created_at; the shim preserves it (not local time).
+        cc = _ChatCompletion(SimpleNamespace(output_text="ok", usage=None, id="r", model="m", created_at=1752694800))
+        assert cc.created == 1752694800
+
+    def test_created_timestamp_falls_back_to_created_then_now(self):
+        # Older/mocked shapes may only carry ``created``; that is honored as a fallback.
+        assert _ChatCompletion(SimpleNamespace(output_text="ok", created=1700000000)).created == 1700000000
+        # When neither is a real number, fall back to the local wall-clock time (a positive int).
+        assert _ChatCompletion(SimpleNamespace(output_text="ok")).created > 0
+
+
+class TestValidateModelConfigByo:
+    def test_byo_config_passes_through_validation(self):
+        from azure.ai.evaluation._common.utils import validate_model_config
+
+        cfg = {
+            "byo_model": "my-conn/gpt-4o",
+            "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p1",
+        }
+        # BYO configs intentionally omit azure_endpoint/azure_deployment and must not be rejected.
+        assert validate_model_config(cfg) is cfg
+
+    def test_non_byo_invalid_config_still_raises(self):
+        from azure.ai.evaluation._common.utils import validate_model_config
+        from azure.ai.evaluation._exceptions import EvaluationException
+
+        with pytest.raises(EvaluationException):
+            validate_model_config({"not": "a valid model config"})
+
+
+class TestAsyncPromptyByoIntegration:
+    """Exercise the BYO branch through AsyncPrompty end-to-end (not just the shim in isolation)."""
+
+    _PROMPTY = (
+        "---\n"
+        "name: byo-judge\n"
+        "model:\n"
+        "  api: chat\n"
+        "  parameters:\n"
+        "    temperature: 0.0\n"
+        "inputs:\n"
+        "  question:\n"
+        "    type: string\n"
+        "---\n"
+        "system:\n"
+        "You are a judge.\n"
+        "\n"
+        "user:\n"
+        "{{question}}\n"
+    )
+
+    @patch("azure.ai.projects.aio.AIProjectClient")
+    def test_async_prompty_routes_byo_config_through_shim(self, mock_aipc, tmp_path):
+        from azure.ai.evaluation._legacy.prompty._prompty import AsyncPrompty
+
+        prompty_path = tmp_path / "byo_judge.prompty"
+        prompty_path.write_text(self._PROMPTY, encoding="utf-8")
+
+        resp = SimpleNamespace(output_text="score: 5", usage=None, id="r", model="conn/dep", created_at=1)
+        oai = MagicMock()
+        oai.responses.create = AsyncMock(return_value=resp)
+        mock_aipc.return_value.get_openai_client.return_value = oai
+
+        model = {
+            "configuration": {
+                "byo_model": "conn/dep",
+                "project_endpoint": "https://acct.services.ai.azure.com/api/projects/p",
+            },
+            "parameters": {"temperature": 0.0},
+        }
+        flow = AsyncPrompty.load(source=str(prompty_path), model=model, token_credential=MagicMock())
+        result = asyncio.run(flow(question="rate this"))
+
+        # The BYO branch built the project client with the project endpoint and routed to responses.
+        mock_aipc.assert_called_once()
+        _, pkwargs = mock_aipc.call_args
+        assert pkwargs["endpoint"] == "https://acct.services.ai.azure.com/api/projects/p"
+        _, rkwargs = oai.responses.create.call_args
+        assert rkwargs["model"] == "conn/dep"
+        # The formatted judge output flows back through AsyncPrompty's response formatting.
+        assert "score: 5" in str(result)
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
index 27e35bd88ab6..2ac7d282f237 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
@@ -551,11 +551,14 @@ def test_evaluate_output_dir_not_exist(self, mock_model_config, questions_file):
         assert "The output directory './not_exist_dir' does not exist." in exc_info.value.args[0]
 
     @pytest.mark.parametrize("use_relative_path", [True, False])
-    def test_evaluate_output_path(self, evaluate_test_data_jsonl_file, tmpdir, use_relative_path):
+    def test_evaluate_output_path(self, evaluate_test_data_jsonl_file, tmpdir, monkeypatch, use_relative_path):
         # output_path is a file
         if use_relative_path:
             output_path = os.path.join(tmpdir, "eval_test_results.jsonl")
         else:
+            # A bare relative filename races across pytest-xdist workers sharing the CWD (intermittent
+            # FileNotFoundError on cleanup); chdir into the per-test tmpdir to isolate it.
+            monkeypatch.chdir(tmpdir)
             output_path = "eval_test_results.jsonl"
 
         result = evaluate(
@@ -2299,7 +2302,11 @@ def test_nan_threshold_gets_injected(self):
 
 
 try:
-    import opentelemetry  # noqa: F401
+    # These tests exercise the OpenTelemetry Events API (opentelemetry._events /
+    # opentelemetry.sdk._events, added in 1.26). Guard on those submodules, not just the top-level
+    # package: the sk_ CI leg pins an older opentelemetry (via semantic-kernel) that lacks them.
+    import opentelemetry._events  # noqa: F401
+    from opentelemetry.sdk._events import EventLoggerProvider  # noqa: F401
 
     MISSING_OPENTELEMETRY = False
 except ImportError: