Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions py/src/braintrust/parameters.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,12 +134,14 @@ def parameters_to_json_schema(parameters: EvalParameters) -> dict[str, Any]:

for name, schema in parameters.items():
if isinstance(schema, dict) and schema.get("type") == "prompt":
# Prompt parameter
result[name] = {
"type": "prompt",
"default": schema.get("default"),
"description": schema.get("description"),
}
# Prompt parameter - only include optional fields if they have values
# to avoid serializing None as null which Zod .optional() rejects
param_data: dict[str, Any] = {"type": "prompt"}
if schema.get("default") is not None:
param_data["default"] = schema["default"]
if schema.get("description") is not None:
param_data["description"] = schema["description"]
result[name] = param_data
else:
# Pydantic model
try:
Expand Down
111 changes: 111 additions & 0 deletions py/src/braintrust/test_parameters.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
"""Tests for parameters module."""

import json
import pytest
from .parameters import parameters_to_json_schema


def test_parameters_to_json_schema_omits_null_description():
"""Test that description field is omitted when not set (not serialized as null).

This is important because the frontend Zod schema uses .optional() which
accepts undefined but rejects null values.

Regression test for: Remote eval "Not connected" when description is missing
"""
params = {
"main": {
"type": "prompt",
"name": "Main prompt",
# description intentionally omitted
"default": {
"prompt": {
"type": "chat",
"messages": [{"role": "user", "content": "{{input}}"}],
},
"options": {"model": "gpt-4o"},
},
}
}

result = parameters_to_json_schema(params)

# Verify the structure is correct
assert "main" in result
assert result["main"]["type"] == "prompt"
assert "default" in result["main"]

# Critical: description should NOT be present (not even as None/null)
assert "description" not in result["main"], \
"description should be omitted when not set, not serialized as null"

# Verify it serializes to JSON without null description
json_str = json.dumps(result)
assert '"description": null' not in json_str


def test_parameters_to_json_schema_includes_description_when_set():
"""Test that description field is included when explicitly set."""
params = {
"main": {
"type": "prompt",
"name": "Main prompt",
"description": "This is the main prompt",
"default": {
"prompt": {
"type": "chat",
"messages": [{"role": "user", "content": "{{input}}"}],
},
"options": {"model": "gpt-4o"},
},
}
}

result = parameters_to_json_schema(params)

assert result["main"]["description"] == "This is the main prompt"


def test_parameters_to_json_schema_omits_null_default():
"""Test that default field is omitted when not set."""
params = {
"main": {
"type": "prompt",
"name": "Main prompt",
# default intentionally omitted
}
}

result = parameters_to_json_schema(params)

assert "main" in result
assert result["main"]["type"] == "prompt"
assert "default" not in result["main"], \
"default should be omitted when not set, not serialized as null"


def test_parameters_to_json_schema_includes_both_when_set():
"""Test that both description and default are included when set."""
params = {
"scoring_prompt": {
"type": "prompt",
"name": "Scoring Prompt",
"description": "The prompt used for scoring",
"default": {
"prompt": {
"type": "chat",
"messages": [
{"role": "system", "content": "You are a scorer."},
{"role": "user", "content": "Score this: {{input}}"},
],
},
"options": {"model": "claude-opus-4-20250514"},
},
}
}

result = parameters_to_json_schema(params)

assert result["scoring_prompt"]["type"] == "prompt"
assert result["scoring_prompt"]["description"] == "The prompt used for scoring"
assert result["scoring_prompt"]["default"]["prompt"]["type"] == "chat"
Loading