diff --git a/generated_types.json b/generated_types.json index 8755844e9..9f0baa3bd 100644 --- a/generated_types.json +++ b/generated_types.json @@ -337,6 +337,9 @@ }, "use_cache": { "type": "boolean" + }, + "use_native_inference": { + "type": "boolean" } }, "required": [ @@ -558,6 +561,26 @@ "function_ids", "triggered_xact_id" ] + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "mark_attempt_failed" + ] + }, + "function_ids": { + "type": "array", + "items": {}, + "minItems": 1 + } + }, + "required": [ + "kind", + "function_ids" + ] } ] }, @@ -748,6 +771,9 @@ "use_cache": { "type": "boolean" }, + "use_native_inference": { + "type": "boolean" + }, "reasoning_enabled": { "type": "boolean" }, @@ -1853,12 +1879,17 @@ ], "additionalProperties": {}, "description": "User-controlled metadata about the dataset" + }, + "url_slug": { + "type": "string", + "description": "URL slug for the dataset. used to construct dataset URLs" } }, "required": [ "id", "project_id", - "name" + "name", + "url_slug" ] }, "DatasetEvent": { @@ -1929,7 +1960,7 @@ }, "span_id": { "type": "string", - "description": "A unique identifier used to link different dataset events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/guides/tracing) for full details on tracing" + "description": "A unique identifier used to link different dataset events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing" }, "root_span_id": { "type": "string", @@ -2046,6 +2077,150 @@ "name" ] }, + "EvalStatusPage": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "description": "Unique identifier for the eval status page" + }, + "project_id": { + "type": "string", + "format": "uuid", + "description": "Unique identifier for the project that the eval status page belongs under" + }, + "user_id": { + "type": [ + "string", + "null" + ], + "format": "uuid", + "description": "Identifies the user who created the eval status page" + }, + "created": { + "type": [ + "string", + "null" + ], + "format": "date-time", + "description": "Date of eval status page creation" + }, + "deleted_at": { + "type": [ + "string", + "null" + ], + "format": "date-time", + "description": "Date of eval status page deletion, or null if the eval status page is still active" + }, + "name": { + "type": "string", + "description": "Name of the eval status page" + }, + "description": { + "type": [ + "string", + "null" + ], + "description": "Textual description of the eval status page" + }, + "logo_url": { + "type": [ + "string", + "null" + ], + "description": "URL of the logo to display on the page" + }, + "theme": { + "$ref": "#/components/schemas/EvalStatusPageTheme" + }, + "config": { + "$ref": "#/components/schemas/EvalStatusPageConfig" + } + }, + "required": [ + "id", + "project_id", + "name", + "theme", + "config" + ], + "description": "A public eval status page that displays aggregate experiment results" + }, + "EvalStatusPageConfig": { + "type": "object", + "properties": { + "score_columns": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "description": "The score columns to display on the page" + }, + "metric_columns": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "description": "The metric columns to display on the page" + }, + "grouping_field": { + "type": [ + "string", + "null" + ], + "description": "The metadata field to use for grouping experiments (model)" + }, + "filter": { + "type": [ + "string", + "null" + ], + "description": "BTQL filter to apply to experiment data" + }, + "sort_by": { + "type": [ + "string", + "null" + ], + "description": "Field to sort results by (format: 'score:' or 'metric:')" + }, + "sort_order": { + "type": [ + "string", + "null" + ], + "enum": [ + "asc", + "desc" + ], + "description": "Sort order (ascending or descending)" + }, + "api_key": { + "type": [ + "string", + "null" + ], + "description": "The API key used for fetching experiment data" + } + }, + "description": "Configuration for what data to display" + }, + "EvalStatusPageTheme": { + "type": "string", + "enum": [ + "light", + "dark" + ], + "description": "The theme for the page" + }, "Experiment": { "type": "object", "properties": { @@ -2333,7 +2508,7 @@ }, "span_id": { "type": "string", - "description": "A unique identifier used to link different experiment events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/guides/tracing) for full details on tracing" + "description": "A unique identifier used to link different experiment events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing" }, "span_parents": { "type": [ @@ -3038,6 +3213,7 @@ "tool", "scorer", "task", + "workflow", "custom_view", "preprocessor", "facet" @@ -3903,6 +4079,9 @@ "use_cache": { "type": "boolean" }, + "use_native_inference": { + "type": "boolean" + }, "reasoning_enabled": { "type": "boolean" }, @@ -4050,6 +4229,9 @@ "use_cache": { "type": "boolean" }, + "use_native_inference": { + "type": "boolean" + }, "reasoning_enabled": { "type": "boolean" }, @@ -4092,6 +4274,9 @@ "use_cache": { "type": "boolean" }, + "use_native_inference": { + "type": "boolean" + }, "reasoning_enabled": { "type": "boolean" }, @@ -4120,6 +4305,9 @@ "use_cache": { "type": "boolean" }, + "use_native_inference": { + "type": "boolean" + }, "reasoning_enabled": { "type": "boolean" }, @@ -4142,6 +4330,9 @@ "use_cache": { "type": "boolean" }, + "use_native_inference": { + "type": "boolean" + }, "reasoning_enabled": { "type": "boolean" }, @@ -5038,7 +5229,7 @@ }, "span_id": { "type": "string", - "description": "A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/guides/tracing) for full details on tracing" + "description": "A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing" }, "span_parents": { "type": [ @@ -7503,7 +7694,7 @@ "license": { "name": "Apache 2.0" }, - "x-internal-git-sha": "87ac73f4945a47eff2d4e42775ba4dbc58854c73" + "x-internal-git-sha": "33ab31eaa0c3d1873a3b877c52d0a3a20015f939" }, "paths": {}, "webhooks": {} diff --git a/js/src/generated_types.ts b/js/src/generated_types.ts index 4389ddd66..f52cc6ae8 100644 --- a/js/src/generated_types.ts +++ b/js/src/generated_types.ts @@ -1,4 +1,4 @@ -// Auto-generated file (internal git SHA 87ac73f4945a47eff2d4e42775ba4dbc58854c73) -- do not modify +// Auto-generated file (internal git SHA 33ab31eaa0c3d1873a3b877c52d0a3a20015f939) -- do not modify import { z } from "zod/v3"; @@ -118,6 +118,7 @@ export const AnyModelParams = z.object({ topP: z.number().optional(), topK: z.number().optional(), use_cache: z.boolean().optional(), + use_native_inference: z.boolean().optional(), }); export type AnyModelParamsType = z.infer; export const ApiKey = z.object({ @@ -182,6 +183,10 @@ export const AsyncScoringControl = z.union([ function_ids: z.array(z.unknown()).min(1), triggered_xact_id: z.string(), }), + z.object({ + kind: z.literal("mark_attempt_failed"), + function_ids: z.array(z.unknown()).min(1), + }), ]); export type AsyncScoringControlType = z.infer; export const BraintrustAttachmentReference = z.object({ @@ -252,6 +257,7 @@ export type BatchedFacetDataType = z.infer; export const BraintrustModelParams = z .object({ use_cache: z.boolean(), + use_native_inference: z.boolean(), reasoning_enabled: z.boolean(), reasoning_budget: z.number(), }) @@ -491,6 +497,7 @@ export const Dataset = z.object({ metadata: z .union([z.object({}).partial().passthrough(), z.null()]) .optional(), + url_slug: z.string(), }); export type DatasetType = z.infer; export const ObjectReferenceNullish = z.union([ @@ -555,6 +562,33 @@ export const EnvVar = z.object({ .default("env_var"), }); export type EnvVarType = z.infer; +export const EvalStatusPageTheme = z.enum(["light", "dark"]); +export type EvalStatusPageThemeType = z.infer; +export const EvalStatusPageConfig = z + .object({ + score_columns: z.union([z.array(z.string()), z.null()]), + metric_columns: z.union([z.array(z.string()), z.null()]), + grouping_field: z.union([z.string(), z.null()]), + filter: z.union([z.string(), z.null()]), + sort_by: z.union([z.string(), z.null()]), + sort_order: z.union([z.enum(["asc", "desc"]), z.null()]), + api_key: z.union([z.string(), z.null()]), + }) + .partial(); +export type EvalStatusPageConfigType = z.infer; +export const EvalStatusPage = z.object({ + id: z.string().uuid(), + project_id: z.string().uuid(), + user_id: z.union([z.string(), z.null()]).optional(), + created: z.union([z.string(), z.null()]).optional(), + deleted_at: z.union([z.string(), z.null()]).optional(), + name: z.string(), + description: z.union([z.string(), z.null()]).optional(), + logo_url: z.union([z.string(), z.null()]).optional(), + theme: EvalStatusPageTheme, + config: EvalStatusPageConfig, +}); +export type EvalStatusPageType = z.infer; export const RepoInfo = z.union([ z .object({ @@ -701,6 +735,7 @@ export const ModelParams = z.union([ z .object({ use_cache: z.boolean(), + use_native_inference: z.boolean(), reasoning_enabled: z.boolean(), reasoning_budget: z.number(), temperature: z.number(), @@ -734,6 +769,7 @@ export const ModelParams = z.union([ z .object({ use_cache: z.boolean().optional(), + use_native_inference: z.boolean().optional(), reasoning_enabled: z.boolean().optional(), reasoning_budget: z.number().optional(), max_tokens: z.number(), @@ -747,6 +783,7 @@ export const ModelParams = z.union([ z .object({ use_cache: z.boolean(), + use_native_inference: z.boolean(), reasoning_enabled: z.boolean(), reasoning_budget: z.number(), temperature: z.number(), @@ -759,6 +796,7 @@ export const ModelParams = z.union([ z .object({ use_cache: z.boolean(), + use_native_inference: z.boolean(), reasoning_enabled: z.boolean(), reasoning_budget: z.number(), temperature: z.number(), @@ -769,6 +807,7 @@ export const ModelParams = z.union([ z .object({ use_cache: z.boolean(), + use_native_inference: z.boolean(), reasoning_enabled: z.boolean(), reasoning_budget: z.number(), }) @@ -1109,6 +1148,7 @@ export const FunctionObjectType = z.enum([ "tool", "scorer", "task", + "workflow", "custom_view", "preprocessor", "facet", diff --git a/py/src/braintrust/_generated_types.py b/py/src/braintrust/_generated_types.py index ab6f7be1d..757f4c749 100644 --- a/py/src/braintrust/_generated_types.py +++ b/py/src/braintrust/_generated_types.py @@ -6,12 +6,11 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence -from typing import Any, Literal, TypeAlias, TypedDict +from typing import Any, Literal, Mapping, Sequence, TypedDict from typing_extensions import NotRequired -AclObjectType: TypeAlias = Literal[ +AclObjectType = Literal[ 'organization', 'project', 'experiment', @@ -24,9 +23,6 @@ 'project_log', 'org_project', ] -""" -The object type that the ACL applies to -""" class AISecret(TypedDict): @@ -140,14 +136,19 @@ class AsyncScoringControlAsyncScoringControl4(TypedDict): class AsyncScoringControlAsyncScoringControl5(TypedDict): kind: Literal['complete_triggered_functions'] - function_ids: Sequence[Any] + function_ids: Sequence triggered_xact_id: str +class AsyncScoringControlAsyncScoringControl6(TypedDict): + kind: Literal['mark_attempt_failed'] + function_ids: Sequence + + class AsyncScoringStateAsyncScoringState(TypedDict): status: Literal['enabled'] token: str - function_ids: Sequence[Any] + function_ids: Sequence skip_logging: NotRequired[bool | None] triggered_functions: NotRequired[Mapping[str, Any] | None] @@ -156,20 +157,7 @@ class AsyncScoringStateAsyncScoringState1(TypedDict): status: Literal['disabled'] -AsyncScoringState: TypeAlias = AsyncScoringStateAsyncScoringState | AsyncScoringStateAsyncScoringState1 | None - - -class PreprocessorPreprocessor(TypedDict): - type: Literal['function'] - id: str - - -class PreprocessorPreprocessor2(TypedDict): - pass - - -class PreprocessorPreprocessor3(PreprocessorPreprocessor, PreprocessorPreprocessor2): - pass +AsyncScoringState = AsyncScoringStateAsyncScoringState | AsyncScoringStateAsyncScoringState1 | None class BatchedFacetDataFacet(TypedDict): @@ -212,6 +200,7 @@ class BraintrustAttachmentReference(TypedDict): class BraintrustModelParams(TypedDict): use_cache: NotRequired[bool | None] + use_native_inference: NotRequired[bool | None] reasoning_enabled: NotRequired[bool | None] reasoning_budget: NotRequired[float | None] @@ -264,7 +253,7 @@ class CallEventCallEvent7(TypedDict): data: Literal[''] -CallEvent: TypeAlias = ( +CallEvent = ( CallEventCallEvent | CallEventCallEvent1 | CallEventCallEvent2 @@ -484,6 +473,10 @@ class Dataset(TypedDict): """ User-controlled metadata about the dataset """ + url_slug: str + """ + URL slug for the dataset. used to construct dataset URLs + """ class DatasetEventMetadata(TypedDict): @@ -532,6 +525,40 @@ class EnvVar(TypedDict): """ +class EvalStatusPageConfig(TypedDict): + score_columns: NotRequired[Sequence[str] | None] + """ + The score columns to display on the page + """ + metric_columns: NotRequired[Sequence[str] | None] + """ + The metric columns to display on the page + """ + grouping_field: NotRequired[str | None] + """ + The metadata field to use for grouping experiments (model) + """ + filter: NotRequired[str | None] + """ + BTQL filter to apply to experiment data + """ + sort_by: NotRequired[str | None] + """ + Field to sort results by (format: 'score:' or 'metric:') + """ + sort_order: NotRequired[Literal['asc', 'desc'] | None] + """ + Sort order (ascending or descending) + """ + api_key: NotRequired[str | None] + """ + The API key used for fetching experiment data + """ + + +EvalStatusPageTheme = Literal['light', 'dark'] + + class ExperimentEventMetadata(TypedDict): model: NotRequired[str | None] """ @@ -619,21 +646,8 @@ class ExternalAttachmentReference(TypedDict): """ -class Preprocessor1Preprocessor1(TypedDict): - type: Literal['function'] - id: str - - -class Preprocessor1Preprocessor12(TypedDict): - pass - - -class Preprocessor1Preprocessor13(Preprocessor1Preprocessor1, Preprocessor1Preprocessor12): - pass - - class FunctionOrigin(TypedDict): - object_type: AclObjectType + object_type: AclObjectType | None object_id: str """ Id of the object the function is originating from @@ -684,7 +698,7 @@ class FunctionDataFunctionData2(TypedDict): parameters: Mapping[str, Any] -FunctionFormat: TypeAlias = Literal['llm', 'code', 'global', 'graph'] +FunctionFormat = Literal['llm', 'code', 'global', 'graph'] class FunctionIdFunctionId(TypedDict): @@ -745,24 +759,19 @@ class FunctionIdFunctionId4(TypedDict): """ -FunctionIdRef: TypeAlias = Mapping[str, Any] +FunctionIdRef = Mapping[str, Any] -FunctionObjectType: TypeAlias = Literal[ - 'prompt', 'tool', 'scorer', 'task', 'custom_view', 'preprocessor', 'facet' -] +FunctionObjectType = Literal['prompt', 'tool', 'scorer', 'task', 'workflow', 'custom_view', 'preprocessor', 'facet'] -FunctionOutputType: TypeAlias = Literal['completion', 'score', 'any'] +FunctionOutputType = Literal['completion', 'score', 'any'] -FunctionTypeEnum: TypeAlias = Literal['llm', 'scorer', 'task', 'tool', 'custom_view', 'preprocessor', 'facet'] -""" -The type of global function. Defaults to 'scorer'. -""" +FunctionTypeEnum = Literal['llm', 'scorer', 'task', 'tool', 'custom_view', 'preprocessor', 'facet'] -FunctionTypeEnumNullish: TypeAlias = Literal['llm', 'scorer', 'task', 'tool', 'custom_view', 'preprocessor', 'facet'] +FunctionTypeEnumNullish = Literal['llm', 'scorer', 'task', 'tool', 'custom_view', 'preprocessor', 'facet'] class GitMetadataSettings(TypedDict): @@ -1055,76 +1064,10 @@ class GroupScope(TypedDict): """ -IfExists: TypeAlias = Literal['error', 'ignore', 'replace'] - - -class InvokeFunctionInvokeFunction(TypedDict): - function_id: str - """ - The ID of the function - """ - version: NotRequired[str | None] - """ - The version of the function - """ - - -class InvokeFunctionInvokeFunction1(TypedDict): - project_name: str - """ - The name of the project containing the function - """ - slug: str - """ - The slug of the function - """ - version: NotRequired[str | None] - """ - The version of the function - """ - - -class InvokeFunctionInvokeFunction2(TypedDict): - global_function: str - """ - The name of the global function. Currently, the global namespace includes the functions in autoevals - """ - function_type: NotRequired[FunctionTypeEnum | None] - - -class InvokeFunctionInvokeFunction3(TypedDict): - prompt_session_id: str - """ - The ID of the prompt session - """ - prompt_session_function_id: str - """ - The ID of the function in the prompt session - """ - version: NotRequired[str | None] - """ - The version of the function - """ +IfExists = Literal['error', 'ignore', 'replace'] -class InvokeFunctionInvokeFunction4InlineContext(TypedDict): - runtime: Literal['node', 'python', 'browser', 'quickjs'] - version: str - - -class InvokeFunctionInvokeFunction4(TypedDict): - inline_context: InvokeFunctionInvokeFunction4InlineContext - code: str - """ - The inline code to execute - """ - name: NotRequired[str | None] - """ - The name of the inline code function - """ - - -class InvokeFunctionMcpAuth(TypedDict): +class McpAuth(TypedDict): oauth_token: NotRequired[str | None] """ The OAuth token to use @@ -1162,10 +1105,7 @@ class InvokeParentInvokeParent(TypedDict): """ -InvokeParent: TypeAlias = InvokeParentInvokeParent | str -""" -Options for tracing the function call -""" +InvokeParent = InvokeParentInvokeParent | str class MCPServer(TypedDict): @@ -1203,7 +1143,7 @@ class MCPServer(TypedDict): """ -MessageRole: TypeAlias = Literal['system', 'user', 'assistant', 'function', 'tool', 'model', 'developer'] +MessageRole = Literal['system', 'user', 'assistant', 'function', 'tool', 'model', 'developer'] class ModelParamsModelParamsToolChoiceFunction(TypedDict): @@ -1221,6 +1161,7 @@ class ModelParamsModelParamsFunctionCall(TypedDict): class ModelParamsModelParams1(TypedDict): use_cache: NotRequired[bool | None] + use_native_inference: NotRequired[bool | None] reasoning_enabled: NotRequired[bool | None] reasoning_budget: NotRequired[float | None] max_tokens: float @@ -1236,6 +1177,7 @@ class ModelParamsModelParams1(TypedDict): class ModelParamsModelParams2(TypedDict): use_cache: NotRequired[bool | None] + use_native_inference: NotRequired[bool | None] reasoning_enabled: NotRequired[bool | None] reasoning_budget: NotRequired[float | None] temperature: NotRequired[float | None] @@ -1246,6 +1188,7 @@ class ModelParamsModelParams2(TypedDict): class ModelParamsModelParams3(TypedDict): use_cache: NotRequired[bool | None] + use_native_inference: NotRequired[bool | None] reasoning_enabled: NotRequired[bool | None] reasoning_budget: NotRequired[float | None] temperature: NotRequired[float | None] @@ -1254,6 +1197,7 @@ class ModelParamsModelParams3(TypedDict): class ModelParamsModelParams4(TypedDict): use_cache: NotRequired[bool | None] + use_native_inference: NotRequired[bool | None] reasoning_enabled: NotRequired[bool | None] reasoning_budget: NotRequired[float | None] @@ -1269,12 +1213,9 @@ class NullableSavedFunctionIdNullableSavedFunctionId1(TypedDict): function_type: NotRequired[FunctionTypeEnum | None] -NullableSavedFunctionId: TypeAlias = ( +NullableSavedFunctionId = ( NullableSavedFunctionIdNullableSavedFunctionId | NullableSavedFunctionIdNullableSavedFunctionId1 | None ) -""" -Default preprocessor for this project. When set, functions that use preprocessors will use this instead of their built-in default. -""" class ObjectReference(TypedDict): @@ -1342,14 +1283,7 @@ class Organization(TypedDict): """ -Permission: TypeAlias = Literal[ - 'create', 'read', 'update', 'delete', 'create_acls', 'read_acls', 'update_acls', 'delete_acls' -] -""" -Each permission permits a certain type of operation on an object in the system - -Permissions can be assigned to to objects on an individual basis, or grouped into roles -""" +Permission = Literal['create', 'read', 'update', 'delete', 'create_acls', 'read_acls', 'update_acls', 'delete_acls'] class ProjectAutomationConfigAction(TypedDict): @@ -1574,10 +1508,7 @@ class ProjectScoreCategory(TypedDict): """ -ProjectScoreType: TypeAlias = Literal['slider', 'categorical', 'weighted', 'minimum', 'maximum', 'online', 'free-form'] -""" -The type of the configured score -""" +ProjectScoreType = Literal['slider', 'categorical', 'weighted', 'minimum', 'maximum', 'online', 'free-form'] class ProjectSettingsSpanFieldOrderItem(TypedDict): @@ -1796,7 +1727,7 @@ class ResponseFormatNullishResponseFormatNullish2(TypedDict): type: Literal['text'] -ResponseFormatNullish: TypeAlias = ( +ResponseFormatNullish = ( ResponseFormatNullishResponseFormatNullish | ResponseFormatNullishResponseFormatNullish1 | ResponseFormatNullishResponseFormatNullish2 @@ -1804,10 +1735,7 @@ class ResponseFormatNullishResponseFormatNullish2(TypedDict): ) -RetentionObjectType: TypeAlias = Literal['project_logs', 'experiment', 'dataset'] -""" -The object type that the retention policy applies to -""" +RetentionObjectType = Literal['project_logs', 'experiment', 'dataset'] class RoleMemberPermission(TypedDict): @@ -1872,139 +1800,7 @@ class RunEvalData1(TypedDict): class RunEvalData2(TypedDict): - data: Sequence[Any] - - -class TaskTask(TypedDict): - function_id: str - """ - The ID of the function - """ - version: NotRequired[str | None] - """ - The version of the function - """ - - -class TaskTask1(TypedDict): - project_name: str - """ - The name of the project containing the function - """ - slug: str - """ - The slug of the function - """ - version: NotRequired[str | None] - """ - The version of the function - """ - - -class TaskTask2(TypedDict): - global_function: str - """ - The name of the global function. Currently, the global namespace includes the functions in autoevals - """ - function_type: NotRequired[FunctionTypeEnum | None] - - -class TaskTask3(TypedDict): - prompt_session_id: str - """ - The ID of the prompt session - """ - prompt_session_function_id: str - """ - The ID of the function in the prompt session - """ - version: NotRequired[str | None] - """ - The version of the function - """ - - -class TaskTask4InlineContext(TypedDict): - runtime: Literal['node', 'python', 'browser', 'quickjs'] - version: str - - -class TaskTask4(TypedDict): - inline_context: TaskTask4InlineContext - code: str - """ - The inline code to execute - """ - name: NotRequired[str | None] - """ - The name of the inline code function - """ - - -class TaskTask7(TypedDict): - pass - - -class TaskTask8(TaskTask, TaskTask7): - pass - - -class TaskTask9(TaskTask1, TaskTask7): - pass - - -class TaskTask10(TaskTask2, TaskTask7): - pass - - -class TaskTask11(TaskTask3, TaskTask7): - pass - - -class TaskTask12(TaskTask4, TaskTask7): - pass - - -class ParentParentRowIds(TypedDict): - id: str - """ - The id of the row - """ - span_id: str - """ - The span_id of the row - """ - root_span_id: str - """ - The root_span_id of the row - """ - - -class ParentParent(TypedDict): - object_type: Literal['project_logs', 'experiment', 'playground_logs'] - object_id: str - """ - The id of the container object you are logging to - """ - row_ids: NotRequired[ParentParentRowIds | None] - """ - Identifiers for the row to to log a subspan under - """ - propagated_event: NotRequired[Mapping[str, Any] | None] - """ - Include these properties in every span created under this parent - """ - - -class ParentParent1(TypedDict): - pass - - -class ParentParent2(ParentParent, ParentParent1): - pass - - -Parent: TypeAlias = ParentParent2 + data: Sequence class RunEvalMcpAuth(TypedDict): @@ -2025,7 +1821,7 @@ class SavedFunctionIdSavedFunctionId1(TypedDict): function_type: NotRequired[FunctionTypeEnum | None] -SavedFunctionId: TypeAlias = SavedFunctionIdSavedFunctionId | SavedFunctionIdSavedFunctionId1 +SavedFunctionId = SavedFunctionIdSavedFunctionId | SavedFunctionIdSavedFunctionId1 class ServiceToken(TypedDict): @@ -2103,12 +1899,7 @@ class SpanScope(TypedDict): type: Literal['span'] -SpanType: TypeAlias = Literal[ - 'llm', 'score', 'function', 'eval', 'task', 'tool', 'automation', 'facet', 'preprocessor' -] -""" -Type of the span, for display purposes only -""" +SpanType = Literal['llm', 'score', 'function', 'eval', 'task', 'tool', 'automation', 'facet', 'preprocessor'] class SSEConsoleEventData(TypedDict): @@ -2130,10 +1921,7 @@ class SSEProgressEventData(TypedDict): data: str -StreamingMode: TypeAlias = Literal['auto', 'parallel', 'json', 'text'] -""" -The mode format of the returned value (defaults to 'auto') -""" +StreamingMode = Literal['auto', 'parallel', 'json', 'text'] class ToolFunctionDefinitionFunction(TypedDict): @@ -2189,7 +1977,7 @@ class TriggeredFunctionState(TypedDict): """ -UploadStatus: TypeAlias = Literal['uploading', 'done', 'error'] +UploadStatus = Literal['uploading', 'done', 'error'] class User(TypedDict): @@ -2301,10 +2089,7 @@ class ViewOptionsViewOptions1(TypedDict): freezeColumns: NotRequired[bool | None] -ViewOptions: TypeAlias = ViewOptionsViewOptions | ViewOptionsViewOptions1 | None -""" -Options for the view in the app -""" +ViewOptions = ViewOptionsViewOptions | ViewOptionsViewOptions1 | None class Acl(TypedDict): @@ -2312,7 +2097,7 @@ class Acl(TypedDict): """ Unique identifier for the acl """ - object_type: AclObjectType + object_type: AclObjectType | None object_id: str """ The id of the object the ACL applies to @@ -2370,6 +2155,7 @@ class AnyModelParams(TypedDict): topP: NotRequired[float | None] topK: NotRequired[float | None] use_cache: NotRequired[bool | None] + use_native_inference: NotRequired[bool | None] class AsyncScoringControlAsyncScoringControl1(TypedDict): @@ -2377,17 +2163,18 @@ class AsyncScoringControlAsyncScoringControl1(TypedDict): state: AsyncScoringState -AsyncScoringControl: TypeAlias = ( +AsyncScoringControl = ( AsyncScoringControlAsyncScoringControl | AsyncScoringControlAsyncScoringControl1 | AsyncScoringControlAsyncScoringControl2 | AsyncScoringControlAsyncScoringControl3 | AsyncScoringControlAsyncScoringControl4 | AsyncScoringControlAsyncScoringControl5 + | AsyncScoringControlAsyncScoringControl6 ) -AttachmentReference: TypeAlias = BraintrustAttachmentReference | ExternalAttachmentReference +AttachmentReference = BraintrustAttachmentReference | ExternalAttachmentReference class AttachmentStatus(TypedDict): @@ -2398,26 +2185,13 @@ class AttachmentStatus(TypedDict): """ -class PreprocessorPreprocessor1(TypedDict): - type: Literal['global'] - name: str - function_type: NotRequired[FunctionTypeEnum | None] - - -class PreprocessorPreprocessor4(PreprocessorPreprocessor1, PreprocessorPreprocessor2): - pass - - -Preprocessor: TypeAlias = PreprocessorPreprocessor3 | PreprocessorPreprocessor4 - - class BatchedFacetData(TypedDict): type: Literal['batched_facet'] - preprocessor: NotRequired[Preprocessor | None] + preprocessor: NotRequired[NullableSavedFunctionId | None] facets: Sequence[BatchedFacetDataFacet] -ChatCompletionContentPart: TypeAlias = ( +ChatCompletionContentPart = ( ChatCompletionContentPartTextWithTitle | ChatCompletionContentPartImageWithTitle | ChatCompletionContentPartFileWithTitle @@ -2439,7 +2213,7 @@ class ChatCompletionMessageParamChatCompletionMessageParam2(TypedDict): reasoning: NotRequired[Sequence[ChatCompletionMessageReasoning] | None] -ChatCompletionMessageParam: TypeAlias = ( +ChatCompletionMessageParam = ( ChatCompletionMessageParamChatCompletionMessageParam | ChatCompletionMessageParamChatCompletionMessageParam1 | ChatCompletionMessageParamChatCompletionMessageParam2 @@ -2456,7 +2230,7 @@ class ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam1(TypedDic name: NotRequired[str | None] -ChatCompletionOpenAIMessageParam: TypeAlias = ( +ChatCompletionOpenAIMessageParam = ( ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam | ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam1 | ChatCompletionOpenAIMessageParamChatCompletionOpenAIMessageParam2 @@ -2509,7 +2283,7 @@ class DatasetEvent(TypedDict): """ span_id: str """ - A unique identifier used to link different dataset events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/guides/tracing) for full details on tracing + A unique identifier used to link different dataset events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing """ root_span_id: str """ @@ -2530,6 +2304,43 @@ class DatasetEvent(TypedDict): """ +class EvalStatusPage(TypedDict): + id: str + """ + Unique identifier for the eval status page + """ + project_id: str + """ + Unique identifier for the project that the eval status page belongs under + """ + user_id: NotRequired[str | None] + """ + Identifies the user who created the eval status page + """ + created: NotRequired[str | None] + """ + Date of eval status page creation + """ + deleted_at: NotRequired[str | None] + """ + Date of eval status page deletion, or null if the eval status page is still active + """ + name: str + """ + Name of the eval status page + """ + description: NotRequired[str | None] + """ + Textual description of the eval status page + """ + logo_url: NotRequired[str | None] + """ + URL of the logo to display on the page + """ + theme: EvalStatusPageTheme + config: EvalStatusPageConfig + + class Experiment(TypedDict): id: str """ @@ -2596,29 +2407,16 @@ class ExtendedSavedFunctionIdExtendedSavedFunctionId1(TypedDict): function_type: NotRequired[FunctionTypeEnum | None] -ExtendedSavedFunctionId: TypeAlias = ( +ExtendedSavedFunctionId = ( ExtendedSavedFunctionIdExtendedSavedFunctionId | ExtendedSavedFunctionIdExtendedSavedFunctionId1 | ExtendedSavedFunctionIdExtendedSavedFunctionId2 ) -class Preprocessor1Preprocessor11(TypedDict): - type: Literal['global'] - name: str - function_type: NotRequired[FunctionTypeEnum | None] - - -class Preprocessor1Preprocessor14(Preprocessor1Preprocessor11, Preprocessor1Preprocessor12): - pass - - -Preprocessor1: TypeAlias = Preprocessor1Preprocessor13 | Preprocessor1Preprocessor14 - - class FacetData(TypedDict): type: Literal['facet'] - preprocessor: NotRequired[Preprocessor1 | None] + preprocessor: NotRequired[NullableSavedFunctionId | None] prompt: str """ The prompt to use for LLM extraction. The preprocessed text will be provided as context. @@ -2651,69 +2449,9 @@ class FunctionIdFunctionId2(TypedDict): function_type: NotRequired[FunctionTypeEnum | None] -class InvokeFunctionInvokeFunction7(TypedDict): - input: NotRequired[Any | None] - """ - Argument to the function, which can be any JSON serializable value - """ - expected: NotRequired[Any | None] - """ - The expected output of the function - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - Any relevant metadata. This will be logged and available as the `metadata` argument. - """ - tags: NotRequired[Sequence[str] | None] - """ - Any relevant tags to log on the span. - """ - messages: NotRequired[Sequence[ChatCompletionMessageParam] | None] - """ - If the function is an LLM, additional messages to pass along to it - """ - parent: NotRequired[InvokeParent | None] - stream: NotRequired[bool | None] - """ - Whether to stream the response. If true, results will be returned in the Braintrust SSE format. - """ - mode: NotRequired[StreamingMode | None] - strict: NotRequired[bool | None] - """ - If true, throw an error if one of the variables in the prompt is not present in the input - """ - mcp_auth: NotRequired[Mapping[str, InvokeFunctionMcpAuth] | None] - """ - Map of MCP server URL to auth credentials - """ - overrides: NotRequired[Mapping[str, Any] | None] - """ - Partial function definition to merge with the function being invoked. Fields are validated against the function type's schema at runtime. For facets: { preprocessor?, prompt?, model? }. For prompts: { model?, ... }. - """ - - -class InvokeFunctionInvokeFunction8(InvokeFunctionInvokeFunction, InvokeFunctionInvokeFunction7): - pass - - -class InvokeFunctionInvokeFunction9(InvokeFunctionInvokeFunction1, InvokeFunctionInvokeFunction7): - pass - - -class InvokeFunctionInvokeFunction10(InvokeFunctionInvokeFunction2, InvokeFunctionInvokeFunction7): - pass - - -class InvokeFunctionInvokeFunction11(InvokeFunctionInvokeFunction3, InvokeFunctionInvokeFunction7): - pass - - -class InvokeFunctionInvokeFunction12(InvokeFunctionInvokeFunction4, InvokeFunctionInvokeFunction7): - pass - - class ModelParamsModelParams(TypedDict): use_cache: NotRequired[bool | None] + use_native_inference: NotRequired[bool | None] reasoning_enabled: NotRequired[bool | None] reasoning_budget: NotRequired[float | None] temperature: NotRequired[float | None] @@ -2736,7 +2474,7 @@ class ModelParamsModelParams(TypedDict): verbosity: NotRequired[Literal['low', 'medium', 'high'] | None] -ModelParams: TypeAlias = ( +ModelParams = ( ModelParamsModelParams | ModelParamsModelParams1 | ModelParamsModelParams2 @@ -2851,7 +2589,7 @@ class ProjectAutomation(TypedDict): """ -ProjectScoreCategories: TypeAlias = Sequence[ProjectScoreCategory] | Mapping[str, float] | Sequence[str] | None +ProjectScoreCategories = Sequence[ProjectScoreCategory] | Mapping[str, float] | Sequence[str] | None class ProjectScoreConfig(TypedDict): @@ -2866,7 +2604,7 @@ class PromptBlockDataPromptBlockData(TypedDict): tools: NotRequired[str | None] -PromptBlockData: TypeAlias = PromptBlockDataPromptBlockData | PromptBlockDataPromptBlockData1 +PromptBlockData = PromptBlockDataPromptBlockData | PromptBlockDataPromptBlockData1 class PromptBlockDataNullishPromptBlockDataNullish(TypedDict): @@ -2875,7 +2613,7 @@ class PromptBlockDataNullishPromptBlockDataNullish(TypedDict): tools: NotRequired[str | None] -PromptBlockDataNullish: TypeAlias = ( +PromptBlockDataNullish = ( PromptBlockDataNullishPromptBlockDataNullish | PromptBlockDataNullishPromptBlockDataNullish1 | None ) @@ -2897,9 +2635,7 @@ class ResponseFormatResponseFormat1(TypedDict): json_schema: ResponseFormatJsonSchema -ResponseFormat: TypeAlias = ( - ResponseFormatResponseFormat | ResponseFormatResponseFormat1 | ResponseFormatResponseFormat2 -) +ResponseFormat = ResponseFormatResponseFormat | ResponseFormatResponseFormat1 | ResponseFormatResponseFormat2 class SpanAttributes(TypedDict): @@ -2978,7 +2714,7 @@ class ExperimentEvent(TypedDict): """ span_id: str """ - A unique identifier used to link different experiment events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/guides/tracing) for full details on tracing + A unique identifier used to link different experiment events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing """ span_parents: NotRequired[Sequence[str] | None] """ @@ -3017,7 +2753,7 @@ class GraphNodeGraphNode7(TypedDict): prompt: PromptBlockData -GraphNode: TypeAlias = ( +GraphNode = ( GraphNodeGraphNode | GraphNodeGraphNode1 | GraphNodeGraphNode2 @@ -3096,7 +2832,7 @@ class ProjectLogsEvent(TypedDict): """ span_id: str """ - A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/guides/tracing) for full details on tracing + A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing """ span_parents: NotRequired[Sequence[str] | None] """ @@ -3177,42 +2913,12 @@ class PromptDataNullish(TypedDict): origin: NotRequired[PromptDataNullishOrigin | None] -class TaskTask5(TypedDict): - inline_prompt: NotRequired[PromptData | None] - inline_function: Mapping[str, Any] - function_type: NotRequired[FunctionTypeEnum | None] - name: NotRequired[str | None] - """ - The name of the inline function - """ - - -class TaskTask6(TypedDict): - inline_prompt: PromptData - function_type: NotRequired[FunctionTypeEnum | None] - name: NotRequired[str | None] - """ - The name of the inline prompt - """ - - -class TaskTask13(TaskTask5, TaskTask7): - pass - - -class TaskTask14(TaskTask6, TaskTask7): - pass - - -Task: TypeAlias = TaskTask8 | TaskTask9 | TaskTask10 | TaskTask11 | TaskTask12 | TaskTask13 | TaskTask14 - - class View(TypedDict): id: str """ Unique identifier for the view """ - object_type: AclObjectType + object_type: AclObjectType | None object_id: str """ The id of the object the view applies to @@ -3274,7 +2980,7 @@ class FunctionIdFunctionId6(TypedDict): """ -FunctionId: TypeAlias = ( +FunctionId = ( FunctionIdFunctionId | FunctionIdFunctionId1 | FunctionIdFunctionId2 @@ -3283,9 +2989,6 @@ class FunctionIdFunctionId6(TypedDict): | FunctionIdFunctionId5 | FunctionIdFunctionId6 ) -""" -Options for identifying a function -""" class GraphData(TypedDict): @@ -3294,45 +2997,45 @@ class GraphData(TypedDict): edges: Mapping[str, GraphEdge] -class InvokeFunctionInvokeFunction5(TypedDict): - inline_prompt: NotRequired[PromptData | None] - inline_function: Mapping[str, Any] - function_type: NotRequired[FunctionTypeEnum | None] - name: NotRequired[str | None] +class InvokeFunction(TypedDict): + input: NotRequired[Any | None] """ - The name of the inline function + Argument to the function, which can be any JSON serializable value """ - - -class InvokeFunctionInvokeFunction6(TypedDict): - inline_prompt: PromptData - function_type: NotRequired[FunctionTypeEnum | None] - name: NotRequired[str | None] + expected: NotRequired[Any | None] """ - The name of the inline prompt + The expected output of the function + """ + metadata: NotRequired[Mapping[str, Any] | None] + """ + Any relevant metadata. This will be logged and available as the `metadata` argument. + """ + tags: NotRequired[Sequence[str] | None] + """ + Any relevant tags to log on the span. + """ + messages: NotRequired[Sequence[ChatCompletionMessageParam] | None] + """ + If the function is an LLM, additional messages to pass along to it + """ + parent: NotRequired[InvokeParent | None] + stream: NotRequired[bool | None] + """ + Whether to stream the response. If true, results will be returned in the Braintrust SSE format. + """ + mode: NotRequired[StreamingMode | None] + strict: NotRequired[bool | None] + """ + If true, throw an error if one of the variables in the prompt is not present in the input + """ + mcp_auth: NotRequired[Mapping[str, McpAuth] | None] + """ + Map of MCP server URL to auth credentials + """ + overrides: NotRequired[Mapping[str, Any] | None] + """ + Partial function definition to merge with the function being invoked. Fields are validated against the function type's schema at runtime. For facets: { preprocessor?, prompt?, model? }. For prompts: { model?, ... }. """ - - -class InvokeFunctionInvokeFunction13(InvokeFunctionInvokeFunction5, InvokeFunctionInvokeFunction7): - pass - - -class InvokeFunctionInvokeFunction14(InvokeFunctionInvokeFunction6, InvokeFunctionInvokeFunction7): - pass - - -InvokeFunction: TypeAlias = ( - InvokeFunctionInvokeFunction8 - | InvokeFunctionInvokeFunction9 - | InvokeFunctionInvokeFunction10 - | InvokeFunctionInvokeFunction11 - | InvokeFunctionInvokeFunction12 - | InvokeFunctionInvokeFunction13 - | InvokeFunctionInvokeFunction14 -) -""" -Options for identifying a function -""" class Prompt(TypedDict): @@ -3393,7 +3096,7 @@ class RunEval(TypedDict): """ The dataset to use """ - task: Task + task: FunctionId scores: Sequence[FunctionId] """ The functions to score the eval on @@ -3406,7 +3109,7 @@ class RunEval(TypedDict): """ Optional experiment-level metadata to store about the evaluation. You can later use this to slice & dice across experiments. """ - parent: NotRequired[Parent | None] + parent: NotRequired[InvokeParent | None] stream: NotRequired[bool | None] """ Whether to stream the results of the eval. If true, the request will return two events: one to indicate the experiment has started, and another upon completion. If false, the request will return the evaluation's summary upon completion. @@ -3456,7 +3159,7 @@ class RunEval(TypedDict): mcp_auth: NotRequired[Mapping[str, RunEvalMcpAuth] | None] -FunctionData: TypeAlias = ( +FunctionData = ( FunctionDataFunctionData | FunctionDataFunctionData1 | GraphData diff --git a/py/src/braintrust/generated_types.py b/py/src/braintrust/generated_types.py index 65be4dade..5bccab61e 100644 --- a/py/src/braintrust/generated_types.py +++ b/py/src/braintrust/generated_types.py @@ -1,4 +1,4 @@ -"""Auto-generated file (internal git SHA 87ac73f4945a47eff2d4e42775ba4dbc58854c73) -- do not modify""" +"""Auto-generated file (internal git SHA 33ab31eaa0c3d1873a3b877c52d0a3a20015f939) -- do not modify""" from ._generated_types import ( Acl, @@ -29,6 +29,9 @@ Dataset, DatasetEvent, EnvVar, + EvalStatusPage, + EvalStatusPageConfig, + EvalStatusPageTheme, Experiment, ExperimentEvent, ExtendedSavedFunctionId, @@ -136,6 +139,9 @@ "Dataset", "DatasetEvent", "EnvVar", + "EvalStatusPage", + "EvalStatusPageConfig", + "EvalStatusPageTheme", "Experiment", "ExperimentEvent", "ExtendedSavedFunctionId",