diff --git a/python/samples/getting_started_with_agents/multi_agent_orchestration/step4c_handoff_mix_agent_types.py b/python/samples/getting_started_with_agents/multi_agent_orchestration/step4c_handoff_mix_agent_types.py index d10003c053c9..4ada917bab21 100644 --- a/python/samples/getting_started_with_agents/multi_agent_orchestration/step4c_handoff_mix_agent_types.py +++ b/python/samples/getting_started_with_agents/multi_agent_orchestration/step4c_handoff_mix_agent_types.py @@ -4,6 +4,7 @@ from azure.ai.projects.aio import AIProjectClient from azure.identity.aio import DefaultAzureCredential +from opentelemetry.trace import NoOpTracerProvider from samples.getting_started_with_agents.multi_agent_orchestration.observability import enable_observability from semantic_kernel.agents import ( @@ -197,7 +198,7 @@ async def main(): ) # 2. Create a runtime and start it - runtime = InProcessRuntime() + runtime = InProcessRuntime(tracer_provider=NoOpTracerProvider()) runtime.start() try: diff --git a/python/semantic_kernel/functions/kernel_arguments.py b/python/semantic_kernel/functions/kernel_arguments.py index 5a9e19375f94..a6048d14e631 100644 --- a/python/semantic_kernel/functions/kernel_arguments.py +++ b/python/semantic_kernel/functions/kernel_arguments.py @@ -1,7 +1,10 @@ # Copyright (c) Microsoft. All rights reserved. +import json from typing import TYPE_CHECKING, Any +from pydantic import BaseModel + from semantic_kernel.const import DEFAULT_SERVICE_NAME if TYPE_CHECKING: @@ -103,3 +106,17 @@ def __ior__(self, value: "SupportsKeysAndGetItem[Any, Any] | Iterable[tuple[Any, self.execution_settings = value.execution_settings.copy() return self + + def dumps(self, include_execution_settings: bool = False) -> str: + """Serializes the KernelArguments to a JSON string.""" + data = dict(self) + if include_execution_settings and self.execution_settings: + data["execution_settings"] = self.execution_settings + + def default(obj): + if isinstance(obj, BaseModel): + return obj.model_dump() + + return str(obj) + + return json.dumps(data, default=default) diff --git a/python/semantic_kernel/functions/kernel_function.py b/python/semantic_kernel/functions/kernel_function.py index 20af08e01754..ccc086582876 100644 --- a/python/semantic_kernel/functions/kernel_function.py +++ b/python/semantic_kernel/functions/kernel_function.py @@ -1,5 +1,6 @@ # Copyright (c) Microsoft. All rights reserved. +import json import logging import time from abc import abstractmethod @@ -32,6 +33,7 @@ from semantic_kernel.prompt_template.kernel_prompt_template import KernelPromptTemplate from semantic_kernel.prompt_template.prompt_template_base import PromptTemplateBase from semantic_kernel.utils.telemetry.model_diagnostics import function_tracer +from semantic_kernel.utils.telemetry.model_diagnostics.gen_ai_attributes import TOOL_CALL_ARGUMENTS, TOOL_CALL_RESULT if TYPE_CHECKING: from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings @@ -260,6 +262,9 @@ async def invoke( KernelFunctionLogMessages.log_function_invoking(logger, self.fully_qualified_name) KernelFunctionLogMessages.log_function_arguments(logger, arguments) + if function_tracer.are_sensitive_events_enabled(): + current_span.set_attribute(TOOL_CALL_ARGUMENTS, arguments.dumps()) + attributes = {MEASUREMENT_FUNCTION_TAG_NAME: self.fully_qualified_name} starting_time_stamp = time.perf_counter() try: @@ -272,6 +277,13 @@ async def invoke( KernelFunctionLogMessages.log_function_invoked_success(logger, self.fully_qualified_name) KernelFunctionLogMessages.log_function_result_value(logger, function_context.result) + if function_tracer.are_sensitive_events_enabled(): + try: + result = str(function_context.result.value) if function_context.result else None + except Exception as e: + result = str(e) + current_span.set_attribute(TOOL_CALL_RESULT, result) + return function_context.result except Exception as e: self._handle_exception(current_span, e, attributes) @@ -322,6 +334,9 @@ async def invoke_stream( KernelFunctionLogMessages.log_function_streaming_invoking(logger, self.fully_qualified_name) KernelFunctionLogMessages.log_function_arguments(logger, arguments) + if function_tracer.are_sensitive_events_enabled(): + current_span.set_attribute(TOOL_CALL_ARGUMENTS, arguments.dumps()) + attributes = {MEASUREMENT_FUNCTION_TAG_NAME: self.fully_qualified_name} starting_time_stamp = time.perf_counter() try: @@ -331,15 +346,27 @@ async def invoke_stream( ) await stack(function_context) + function_results: list[Any] = [] if function_context.result is not None: if isasyncgen(function_context.result.value): async for partial in function_context.result.value: + function_results.append(partial) yield partial elif isgenerator(function_context.result.value): for partial in function_context.result.value: + function_results.append(partial) yield partial else: + function_results.append(function_context.result.value) yield function_context.result + + if function_tracer.are_sensitive_events_enabled(): + results: list[str] = [] + try: + results.append(str(function_results)) + except Exception as e: + results.append(str(e)) + current_span.set_attribute(TOOL_CALL_RESULT, json.dumps(results)) except Exception as e: self._handle_exception(current_span, e, attributes) raise e diff --git a/python/semantic_kernel/utils/telemetry/model_diagnostics/function_tracer.py b/python/semantic_kernel/utils/telemetry/model_diagnostics/function_tracer.py index b7071e79aae1..da67efa90c5a 100644 --- a/python/semantic_kernel/utils/telemetry/model_diagnostics/function_tracer.py +++ b/python/semantic_kernel/utils/telemetry/model_diagnostics/function_tracer.py @@ -4,12 +4,14 @@ from opentelemetry import trace +from semantic_kernel.utils.feature_stage_decorator import experimental from semantic_kernel.utils.telemetry.model_diagnostics.gen_ai_attributes import ( OPERATION, TOOL_CALL_ID, TOOL_DESCRIPTION, TOOL_NAME, ) +from semantic_kernel.utils.telemetry.model_diagnostics.model_diagnostics_settings import ModelDiagnosticSettings if TYPE_CHECKING: from semantic_kernel.functions.kernel_function import KernelFunction @@ -19,6 +21,20 @@ # https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#execute-tool-span OPERATION_NAME = "execute_tool" +# To enable these features, set one of the following environment variables to true: +# SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS +# SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE +MODEL_DIAGNOSTICS_SETTINGS = ModelDiagnosticSettings() + + +@experimental +def are_sensitive_events_enabled() -> bool: + """Check if sensitive events are enabled. + + Sensitive events are enabled if the diagnostic with sensitive events is enabled. + """ + return MODEL_DIAGNOSTICS_SETTINGS.enable_otel_diagnostics_sensitive + def start_as_current_span( tracer: trace.Tracer, diff --git a/python/semantic_kernel/utils/telemetry/model_diagnostics/gen_ai_attributes.py b/python/semantic_kernel/utils/telemetry/model_diagnostics/gen_ai_attributes.py index 3d2b681d6185..f7aecf858316 100644 --- a/python/semantic_kernel/utils/telemetry/model_diagnostics/gen_ai_attributes.py +++ b/python/semantic_kernel/utils/telemetry/model_diagnostics/gen_ai_attributes.py @@ -26,6 +26,8 @@ INPUT_TOKENS = "gen_ai.usage.input_tokens" OUTPUT_TOKENS = "gen_ai.usage.output_tokens" TOOL_CALL_ID = "gen_ai.tool.call.id" +TOOL_CALL_ARGUMENTS = "gen_ai.tool.call.arguments" +TOOL_CALL_RESULT = "gen_ai.tool.call.result" TOOL_DESCRIPTION = "gen_ai.tool.description" TOOL_NAME = "gen_ai.tool.name" ADDRESS = "server.address"