From fd241407b59067ca3352f4650de2c0bc21b9b7cd Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Wed, 14 May 2025 17:35:21 -0700 Subject: [PATCH 1/9] Magentic orchestration --- python/.cspell.json | 3 +- .../getting_started_with_agents/README.md | 14 + .../multi_agent_orchestration/README.md | 48 ++ ...> step1a_concurrent_structured_outputs.py} | 0 ...py => step4a_handoff_structured_inputs.py} | 0 .../step5_magentic.py | 204 +++++ python/semantic_kernel/agents/__init__.py | 9 + python/semantic_kernel/agents/__init__.pyi | 17 +- .../agents/orchestration/magentic.py | 771 ++++++++++++++++++ .../agents/orchestration/prompts}/__init__.py | 0 .../prompts/_magentic_prompts.py | 150 ++++ .../agents/orchestration/test_magentic.py | 565 +++++++++++++ 12 files changed, 1779 insertions(+), 2 deletions(-) create mode 100644 python/samples/getting_started_with_agents/multi_agent_orchestration/README.md rename python/samples/getting_started_with_agents/multi_agent_orchestration/{step1a_concurrent_structure_output.py => step1a_concurrent_structured_outputs.py} (100%) rename python/samples/getting_started_with_agents/multi_agent_orchestration/{step4a_handoff_structure_input.py => step4a_handoff_structured_inputs.py} (100%) create mode 100644 python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py create mode 100644 python/semantic_kernel/agents/orchestration/magentic.py rename python/{samples/getting_started_with_agents/multi_agent_orchestration => semantic_kernel/agents/orchestration/prompts}/__init__.py (100%) create mode 100644 python/semantic_kernel/agents/orchestration/prompts/_magentic_prompts.py create mode 100644 python/tests/unit/agents/orchestration/test_magentic.py diff --git a/python/.cspell.json b/python/.cspell.json index f8ea3b8ea374..592c977609a1 100644 --- a/python/.cspell.json +++ b/python/.cspell.json @@ -47,6 +47,7 @@ "logit", "logprobs", "lowlevel", + "Magentic", "mistralai", "mongocluster", "nd", @@ -77,4 +78,4 @@ "vertexai", "Weaviate" ] -} \ No newline at end of file +} diff --git a/python/samples/getting_started_with_agents/README.md b/python/samples/getting_started_with_agents/README.md index 1d8dc9286032..9cdd147f59fc 100644 --- a/python/samples/getting_started_with_agents/README.md +++ b/python/samples/getting_started_with_agents/README.md @@ -71,6 +71,20 @@ Example|Description [step6_responses_agent_vision](../getting_started_with_agents/openai_responses/step6_responses_agent_vision.py)|How to provide an image as input to an OpenAI Responses agent. [step7_responses_agent_structured_outputs](../getting_started_with_agents/openai_responses/step7_responses_agent_structured_outputs.py)|How to use have an OpenAI Responses agent use structured outputs. +## Multi-Agent Orchestration +Example|Description +---|--- +[step1_concurrent](../getting_started_with_agents/multi_agent_orchestration/step1_concurrent.py)|How to run agents in parallel on the same task. +[step1a_concurrent_structure_output](../getting_started_with_agents/multi_agent_orchestration/step1a_concurrent_structure_output.py)|How to run agents in parallel on the same task and return structured output. +[step2_sequential](../getting_started_with_agents/multi_agent_orchestration/step2_sequential.py)|How to run agents in sequence to complete a task. +[step2a_sequential_cancellation_token](../getting_started_with_agents/multi_agent_orchestration/step2a_sequential_cancellation_token.py)|How to cancel an invocation while it is in progress. +[step3_group_chat](../getting_started_with_agents/multi_agent_orchestration/step3_group_chat.py)|How to run agents in a group chat to complete a task. +[step3a_group_chat_human_in_the_loop](../getting_started_with_agents/multi_agent_orchestration/step3a_group_chat_human_in_the_loop.py)|How to run agents in a group chat with human in the loop. +[step3b_group_chat_with_chat_completion_manager](../getting_started_with_agents/multi_agent_orchestration/step3b_group_chat_with_chat_completion_manager.py)|How to run agents in a group chat with a more dynamic manager. +[step4_handoff](../getting_started_with_agents/multi_agent_orchestration/step4_handoff.py)|How to run agents in a handoff orchestration to complete a task. +[step4a_handoff_structure_input](../getting_started_with_agents/multi_agent_orchestration/step4a_handoff_structure_input.py)|How to run agents in a handoff orchestration to complete a task with structured input. +[step5_magentic](../getting_started_with_agents/multi_agent_orchestration/step5_magentic.py)|How to run agents in a Magentic orchestration to complete a task. + ## Configuring the Kernel Similar to the Semantic Kernel Python concept samples, it is necessary to configure the secrets diff --git a/python/samples/getting_started_with_agents/multi_agent_orchestration/README.md b/python/samples/getting_started_with_agents/multi_agent_orchestration/README.md new file mode 100644 index 000000000000..37c8fd05526f --- /dev/null +++ b/python/samples/getting_started_with_agents/multi_agent_orchestration/README.md @@ -0,0 +1,48 @@ +# Multi-agent orchestration + +The Semantic Kernel Agent Framework now supports orchestrating multiple agents to work together to complete a task. + +## Background + +The following samples are beneficial if you are just getting started with Semantic Kernel. + +- [Chat Completion](../../concepts/chat_completion/) +- [Auto Function Calling](../../concepts/auto_function_calling/) +- [Structured Output](../../concepts/structured_output/) +- [Getting Started with Agents](../../getting_started_with_agents/) +- [More advanced agent samples](../../concepts/agents/) + +## Prerequisites + +The following environment variables are required to run the samples: + +- OPENAI_API_KEY +- OPENAI_CHAT_MODEL_ID + +However, if you are using other model services, feel free to switch to those in the samples. +Refer to [here](../../concepts/setup/README.md) on how to set up the environment variables for your model service. + +## Orchestrations + +| **Orchestrations** | **Description** | +| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Concurrent** | Useful for tasks that will benefit from independent analysis from multiple agents. | +| **Sequential** | Useful for tasks that require a well-defined step-by-step approach. | +| **Handoff** | Useful for tasks that are dynamic in nature and don't have a well-defined step-by-step approach. | +| **GroupChat** | Useful for tasks that will benefit from inputs from multiple agents and a highly configurable conversation flow. | +| **Magentic** | GroupChat like with a planner based manager. Inspired by [Magentic One](https://www.microsoft.com/en-us/research/articles/magentic-one-a-generalist-multi-agent-system-for-solving-complex-tasks/). | + +## Samples + +| Sample | Description | +|-----------------------------------------------------------------------------|--------------| +| [step1_concurrent](step1_concurrent.py) | Run agents in parallel on the same task. | +| [step1a_concurrent_structure_output](step1a_concurrent_structure_output.py) | Run agents in parallel on the same task and return structured output. | +| [step2_sequential](step2_sequential.py) | Run agents in sequence to complete a task. | +| [step2a_sequential_cancellation_token](step2a_sequential_cancellation_token.py) | Cancel an invocation while it is in progress. | +| [step3_group_chat](step3_group_chat.py) | Run agents in a group chat to complete a task. | +| [step3a_group_chat_human_in_the_loop](step3a_group_chat_human_in_the_loop.py) | Run agents in a group chat with human in the loop. | +| [step3b_group_chat_with_chat_completion_manager](step3b_group_chat_with_chat_completion_manager.py) | Run agents in a group chat with a more dynamic manager. | +| [step4_handoff](step4_handoff.py) | Run agents in a handoff orchestration to complete a task. | +| [step4a_handoff_structure_input](step4a_handoff_structure_input.py) | Run agents in a handoff orchestration to complete a task with structured input. | +| [step5_magentic](step5_magentic.py) | Run agents in a Magentic orchestration to complete a task. | diff --git a/python/samples/getting_started_with_agents/multi_agent_orchestration/step1a_concurrent_structure_output.py b/python/samples/getting_started_with_agents/multi_agent_orchestration/step1a_concurrent_structured_outputs.py similarity index 100% rename from python/samples/getting_started_with_agents/multi_agent_orchestration/step1a_concurrent_structure_output.py rename to python/samples/getting_started_with_agents/multi_agent_orchestration/step1a_concurrent_structured_outputs.py diff --git a/python/samples/getting_started_with_agents/multi_agent_orchestration/step4a_handoff_structure_input.py b/python/samples/getting_started_with_agents/multi_agent_orchestration/step4a_handoff_structured_inputs.py similarity index 100% rename from python/samples/getting_started_with_agents/multi_agent_orchestration/step4a_handoff_structure_input.py rename to python/samples/getting_started_with_agents/multi_agent_orchestration/step4a_handoff_structured_inputs.py diff --git a/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py b/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py new file mode 100644 index 000000000000..66ac9e35c042 --- /dev/null +++ b/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py @@ -0,0 +1,204 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio + +from semantic_kernel.agents import Agent, ChatCompletionAgent, MagenticOrchestration, OpenAIAssistantAgent +from semantic_kernel.agents.orchestration.magentic import StandardMagenticManager +from semantic_kernel.agents.runtime import InProcessRuntime +from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion, OpenAIPromptExecutionSettings +from semantic_kernel.contents import ChatMessageContent + +""" +The following sample demonstrates how to create a Magentic orchestration with two agents: +- A Research agent that can perform web searches +- A Coder agent that can run code using the code interpreter + +Read more about Magentic here: +https://www.microsoft.com/en-us/research/articles/magentic-one-a-generalist-multi-agent-system-for-solving-complex-tasks/ + +This sample demonstrates the basic steps of creating and starting a runtime, creating +a Magentic orchestration with two agents and a Magentic manager, invoking the +orchestration, and finally waiting for the results. + +The Magentic manager requires a chat completion model that supports structured output. +""" + + +async def agents() -> list[Agent]: + """Return a list of agents that will participate in the Magentic orchestration. + + Feel free to add or remove agents. + """ + research_agent = ChatCompletionAgent( + name="ResearchAgent", + description="A helpful assistant with access to web search. Ask it to perform web searches.", + instructions=("You are a Researcher. You find information."), + service=OpenAIChatCompletion(ai_model_id="gpt-4o-search-preview"), + ) + + # Create an OpenAI Assistant agent with code interpreter capability + client, model = OpenAIAssistantAgent.setup_resources() + code_interpreter_tool, code_interpreter_tool_resources = OpenAIAssistantAgent.configure_code_interpreter_tool() + definition = await client.beta.assistants.create( + model=model, + name="CoderAgent", + description="A helpful assistant with code interpreter capability.", + instructions="You solve questions using code.", + tools=code_interpreter_tool, + tool_resources=code_interpreter_tool_resources, + ) + coder_agent = OpenAIAssistantAgent( + client=client, + definition=definition, + ) + + return [research_agent, coder_agent] + + +def agent_response_callback(message: ChatMessageContent) -> None: + """Observer function to print the messages from the agents.""" + print(f"**{message.name}**\n{message.content}") + + +async def main(): + """Main function to run the agents.""" + # 1. Create a Magentic orchestration with two agents and a Magentic manager + # Note, the Magentic manager accepts custom prompts for advanced users and scenarios. + magentic_orchestration = MagenticOrchestration( + members=await agents(), + manager=StandardMagenticManager( + chat_completion_service=OpenAIChatCompletion(), + prompt_execution_settings=OpenAIPromptExecutionSettings(), + ), + agent_response_callback=agent_response_callback, + ) + + # 2. Create a runtime and start it + runtime = InProcessRuntime() + runtime.start() + + # 3. Invoke the orchestration with a task and the runtime + orchestration_result = await magentic_orchestration.invoke( + task=( + "What are the 50 tallest buildings in the world? Create a table with their names" + " and heights grouped by country with a column of the average height of the buildings" + " in each country." + ), + runtime=runtime, + ) + + # 4. Wait for the results + value = await orchestration_result.get() + print(value) + + # 5. Stop the runtime when idle + await runtime.stop_when_idle() + + """ + Sample output: + **ResearchAgent** + Based on the available information, here is a list of the 50 tallest buildings in the world, including their names, + heights, and countries: + + | Rank | Building Name | Height (m) | Country | + |------|---------------------------------------|------------|-------------------------| + | 1 | Burj Khalifa | 828 | United Arab Emirates | + | 2 | Merdeka 118 | 679 | Malaysia | + | 3 | Shanghai Tower | 632 | China | + | 4 | Makkah Royal Clock Tower | 601 | Saudi Arabia | + | 5 | Ping An Finance Center | 599 | China | + | 6 | Lotte World Tower | 555 | South Korea | + | 7 | One World Trade Center | 541 | United States | + | 8 | Guangzhou CTF Finance Centre | 530 | China | + | 9 | Tianjin CTF Finance Centre | 530 | China | + | 10 | CITIC Tower | 528 | China | + | 11 | TAIPEI 101 | 508 | Taiwan | + | 12 | Shanghai World Financial Center | 492 | China | + | 13 | International Commerce Centre | 484 | Hong Kong | + | 14 | Wuhan Greenland Center | 476 | China | + | 15 | Central Park Tower | 472 | United States | + | 16 | Lakhta Center | 462 | Russia | + | 17 | Vincom Landmark 81 | 461 | Vietnam | + | 18 | Changsha IFS Tower T1 | 452 | China | + | 19 | Petronas Tower 1 | 452 | Malaysia | + | 20 | Petronas Tower 2 | 452 | Malaysia | + | 21 | Suzhou IFS | 450 | China | + | 22 | Zifeng Tower | 450 | China | + | 23 | The Exchange 106 | 445 | Malaysia | + | 24 | Wuhan Center Tower | 443 | China | + | 25 | Willis Tower | 442 | United States | + | 26 | KK100 | 442 | China | + | 27 | Guangzhou International Finance Center| 438 | China | + | 28 | Wuhan Greenland Center | 438 | China | + | 29 | 432 Park Avenue | 425 | United States | + | 30 | Marina 101 | 425 | United Arab Emirates | + | 31 | Trump International Hotel & Tower | 423 | United States | + | 32 | Jin Mao Tower | 421 | China | + | 33 | Princess Tower | 414 | United Arab Emirates | + | 34 | Al Hamra Tower | 413 | Kuwait | + | 35 | Two International Finance Centre | 412 | Hong Kong | + | 36 | 23 Marina | 392 | United Arab Emirates | + | 37 | CITIC Plaza | 391 | China | + | 38 | Shun Hing Square | 384 | China | + | 39 | Eton Place Dalian Tower 1 | 383 | China | + | 40 | Empire State Building | 381 | United States | + | 41 | Burj Mohammed Bin Rashid | 381 | United Arab Emirates | + | 42 | Elite Residence | 380 | United Arab Emirates | + | 43 | The Address Boulevard | 370 | United Arab Emirates | + | 44 | Bank of China Tower | 367 | Hong Kong | + | 45 | Bank of America Tower | 366 | United States | + | 46 | St. Regis Chicago | 363 | United States | + | 47 | Almas Tower | 360 | United Arab Emirates | + | 48 | Hanking Center | 359 | China | + | 49 | Guangzhou Chow Tai Fook Finance Centre| 530 | China | + | 50 | Tianjin Chow Tai Fook Binhai Center | 530 | China | + + *Note: The heights are measured to the architectural top, including spires but excluding antennas, signage, flag + poles, or other functional or technical equipment.* + + This information is compiled from various sources, including the Council on Tall Buildings and Urban Habitat + (CTBUH) and other reputable architectural databases. + **CoderAgent** + Here's the table of the 50 tallest buildings in the world, grouped by country with the buildings' names, heights, + and the average height for each country: + + | Country | Number of Buildings | Average Height (m) | Building Names & Heights | + |-----------------------|---------------------|---------------------|---------------------------------------------| + | China | 21 | 471.33 | Shanghai Tower (632m), Ping An Finance ... | + | Hong Kong | 3 | 421.00 | International Commerce Centre (484m), T ... | + | Kuwait | 1 | 413.00 | Al Hamra Tower (413m) ... | + | Malaysia | 4 | 507.00 | Merdeka 118 (679m), Petronas Tower 1 (4 ... | + | Russia | 1 | 462.00 | Lakhta Center (462m) ... | + | Saudi Arabia | 1 | 601.00 | Makkah Royal Clock Tower (601m) ... | + | South Korea | 1 | 555.00 | Lotte World Tower (555m) ... | + | Taiwan | 1 | 508.00 | TAIPEI 101 (508m) ... | + | United Arab Emirates | 8 | 443.75 | Burj Khalifa (828m), Marina 101 (425m), ... | + | United States | 8 | 426.63 | One World Trade Center (541m), Central ... | + | Vietnam | 1 | 461.00 | Vincom Landmark 81 (461m) ... | + + This table presents a clear summary of tallest building distributions across various countries, along with the + average height of skyscrapers present in each region. + Here's the information you requested regarding the 50 tallest buildings in the world, organized by country. The + table below includes the names and heights of these buildings, along with the average height for each country: + + | Country | Number of Buildings | Average Height (m) | Building Names & Heights | + |-----------------------|---------------------|---------------------|---------------------------------------------| + | China | 21 | 471.33 | Shanghai Tower (632m), Ping An Finance ... | + | Hong Kong | 3 | 421.00 | International Commerce Centre (484m), T ... | + | Kuwait | 1 | 413.00 | Al Hamra Tower (413m) ... | + | Malaysia | 4 | 507.00 | Merdeka 118 (679m), Petronas Tower 1 (4 ... | + | Russia | 1 | 462.00 | Lakhta Center (462m) ... | + | Saudi Arabia | 1 | 601.00 | Makkah Royal Clock Tower (601m) ... | + | South Korea | 1 | 555.00 | Lotte World Tower (555m) ... | + | Taiwan | 1 | 508.00 | TAIPEI 101 (508m) ... | + | United Arab Emirates | 8 | 443.75 | Burj Khalifa (828m), Marina 101 (425m), ... | + | United States | 8 | 426.63 | One World Trade Center (541m), Central ... | + | Vietnam | 1 | 461.00 | Vincom Landmark 81 (461m) ... | + + This comprehensive list should give you a clear view of the tallest skyscrapers, showcasing their significant + presence across the globe. If you have any more questions or need further details, feel free to ask! + """ + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/semantic_kernel/agents/__init__.py b/python/semantic_kernel/agents/__init__.py index b89af9f0106a..8f5f326a6a00 100644 --- a/python/semantic_kernel/agents/__init__.py +++ b/python/semantic_kernel/agents/__init__.py @@ -40,6 +40,15 @@ "HandoffOrchestration": ".orchestration.handoffs", "OrchestrationHandoffs": ".orchestration.handoffs", "GroupChatOrchestration": ".orchestration.group_chat", + "RoundRobinGroupChatManager": ".orchestration.group_chat", + "BooleanResult": ".orchestration.group_chat", + "StringResult": ".orchestration.group_chat", + "MessageResult": ".orchestration.group_chat", + "GroupChatManager": ".orchestration.group_chat", + "MagenticOrchestration": ".orchestration.magentic", + "ProgressLedger": ".orchestration.magentic", + "MagenticManager": ".orchestration.magentic", + "StandardMagenticManager": ".orchestration.magentic", } diff --git a/python/semantic_kernel/agents/__init__.pyi b/python/semantic_kernel/agents/__init__.pyi index 9b6a795aabb1..5de2b5f9a17d 100644 --- a/python/semantic_kernel/agents/__init__.pyi +++ b/python/semantic_kernel/agents/__init__.pyi @@ -27,8 +27,16 @@ from .open_ai.open_ai_assistant_agent import AssistantAgentThread, OpenAIAssista from .open_ai.openai_responses_agent import OpenAIResponsesAgent, ResponsesAgentThread from .open_ai.run_polling_options import RunPollingOptions from .orchestration.concurrent import ConcurrentOrchestration -from .orchestration.group_chat import GroupChatManager, GroupChatOrchestration, RoundRobinGroupChatManager +from .orchestration.group_chat import ( + BooleanResult, + GroupChatManager, + GroupChatOrchestration, + MessageResult, + RoundRobinGroupChatManager, + StringResult, +) from .orchestration.handoffs import HandoffOrchestration, OrchestrationHandoffs +from .orchestration.magentic import MagenticManager, MagenticOrchestration, ProgressLedger, StandardMagenticManager from .orchestration.sequential import SequentialOrchestration __all__ = [ @@ -49,6 +57,7 @@ __all__ = [ "AzureResponsesAgent", "BedrockAgent", "BedrockAgentThread", + "BooleanResult", "ChatCompletionAgent", "ChatHistoryAgentThread", "ConcurrentOrchestration", @@ -60,15 +69,21 @@ __all__ = [ "GroupChatManager", "GroupChatOrchestration", "HandoffOrchestration", + "MagenticManager", + "MagenticOrchestration", + "MessageResult", "ModelConnection", "ModelSpec", "OpenAIAssistantAgent", "OpenAIResponsesAgent", "OrchestrationHandoffs", + "ProgressLedger", "ResponsesAgentThread", "RoundRobinGroupChatManager", "RunPollingOptions", "SequentialOrchestration", + "StandardMagenticManager", + "StringResult", "ToolSpec", "register_agent_type", ] diff --git a/python/semantic_kernel/agents/orchestration/magentic.py b/python/semantic_kernel/agents/orchestration/magentic.py new file mode 100644 index 000000000000..af03fa5a2987 --- /dev/null +++ b/python/semantic_kernel/agents/orchestration/magentic.py @@ -0,0 +1,771 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import logging +import sys +from abc import ABC, abstractmethod +from collections.abc import Awaitable, Callable + +from semantic_kernel.agents.agent import Agent +from semantic_kernel.agents.orchestration.agent_actor_base import ActorBase, AgentActorBase +from semantic_kernel.agents.orchestration.orchestration_base import DefaultTypeAlias, OrchestrationBase, TIn, TOut +from semantic_kernel.agents.orchestration.prompts._magentic_prompts import ( + ORCHESTRATOR_FINAL_ANSWER_PROMPT, + ORCHESTRATOR_PROGRESS_LEDGER_PROMPT, + ORCHESTRATOR_TASK_LEDGER_FACTS_PROMPT, + ORCHESTRATOR_TASK_LEDGER_FACTS_UPDATE_PROMPT, + ORCHESTRATOR_TASK_LEDGER_FULL_PROMPT, + ORCHESTRATOR_TASK_LEDGER_PLAN_PROMPT, + ORCHESTRATOR_TASK_LEDGER_PLAN_UPDATE_PROMPT, +) +from semantic_kernel.agents.runtime.core.cancellation_token import CancellationToken +from semantic_kernel.agents.runtime.core.core_runtime import CoreRuntime +from semantic_kernel.agents.runtime.core.message_context import MessageContext +from semantic_kernel.agents.runtime.core.routed_agent import message_handler +from semantic_kernel.agents.runtime.core.topic import TopicId +from semantic_kernel.agents.runtime.in_process.type_subscription import TypeSubscription +from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase +from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings +from semantic_kernel.contents.chat_history import ChatHistory +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.functions.kernel_arguments import KernelArguments +from semantic_kernel.kernel import Kernel +from semantic_kernel.kernel_pydantic import KernelBaseModel +from semantic_kernel.prompt_template.kernel_prompt_template import KernelPromptTemplate +from semantic_kernel.prompt_template.prompt_template_config import PromptTemplateConfig + +if sys.version_info >= (3, 12): + from typing import override # pragma: no cover +else: + from typing_extensions import override # pragma: no cover + +logger: logging.Logger = logging.getLogger(__name__) + + +# region Messages and Types + + +class MagenticStartMessage(KernelBaseModel): + """A message to start a magentic group chat.""" + + body: ChatMessageContent + + +class MagenticRequestMessage(KernelBaseModel): + """A request message type for agents in a magentic group chat.""" + + agent_name: str + + +class MagenticResponseMessage(KernelBaseModel): + """A response message type from agents in a magentic group chat.""" + + body: ChatMessageContent + + +class MagenticResetMessage(KernelBaseModel): + """A message to reset a participant's chat history in a magentic group chat.""" + + pass + + +class ProgressLedgerItem(KernelBaseModel): + """A progress ledger item.""" + + reason: str + answer: str | bool + + +class ProgressLedger(KernelBaseModel): + """A progress ledger.""" + + is_request_satisfied: ProgressLedgerItem + is_in_loop: ProgressLedgerItem + is_progress_being_made: ProgressLedgerItem + next_speaker: ProgressLedgerItem + instruction_or_question: ProgressLedgerItem + + +# endregion Messages and Types + +# region MagenticManager + + +class MagenticManager(KernelBaseModel, ABC): + """Base class for the Magentic One manager.""" + + @abstractmethod + async def create_facts_and_plan( + self, + chat_history: ChatHistory, + task: ChatMessageContent, + participant_descriptions: dict[str, str], + old_facts: ChatMessageContent | None = None, + ) -> tuple[ChatMessageContent, ChatMessageContent]: + """Create facts and plan for the task. + + Args: + chat_history (ChatHistory): The chat history. This chat history will be modified by the function. + task (ChatMessageContent): The task. + participant_descriptions (dict[str, str]): The participant descriptions. + old_facts (ChatMessageContent | None): The old facts. If provided, the facts and plan update + prompts will be used. + + Returns: + tuple[ChatMessageContent, ChatMessageContent]: The facts and plan. + """ + ... + + @abstractmethod + async def create_task_ledger( + self, + task: ChatMessageContent, + facts: ChatMessageContent, + plan: ChatMessageContent, + participant_descriptions: dict[str, str], + ) -> str: + """Create a task ledger. + + Args: + task (ChatMessageContent): The task. + facts (ChatMessageContent): The facts. + plan (ChatMessageContent): The plan. + participant_descriptions (dict[str, str]): The participant descriptions. + + Returns: + str: The task ledger. + """ + ... + + @abstractmethod + async def create_progress_ledger( + self, + chat_history: ChatHistory, + task: ChatMessageContent, + participant_descriptions: dict[str, str], + ) -> ProgressLedger: + """Create a progress ledger. + + Args: + chat_history (ChatHistory): The chat history. This chat history will be modified by the function. + task (ChatMessageContent): The task. + participant_descriptions (dict[str, str]): The participant descriptions. + + Returns: + ProgressLedger: The progress ledger. + """ + ... + + @abstractmethod + async def prepare_final_answer(self, chat_history: ChatHistory, task: ChatMessageContent) -> ChatMessageContent: + """Prepare the final answer. + + Args: + chat_history (ChatHistory): The chat history. This chat history will be modified by the function. + task (ChatMessageContent): The task. + + Returns: + ChatMessageContent: The final answer. + """ + ... + + +class StandardMagenticManager(MagenticManager): + """Container for the Magentic pattern.""" + + chat_completion_service: ChatCompletionClientBase + prompt_execution_settings: PromptExecutionSettings + + max_stall_count: int = 3 + + task_ledger_facts_prompt: str = ORCHESTRATOR_TASK_LEDGER_FACTS_PROMPT + task_ledger_plan_prompt: str = ORCHESTRATOR_TASK_LEDGER_PLAN_PROMPT + task_ledger_full_prompt: str = ORCHESTRATOR_TASK_LEDGER_FULL_PROMPT + task_ledger_facts_update_prompt: str = ORCHESTRATOR_TASK_LEDGER_FACTS_UPDATE_PROMPT + task_ledger_plan_update_prompt: str = ORCHESTRATOR_TASK_LEDGER_PLAN_UPDATE_PROMPT + progress_ledger_prompt: str = ORCHESTRATOR_PROGRESS_LEDGER_PROMPT + final_answer_prompt: str = ORCHESTRATOR_FINAL_ANSWER_PROMPT + + @override + async def create_facts_and_plan( + self, + chat_history: ChatHistory, + task: ChatMessageContent, + participant_descriptions: dict[str, str], + old_facts: ChatMessageContent | None = None, + ) -> tuple[ChatMessageContent, ChatMessageContent]: + """Create facts and plan for the task. + + Args: + chat_history (ChatHistory): The chat history. This chat history will be modified by the function. + task (ChatMessageContent): The task. + participant_descriptions (dict[str, str]): The participant descriptions. + old_facts (ChatMessageContent | None): The old facts. If provided, the facts and plan update + prompts will be used. + + Returns: + tuple[ChatMessageContent, ChatMessageContent]: The facts and plan. + """ + # 1. Update the facts + prompt_template = KernelPromptTemplate( + prompt_template_config=PromptTemplateConfig( + template=self.task_ledger_facts_update_prompt if old_facts else self.task_ledger_facts_prompt + ) + ) + chat_history.add_message( + ChatMessageContent( + role=AuthorRole.USER, + content=await prompt_template.render( + Kernel(), + KernelArguments(task=task.content, old_facts=old_facts.content) + if old_facts + else KernelArguments(task=task.content), + ), + ) + ) + facts = await self.chat_completion_service.get_chat_message_content( + chat_history, + self.prompt_execution_settings, + ) + assert facts is not None # nosec B101 + chat_history.add_message(facts) + + # 2. Update the plan + prompt_template = KernelPromptTemplate( + prompt_template_config=PromptTemplateConfig( + template=self.task_ledger_plan_update_prompt if old_facts else self.task_ledger_plan_prompt + ) + ) + chat_history.add_message( + ChatMessageContent( + role=AuthorRole.USER, + content=await prompt_template.render( + Kernel(), + KernelArguments(team=participant_descriptions), + ), + ) + ) + plan = await self.chat_completion_service.get_chat_message_content( + chat_history, + self.prompt_execution_settings, + ) + assert plan is not None # nosec B101 + + return facts, plan + + @override + async def create_task_ledger( + self, + task: ChatMessageContent, + facts: ChatMessageContent, + plan: ChatMessageContent, + participant_descriptions: dict[str, str], + ) -> str: + """Create a task ledger. + + Args: + task (ChatMessageContent): The task. + facts (ChatMessageContent): The facts. + plan (ChatMessageContent): The plan. + participant_descriptions (dict[str, str]): The participant descriptions. + + Returns: + str: The task ledger. + """ + prompt_template = KernelPromptTemplate( + prompt_template_config=PromptTemplateConfig(template=self.task_ledger_full_prompt) + ) + + return await prompt_template.render( + Kernel(), + KernelArguments( + task=task.content, + team=participant_descriptions, + facts=facts.content, + plan=plan.content, + ), + ) + + @override + async def create_progress_ledger( + self, + chat_history: ChatHistory, + task: ChatMessageContent, + participant_descriptions: dict[str, str], + ) -> ProgressLedger: + """Create a progress ledger. + + Args: + chat_history (ChatHistory): The chat history. This chat history will be modified by the function. + task (ChatMessageContent): The task. + participant_descriptions (dict[str, str]): The participant descriptions. + + Returns: + ProgressLedger: The progress ledger. + """ + prompt_template = KernelPromptTemplate( + prompt_template_config=PromptTemplateConfig(template=self.progress_ledger_prompt) + ) + progress_ledger_prompt = await prompt_template.render( + Kernel(), + KernelArguments( + task=task.content, + team=participant_descriptions, + names=", ".join(participant_descriptions.keys()), + ), + ) + chat_history.add_message(ChatMessageContent(role=AuthorRole.USER, content=progress_ledger_prompt)) + + prompt_execution_settings_clone = PromptExecutionSettings.from_prompt_execution_settings( + self.prompt_execution_settings + ) + prompt_execution_settings_clone.update_from_prompt_execution_settings( + # TODO(@taochen): Double check how to make sure the service support json output. + PromptExecutionSettings(extension_data={"response_format": ProgressLedger}) + ) + + response = await self.chat_completion_service.get_chat_message_content( + chat_history, + prompt_execution_settings_clone, + ) + assert response is not None # nosec B101 + + return ProgressLedger.model_validate_json(response.content) + + @override + async def prepare_final_answer(self, chat_history: ChatHistory, task: ChatMessageContent) -> ChatMessageContent: + """Prepare the final answer. + + Args: + chat_history (ChatHistory): The chat history. This chat history will be modified by the function. + task (ChatMessageContent): The task. + + Returns: + ChatMessageContent: The final answer. + """ + prompt_template = KernelPromptTemplate( + prompt_template_config=PromptTemplateConfig(template=self.final_answer_prompt) + ) + chat_history.add_message( + ChatMessageContent( + role=AuthorRole.USER, + content=await prompt_template.render(Kernel(), KernelArguments(task=task)), + ) + ) + + response = await self.chat_completion_service.get_chat_message_content( + chat_history, + self.prompt_execution_settings, + ) + assert response is not None # nosec B101 + + return response + + +# endregion MagenticManager + +# region MagenticManagerActor + + +class MagenticManagerActor(ActorBase): + """Actor for the Magentic One manager.""" + + def __init__( + self, + manager: MagenticManager, + internal_topic_type: str, + participant_descriptions: dict[str, str], + result_callback: Callable[[DefaultTypeAlias], Awaitable[None]] | None = None, + ) -> None: + """Initialize the Magentic One manager actor. + + Args: + manager (MagenticManager): The Magentic One manager. + internal_topic_type (str): The internal topic type. + participant_descriptions (dict[str, str]): The participant descriptions. + result_callback (Callable | None): A callback function to handle the final answer. + """ + self._manager = manager + self._internal_topic_type = internal_topic_type + self._chat_history = ChatHistory() + self._participant_descriptions = participant_descriptions + self._result_callback = result_callback + self._round_count = 0 + self._stall_count = 0 + + super().__init__(description="Magentic One Manager") + + @message_handler + async def _handle_start_message(self, message: MagenticStartMessage, ctx: MessageContext) -> None: + """Handle the start message for the Magentic One manager.""" + logger.debug(f"{self.id}: Received Magentic One start message.") + self._task = message.body + self._facts, self._plan = await self._manager.create_facts_and_plan( + self._chat_history.model_copy(deep=True), + self._task, + self._participant_descriptions, + ) + + await self._run_outer_loop(ctx.cancellation_token) + + @message_handler + async def _handle_response_message(self, message: MagenticResponseMessage, ctx: MessageContext) -> None: + if message.body.role != AuthorRole.USER: + self._chat_history.add_message( + ChatMessageContent( + role=AuthorRole.USER, + content=f"Transferred to {message.body.name}", + ) + ) + self._chat_history.add_message(message.body) + + logger.debug(f"{self.id}: Running inner loop.") + await self._run_inner_loop(ctx.cancellation_token) + + async def _run_outer_loop(self, cancellation_token: CancellationToken) -> None: + # 1. Create a task ledger. + task_ledger = await self._manager.create_task_ledger( + self._task, + self._facts, + self._plan, + self._participant_descriptions, + ) + + # 2. Publish the task ledger to the group chat. + # Need to add the task ledger to the orchestrator's chat history + # since the publisher won't receive the message it sends even though + # the publisher also subscribes to the topic. + self._chat_history.add_message( + ChatMessageContent( + role=AuthorRole.ASSISTANT, + content=task_ledger, + name=self.__class__.__name__, + ) + ) + + logger.debug(f"Initial task ledger:\n{task_ledger}") + await self.publish_message( + MagenticResponseMessage( + body=self._chat_history.messages[-1], + ), + TopicId(self._internal_topic_type, self.id.key), + cancellation_token=cancellation_token, + ) + + # 3. Start the inner loop. + await self._run_inner_loop(cancellation_token) + + async def _run_inner_loop(self, cancellation_token: CancellationToken) -> None: + self._round_count += 1 + + # 1. Create a progress ledger + current_progress_ledger = await self._manager.create_progress_ledger( + self._chat_history.model_copy(deep=True), + self._task, + self._participant_descriptions, + ) + logger.debug(f"Current progress ledger:\n{current_progress_ledger.model_dump_json(indent=2)}") + + # 2. Process the progress ledger + # 2.1 Check for task completion + if current_progress_ledger.is_request_satisfied.answer: + logger.debug("Task completed.") + await self._prepare_final_answer() + return + # 2.2 Check for stalling or looping + if not current_progress_ledger.is_progress_being_made.answer or current_progress_ledger.is_in_loop.answer: + self._stall_count += 1 + else: + self._stall_count = max(0, self._stall_count - 1) + + if self._stall_count > self._manager.max_stall_count: + logger.debug("Stalling detected. Resetting the task.") + self._facts, self._plan = await self._manager.create_facts_and_plan( + self._chat_history.model_copy(deep=True), + self._task, + self._participant_descriptions, + old_facts=self._facts, + ) + await self._reset_for_outer_loop(cancellation_token) + logger.debug("Restarting outer loop.") + await self._run_outer_loop(cancellation_token) + return + + # 2.3 Publish for next step + next_step = current_progress_ledger.instruction_or_question.answer + self._chat_history.add_message( + ChatMessageContent( + role=AuthorRole.ASSISTANT, + content=next_step if isinstance(next_step, str) else str(next_step), + name=self.__class__.__name__, + ) + ) + await self.publish_message( + MagenticResponseMessage( + body=self._chat_history.messages[-1], + ), + TopicId(self._internal_topic_type, self.id.key), + cancellation_token=cancellation_token, + ) + + # 2.4 Request the next speaker to speak + next_speaker = current_progress_ledger.next_speaker.answer + if next_speaker not in self._participant_descriptions: + raise ValueError(f"Unknown speaker: {next_speaker}") + + logger.debug(f"Magentic One manager selected agent: {next_speaker}") + + await self.publish_message( + MagenticRequestMessage(agent_name=next_speaker), + TopicId(self._internal_topic_type, self.id.key), + cancellation_token=cancellation_token, + ) + + async def _reset_for_outer_loop(self, cancellation_token: CancellationToken) -> None: + await self.publish_message( + MagenticResetMessage(), + TopicId(self._internal_topic_type, self.id.key), + cancellation_token=cancellation_token, + ) + self._chat_history.clear() + self._stall_count = 0 + + async def _prepare_final_answer(self) -> None: + final_answer = await self._manager.prepare_final_answer( + self._chat_history.model_copy(deep=True), + self._task, + ) + + if self._result_callback: + await self._result_callback(final_answer) + + +# endregion MagenticManagerActor + +# region MagenticAgentActor + + +class MagenticAgentActor(AgentActorBase): + """An agent actor that process messages in a Magentic One group chat.""" + + @message_handler + async def _handle_response_message(self, message: MagenticResponseMessage, ctx: MessageContext) -> None: + logger.debug(f"{self.id}: Received response message.") + if self._agent_thread is not None: + if message.body.role != AuthorRole.USER: + await self._agent_thread.on_new_message( + ChatMessageContent( + role=AuthorRole.USER, + content=f"Transferred to {message.body.name}", + ) + ) + await self._agent_thread.on_new_message(message.body) + else: + if message.body.role != AuthorRole.USER: + self._chat_history.add_message( + ChatMessageContent( + role=AuthorRole.USER, + content=f"Transferred to {message.body.name}", + ) + ) + self._chat_history.add_message(message.body) + + @message_handler + async def _handle_request_message(self, message: MagenticRequestMessage, ctx: MessageContext) -> None: + if message.agent_name != self._agent.name: + return + + logger.debug(f"{self.id}: Received request message.") + if self._agent_thread is None: + # Add a user message to steer the agent to respond more closely to the instructions. + self._chat_history.add_message( + ChatMessageContent( + role=AuthorRole.USER, + content=f"Transferred to {self._agent.name}, adopt the persona immediately.", + ) + ) + response_item = await self._agent.get_response(messages=self._chat_history.messages) # type: ignore[arg-type] + self._agent_thread = response_item.thread + else: + # Add a user message to steer the agent to respond more closely to the instructions. + new_message = ChatMessageContent( + role=AuthorRole.USER, + content=f"Transferred to {self._agent.name}, adopt the persona immediately.", + ) + response_item = await self._agent.get_response(messages=new_message, thread=self._agent_thread) + + logger.debug(f"{self.id} responded with {response_item.message.content}.") + await self._call_agent_response_callback(response_item.message) + + await self.publish_message( + MagenticResponseMessage(body=response_item.message), + TopicId(self._internal_topic_type, self.id.key), + cancellation_token=ctx.cancellation_token, + ) + + @message_handler + async def _handle_reset_message(self, message: MagenticResetMessage, ctx: MessageContext) -> None: + """Handle the reset message for the Magentic One group chat.""" + logger.debug(f"{self.id}: Received reset message.") + self._chat_history.clear() + if self._agent_thread: + await self._agent_thread.delete() + self._agent_thread = None + + +# endregion MagenticAgentActor + +# region MagenticOrchestration + + +class MagenticOrchestration(OrchestrationBase[TIn, TOut]): + """The Magentic One pattern orchestration.""" + + def __init__( + self, + members: list[Agent], + manager: MagenticManager, + name: str | None = None, + description: str | None = None, + input_transform: Callable[[TIn], Awaitable[DefaultTypeAlias] | DefaultTypeAlias] | None = None, + output_transform: Callable[[DefaultTypeAlias], Awaitable[TOut] | TOut] | None = None, + agent_response_callback: Callable[[DefaultTypeAlias], Awaitable[None] | None] | None = None, + ) -> None: + """Initialize the Magentic One orchestration. + + Args: + members (list[Agent | OrchestrationBase]): A list of agents or orchestration bases. + manager (MagenticManager): The manager for the Magentic One pattern. + name (str | None): The name of the orchestration. + description (str | None): The description of the orchestration. + input_transform (Callable | None): A function that transforms the external input message. + output_transform (Callable | None): A function that transforms the internal output message. + agent_response_callback (Callable | None): A function that is called when a response is produced + by the agents. + """ + self._manager = manager + + for member in members: + if member.description is None: + raise ValueError("All members must have a description.") + + super().__init__( + members=members, + name=name, + description=description, + input_transform=input_transform, + output_transform=output_transform, + agent_response_callback=agent_response_callback, + ) + + @override + async def _start( + self, + task: DefaultTypeAlias, + runtime: CoreRuntime, + internal_topic_type: str, + cancellation_token: CancellationToken, + ) -> None: + """Start the Magentic One pattern. + + This ensures that all initial messages are sent to the individual actors + and processed before the group chat begins. It's important because if the + manager actor processes its start message too quickly (or other actors are + too slow), it might send a request to the next agent before the other actors + have the necessary context. + """ + if not isinstance(task, ChatMessageContent): + # Magentic One only supports ChatMessageContent as input. + raise ValueError("The task must be a ChatMessageContent object.") + + async def send_start_message(agent: Agent) -> None: + target_actor_id = await runtime.get(self._get_agent_actor_type(agent, internal_topic_type)) + await runtime.send_message( + MagenticStartMessage(body=task), + target_actor_id, + cancellation_token=cancellation_token, + ) + + await asyncio.gather(*[send_start_message(agent) for agent in self._members]) + + target_actor_id = await runtime.get(self._get_manager_actor_type(internal_topic_type)) + await runtime.send_message( + MagenticStartMessage(body=task), + target_actor_id, + cancellation_token=cancellation_token, + ) + + @override + async def _prepare( + self, + runtime: CoreRuntime, + internal_topic_type: str, + result_callback: Callable[[DefaultTypeAlias], Awaitable[None]], + ) -> None: + """Register the actors and orchestrations with the runtime and add the required subscriptions.""" + await self._register_members(runtime, internal_topic_type) + await self._register_manager(runtime, internal_topic_type, result_callback=result_callback) + await self._add_subscriptions(runtime, internal_topic_type) + + async def _register_members(self, runtime: CoreRuntime, internal_topic_type: str) -> None: + """Register the agents.""" + await asyncio.gather(*[ + MagenticAgentActor.register( + runtime, + self._get_agent_actor_type(agent, internal_topic_type), + lambda agent=agent: MagenticAgentActor( # type: ignore[misc] + agent, + internal_topic_type, + self._agent_response_callback, + ), + ) + for agent in self._members + ]) + + async def _register_manager( + self, + runtime: CoreRuntime, + internal_topic_type: str, + result_callback: Callable[[DefaultTypeAlias], Awaitable[None]] | None = None, + ) -> None: + """Register the group chat manager.""" + await MagenticManagerActor.register( + runtime, + self._get_manager_actor_type(internal_topic_type), + lambda: MagenticManagerActor( + self._manager, + internal_topic_type=internal_topic_type, + participant_descriptions={agent.name: agent.description for agent in self._members}, # type: ignore[misc] + result_callback=result_callback, + ), + ) + + async def _add_subscriptions(self, runtime: CoreRuntime, internal_topic_type: str) -> None: + subscriptions: list[TypeSubscription] = [] + for agent in self._members: + subscriptions.append( + TypeSubscription(internal_topic_type, self._get_agent_actor_type(agent, internal_topic_type)) + ) + subscriptions.append(TypeSubscription(internal_topic_type, self._get_manager_actor_type(internal_topic_type))) + + await asyncio.gather(*[runtime.add_subscription(sub) for sub in subscriptions]) + + def _get_agent_actor_type(self, agent: Agent, internal_topic_type: str) -> str: + """Get the actor type for an agent. + + The type is appended with the internal topic type to ensure uniqueness in the runtime + that may be shared by multiple orchestrations. + """ + return f"{agent.name}_{internal_topic_type}" + + def _get_manager_actor_type(self, internal_topic_type: str) -> str: + """Get the actor type for the group chat manager. + + The type is appended with the internal topic type to ensure uniqueness in the runtime + that may be shared by multiple orchestrations. + """ + return f"{MagenticManagerActor.__name__}_{internal_topic_type}" + + +# endregion MagenticOrchestration diff --git a/python/samples/getting_started_with_agents/multi_agent_orchestration/__init__.py b/python/semantic_kernel/agents/orchestration/prompts/__init__.py similarity index 100% rename from python/samples/getting_started_with_agents/multi_agent_orchestration/__init__.py rename to python/semantic_kernel/agents/orchestration/prompts/__init__.py diff --git a/python/semantic_kernel/agents/orchestration/prompts/_magentic_prompts.py b/python/semantic_kernel/agents/orchestration/prompts/_magentic_prompts.py new file mode 100644 index 000000000000..4cd79f80c1fe --- /dev/null +++ b/python/semantic_kernel/agents/orchestration/prompts/_magentic_prompts.py @@ -0,0 +1,150 @@ +# Copyright (c) Microsoft. All rights reserved. + +ORCHESTRATOR_TASK_LEDGER_FACTS_PROMPT = """Below I will present you a request. + +Before we begin addressing the request, please answer the following pre-survey to the best of your ability. +Keep in mind that you are Ken Jennings-level with trivia, and Mensa-level with puzzles, so there should be +a deep well to draw from. + +Here is the request: + +{{$task}} + +Here is the pre-survey: + + 1. Please list any specific facts or figures that are GIVEN in the request itself. It is possible that + there are none. + 2. Please list any facts that may need to be looked up, and WHERE SPECIFICALLY they might be found. + In some cases, authoritative sources are mentioned in the request itself. + 3. Please list any facts that may need to be derived (e.g., via logical deduction, simulation, or computation) + 4. Please list any facts that are recalled from memory, hunches, well-reasoned guesses, etc. + +When answering this survey, keep in mind that "facts" will typically be specific names, dates, statistics, etc. +Your answer should use headings: + + 1. GIVEN OR VERIFIED FACTS + 2. FACTS TO LOOK UP + 3. FACTS TO DERIVE + 4. EDUCATED GUESSES + +DO NOT include any other headings or sections in your response. DO NOT list next steps or plans until asked to do so. +""" + +ORCHESTRATOR_TASK_LEDGER_PLAN_PROMPT = """Fantastic. To address this request we have assembled the following team: + +{{$team}} + +Based on the team composition, and known and unknown facts, please devise a short bullet-point plan for addressing the +original request. Remember, there is no requirement to involve all team members -- a team member's particular expertise +may not be needed for this task. +""" + +ORCHESTRATOR_TASK_LEDGER_FULL_PROMPT = """ +We are working to address the following user request: + +{{$task}} + + +To answer this request we have assembled the following team: + +{{$team}} + + +Here is an initial fact sheet to consider: + +{{$facts}} + + +Here is the plan to follow as best as possible: + +{{$plan}} +""" + +ORCHESTRATOR_PROGRESS_LEDGER_PROMPT = """ +Recall we are working on the following request: + +{{$task}} + +And we have assembled the following team: + +{{$team}} + +To make progress on the request, please answer the following questions, including necessary reasoning: + + - Is the request fully satisfied? (True if complete, or False if the original request has yet to be + SUCCESSFULLY and FULLY addressed) + - Are we in a loop where we are repeating the same requests and / or getting the same responses as before? + Loops can span multiple turns, and can include repeated actions like scrolling up or down more than a + handful of times. + - Are we making forward progress? (True if just starting, or recent messages are adding value. False if recent + messages show evidence of being stuck in a loop or if there is evidence of significant barriers to success + such as the inability to read from a required file) + - Who should speak next? (select from: {{$names}}) + - What instruction or question would you give this team member? (Phrase as if speaking directly to them, and + include any specific information they may need) + +Please output an answer in pure JSON format according to the following schema. The JSON object must be parsable as-is. +DO NOT OUTPUT ANYTHING OTHER THAN JSON, AND DO NOT DEVIATE FROM THIS SCHEMA: + +{ + "is_request_satisfied": { + "reason": string, + "answer": boolean + }, + "is_in_loop": { + "reason": string, + "answer": boolean + }, + "is_progress_being_made": { + "reason": string, + "answer": boolean + }, + "next_speaker": { + "reason": string, + "answer": string (select from: {{$names}}) + }, + "instruction_or_question": { + "reason": string, + "answer": string + } +} +""" + +ORCHESTRATOR_TASK_LEDGER_FACTS_UPDATE_PROMPT = """As a reminder, we are working to solve the following task: + +{{$task}} + +It's clear we aren't making as much progress as we would like, but we may have learned something new. +Please rewrite the following fact sheet, updating it to include anything new we have learned that may be helpful. + +Example edits can include (but are not limited to) adding new guesses, moving educated guesses to verified facts +if appropriate, etc. Updates may be made to any section of the fact sheet, and more than one section of the fact +sheet can be edited. This is an especially good time to update educated guesses, so please at least add or update +one educated guess or hunch, and explain your reasoning. + +Here is the old fact sheet: + +{{$old_facts}} +""" + + +ORCHESTRATOR_TASK_LEDGER_PLAN_UPDATE_PROMPT = """Please briefly explain what went wrong on this last run (the root +cause of the failure), and then come up with a new plan that takes steps and/or includes hints to overcome prior +challenges and especially avoids repeating the same mistakes. As before, the new plan should be concise, be expressed +in bullet-point form, and consider the following team composition (do not involve any other outside people since we +cannot contact anyone else): + +{{$team}} +""" + +ORCHESTRATOR_FINAL_ANSWER_PROMPT = """ +We are working on the following task: +{{$task}} + +We have completed the task. + +The above messages contain the conversation that took place to complete the task. + +Based on the information gathered, provide the final answer to the original request. +The answer should be phrased as if you were speaking to the user. +""" diff --git a/python/tests/unit/agents/orchestration/test_magentic.py b/python/tests/unit/agents/orchestration/test_magentic.py new file mode 100644 index 000000000000..d354d1b4d13e --- /dev/null +++ b/python/tests/unit/agents/orchestration/test_magentic.py @@ -0,0 +1,565 @@ +# Copyright (c) Microsoft. All rights reserved. + +from unittest.mock import AsyncMock, patch +from uuid import uuid4 + +import pytest + +from semantic_kernel.agents.orchestration.magentic import ( + MagenticManager, + MagenticOrchestration, + ProgressLedger, + ProgressLedgerItem, +) +from semantic_kernel.agents.orchestration.orchestration_base import DefaultTypeAlias, OrchestrationResult +from semantic_kernel.agents.orchestration.prompts._magentic_prompts import ( + ORCHESTRATOR_FINAL_ANSWER_PROMPT, + ORCHESTRATOR_PROGRESS_LEDGER_PROMPT, + ORCHESTRATOR_TASK_LEDGER_FACTS_PROMPT, + ORCHESTRATOR_TASK_LEDGER_FACTS_UPDATE_PROMPT, + ORCHESTRATOR_TASK_LEDGER_FULL_PROMPT, + ORCHESTRATOR_TASK_LEDGER_PLAN_PROMPT, + ORCHESTRATOR_TASK_LEDGER_PLAN_UPDATE_PROMPT, +) +from semantic_kernel.agents.runtime.in_process.in_process_runtime import InProcessRuntime +from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase +from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings +from semantic_kernel.contents.chat_history import ChatHistory +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from tests.unit.agents.orchestration.conftest import MockAgent, MockRuntime + + +class MockChatCompletionService(ChatCompletionClientBase): + """A mock chat completion service for testing purposes.""" + + pass + + +# region MagenticOrchestration + + +async def test_init_member_without_description_throws(): + """Test the prepare method of the MagenticOrchestration with a member without description.""" + agent_a = MockAgent() + agent_b = MockAgent() + + with pytest.raises(ValueError): + MagenticOrchestration( + members=[agent_a, agent_b], + manager=MagenticManager( + chat_completion_service=MockChatCompletionService(ai_model_id="test"), + prompt_execution_settings=PromptExecutionSettings(), + ), + ) + + +async def test_prepare(): + """Test the prepare method of the MagenticOrchestration.""" + agent_a = MockAgent(description="test agent") + agent_b = MockAgent(description="test agent") + + runtime = MockRuntime() + + package_path = "semantic_kernel.agents.orchestration.magentic" + with ( + patch(f"{package_path}.MagenticOrchestration._start"), + patch(f"{package_path}.MagenticAgentActor.register") as mock_agent_actor_register, + patch(f"{package_path}.MagenticManagerActor.register") as mock_manager_actor_register, + patch.object(runtime, "add_subscription") as mock_add_subscription, + ): + orchestration = MagenticOrchestration( + members=[agent_a, agent_b], + manager=MagenticManager( + chat_completion_service=MockChatCompletionService(ai_model_id="test"), + prompt_execution_settings=PromptExecutionSettings(), + ), + ) + await orchestration.invoke(task="test_message", runtime=runtime) + + assert mock_agent_actor_register.call_count == 2 + assert mock_manager_actor_register.call_count == 1 + assert mock_add_subscription.call_count == 3 + + +ManagerProgressList = [ + ProgressLedger( + is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"), + next_speaker=ProgressLedgerItem(answer="agent_a", reason="mock_reasoning"), + instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"), + ), + ProgressLedger( + is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"), + next_speaker=ProgressLedgerItem(answer="agent_b", reason="mock_reasoning"), + instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"), + ), + ProgressLedger( + is_request_satisfied=ProgressLedgerItem(answer=True, reason="mock_reasoning"), + is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"), + next_speaker=ProgressLedgerItem(answer="N/A", reason="mock_reasoning"), + instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"), + ), +] + +ManagerProgressListStalling = [ + ProgressLedger( + is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"), + next_speaker=ProgressLedgerItem(answer="agent_a", reason="mock_reasoning"), + instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"), + ), + ProgressLedger( + is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_in_loop=ProgressLedgerItem(answer=True, reason="mock_reasoning"), # is_in_loop=True + is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"), + next_speaker=ProgressLedgerItem(answer="agent_a", reason="mock_reasoning"), + instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"), + ), + ProgressLedger( + is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_in_loop=ProgressLedgerItem(answer=True, reason="mock_reasoning"), # is_in_loop=True + is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"), + next_speaker=ProgressLedgerItem(answer="N/A", reason="mock_reasoning"), + instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"), + ), + ProgressLedger( + is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"), + next_speaker=ProgressLedgerItem(answer="agent_b", reason="mock_reasoning"), + instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"), + ), + ProgressLedger( + is_request_satisfied=ProgressLedgerItem(answer=True, reason="mock_reasoning"), + is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"), + next_speaker=ProgressLedgerItem(answer="N/A", reason="mock_reasoning"), + instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"), + ), +] + +ManagerProgressListUnknownSpeaker = [ + ProgressLedger( + is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"), + next_speaker=ProgressLedgerItem(answer="unknown", reason="mock_reasoning"), + instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"), + ), +] + + +async def test_invoke(): + """Test the invoke method of the MagenticOrchestration.""" + with ( + patch.object(MockAgent, "get_response", wraps=MockAgent.get_response, autospec=True) as mock_get_response, + patch.object( + MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock + ) as mock_get_chat_message_content, + patch.object( + MagenticManager, "create_progress_ledger", new_callable=AsyncMock, side_effect=ManagerProgressList + ), + ): + mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response") + chat_completion_service = MockChatCompletionService(ai_model_id="test") + prompt_execution_settings = PromptExecutionSettings() + + manager = MagenticManager( + chat_completion_service=chat_completion_service, + prompt_execution_settings=prompt_execution_settings, + ) + + agent_a = MockAgent(name="agent_a", description="test agent") + agent_b = MockAgent(name="agent_b", description="test agent") + + runtime = InProcessRuntime() + runtime.start() + + try: + orchestration = MagenticOrchestration(members=[agent_a, agent_b], manager=manager) + orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime) + result = await orchestration_result.get() + finally: + await runtime.stop_when_idle() + + assert isinstance(orchestration_result, OrchestrationResult) + assert isinstance(result, ChatMessageContent) + assert result.role == AuthorRole.ASSISTANT + assert result.content == "mock_response" + + assert mock_get_response.call_count == 2 + assert mock_get_chat_message_content.call_count == 3 + + +async def test_invoke_with_list_error(): + """Test the invoke method of the MagenticOrchestration with a list of messages which raises an error.""" + chat_completion_service = MockChatCompletionService(ai_model_id="test") + prompt_execution_settings = PromptExecutionSettings() + + manager = MagenticManager( + chat_completion_service=chat_completion_service, + prompt_execution_settings=prompt_execution_settings, + ) + + agent_a = MockAgent(name="agent_a", description="test agent") + agent_b = MockAgent(name="agent_b", description="test agent") + + messages = [ + ChatMessageContent(role=AuthorRole.USER, content="test_message_1"), + ChatMessageContent(role=AuthorRole.USER, content="test_message_2"), + ] + + runtime = MockRuntime() + + package_path = "semantic_kernel.agents.orchestration.magentic" + with ( + patch(f"{package_path}.MagenticAgentActor.register"), + patch(f"{package_path}.MagenticManagerActor.register"), + patch.object(runtime, "add_subscription"), + pytest.raises(ValueError), + ): + orchestration = MagenticOrchestration(members=[agent_a, agent_b], manager=manager) + orchestration_result = await orchestration.invoke(task=messages, runtime=runtime) + await orchestration_result.get() + + +async def test_invoke_with_response_callback(): + """Test the invoke method of the MagenticOrchestration with a response callback.""" + + runtime = InProcessRuntime() + runtime.start() + + responses: list[DefaultTypeAlias] = [] + with ( + patch.object(MockAgent, "get_response", wraps=MockAgent.get_response, autospec=True), + patch.object( + MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock + ) as mock_get_chat_message_content, + patch.object( + MagenticManager, "create_progress_ledger", new_callable=AsyncMock, side_effect=ManagerProgressList + ), + ): + mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response") + + agent_a = MockAgent(name="agent_a", description="test agent") + agent_b = MockAgent(name="agent_b", description="test agent") + + try: + orchestration = MagenticOrchestration( + members=[agent_a, agent_b], + manager=MagenticManager( + chat_completion_service=MockChatCompletionService(ai_model_id="test"), + prompt_execution_settings=PromptExecutionSettings(), + ), + agent_response_callback=lambda x: responses.append(x), + ) + orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime) + await orchestration_result.get(1.0) + finally: + await runtime.stop_when_idle() + + assert len(responses) == 2 + assert all(isinstance(item, ChatMessageContent) for item in responses) + assert all(item.content == "mock_response" for item in responses) + + +async def test_invoke_with_max_stall_count_exceeded(): + """ "Test the invoke method of the MagenticOrchestration with max stall count exceeded.""" + runtime = InProcessRuntime() + runtime.start() + + with ( + patch.object(MockAgent, "get_response", wraps=MockAgent.get_response, autospec=True) as mock_get_response, + patch.object( + MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock + ) as mock_get_chat_message_content, + patch.object( + MagenticManager, + "create_progress_ledger", + new_callable=AsyncMock, + side_effect=ManagerProgressListStalling, + ), + ): + mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response") + + agent_a = MockAgent(name="agent_a", description="test agent") + agent_b = MockAgent(name="agent_b", description="test agent") + + try: + orchestration = MagenticOrchestration( + members=[agent_a, agent_b], + manager=MagenticManager( + chat_completion_service=MockChatCompletionService(ai_model_id="test"), + prompt_execution_settings=PromptExecutionSettings(), + max_stall_count=1, + ), + ) + orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime) + await orchestration_result.get(1.0) + finally: + await runtime.stop_when_idle() + + assert mock_get_response.call_count == 3 + # Exceeding max stall count will trigger replanning, which will recreate the facts and plan, + # resulting in two additional calls to get_chat_message_content compared to the `test_invoke` test. + assert mock_get_chat_message_content.call_count == 5 + + +async def test_invoke_with_unknown_speaker(): + """Test the invoke method of the MagenticOrchestration with an unknown speaker.""" + runtime = InProcessRuntime() + runtime.start() + + with ( + patch.object(MockAgent, "get_response", wraps=MockAgent.get_response, autospec=True), + patch.object( + MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock + ) as mock_get_chat_message_content, + patch.object( + MagenticManager, + "create_progress_ledger", + new_callable=AsyncMock, + side_effect=ManagerProgressListUnknownSpeaker, + ), + pytest.raises(ValueError), + ): + mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response") + + agent_a = MockAgent(name="agent_a", description="test agent") + agent_b = MockAgent(name="agent_b", description="test agent") + + try: + orchestration = MagenticOrchestration( + members=[agent_a, agent_b], + manager=MagenticManager( + chat_completion_service=MockChatCompletionService(ai_model_id="test"), + prompt_execution_settings=PromptExecutionSettings(), + ), + ) + orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime) + await orchestration_result.get() + finally: + await runtime.stop_when_idle() + + +# endregion MagenticOrchestration + +# region MagenticManager + + +def test_magentic_manager_init(): + """Test the initialization of the MagenticManager.""" + chat_completion_service = MockChatCompletionService(ai_model_id="test") + prompt_execution_settings = PromptExecutionSettings() + + manager = MagenticManager( + chat_completion_service=chat_completion_service, + prompt_execution_settings=prompt_execution_settings, + ) + + assert manager.max_stall_count > 0 + assert ( + manager.task_ledger_facts_prompt is not None + and manager.task_ledger_facts_prompt == ORCHESTRATOR_TASK_LEDGER_FACTS_PROMPT + ) + assert ( + manager.task_ledger_plan_prompt is not None + and manager.task_ledger_plan_prompt == ORCHESTRATOR_TASK_LEDGER_PLAN_PROMPT + ) + assert ( + manager.task_ledger_full_prompt is not None + and manager.task_ledger_full_prompt == ORCHESTRATOR_TASK_LEDGER_FULL_PROMPT + ) + assert ( + manager.task_ledger_facts_update_prompt is not None + and manager.task_ledger_facts_update_prompt == ORCHESTRATOR_TASK_LEDGER_FACTS_UPDATE_PROMPT + ) + assert ( + manager.task_ledger_plan_update_prompt is not None + and manager.task_ledger_plan_update_prompt == ORCHESTRATOR_TASK_LEDGER_PLAN_UPDATE_PROMPT + ) + assert ( + manager.progress_ledger_prompt is not None + and manager.progress_ledger_prompt == ORCHESTRATOR_PROGRESS_LEDGER_PROMPT + ) + assert manager.final_answer_prompt is not None and manager.final_answer_prompt == ORCHESTRATOR_FINAL_ANSWER_PROMPT + + +def test_magentic_manager_init_with_custom_prompts(): + """Test the initialization of the MagenticManager with custom prompts.""" + chat_completion_service = MockChatCompletionService(ai_model_id="test") + prompt_execution_settings = PromptExecutionSettings() + + manager = MagenticManager( + chat_completion_service=chat_completion_service, + prompt_execution_settings=prompt_execution_settings, + task_ledger_facts_prompt="custom_task_ledger_facts_prompt", + task_ledger_plan_prompt="custom_task_ledger_plan_prompt", + task_ledger_full_prompt="custom_task_ledger_full_prompt", + task_ledger_facts_update_prompt="custom_task_ledger_facts_update_prompt", + task_ledger_plan_update_prompt="custom_task_ledger_plan_update_prompt", + progress_ledger_prompt="custom_progress_ledger_prompt", + final_answer_prompt="custom_final_answer_prompt", + ) + + assert manager.task_ledger_facts_prompt == "custom_task_ledger_facts_prompt" + assert manager.task_ledger_plan_prompt == "custom_task_ledger_plan_prompt" + assert manager.task_ledger_full_prompt == "custom_task_ledger_full_prompt" + assert manager.task_ledger_facts_update_prompt == "custom_task_ledger_facts_update_prompt" + assert manager.task_ledger_plan_update_prompt == "custom_task_ledger_plan_update_prompt" + assert manager.progress_ledger_prompt == "custom_progress_ledger_prompt" + assert manager.final_answer_prompt == "custom_final_answer_prompt" + + +async def test_magentic_manager_create_facts_and_prompt(): + """Test the create_facts_and_prompt method of the MagenticManager.""" + + with patch.object( + MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock + ) as mock_get_chat_message_content: + mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response") + chat_completion_service = MockChatCompletionService(ai_model_id="test") + prompt_execution_settings = PromptExecutionSettings() + + manager = MagenticManager( + chat_completion_service=chat_completion_service, + prompt_execution_settings=prompt_execution_settings, + task_ledger_facts_prompt="custom_task_ledger_facts_prompt", + task_ledger_plan_prompt="custom_task_ledger_plan_prompt {{$team}}", + ) + + facts, prompt = await manager.create_facts_and_plan( + ChatHistory(), + ChatMessageContent(role="user", content="test_message"), + {"agent_a": "test_agent_a", "agent_b": "test_agent_b"}, + ) + + assert isinstance(facts, ChatMessageContent) and facts.content == "mock_response" + assert isinstance(prompt, ChatMessageContent) and prompt.content == "mock_response" + + assert mock_get_chat_message_content.call_count == 2 + assert ( + mock_get_chat_message_content.call_args_list[0][0][0].messages[0].content + == "custom_task_ledger_facts_prompt" + ) + assert ( + mock_get_chat_message_content.call_args_list[1][0][0].messages[2].content + == "custom_task_ledger_plan_prompt {'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}" + ) + + +async def test_magentic_manager_create_facts_and_prompt_with_old_facts(): + """Test the create_facts_and_prompt method of the MagenticManager with old facts.""" + + with patch.object( + MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock + ) as mock_get_chat_message_content: + mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response") + + chat_completion_service = MockChatCompletionService(ai_model_id="test") + prompt_execution_settings = PromptExecutionSettings() + + manager = MagenticManager( + chat_completion_service=chat_completion_service, + prompt_execution_settings=prompt_execution_settings, + task_ledger_facts_update_prompt="custom_task_ledger_facts_prompt {{$old_facts}}", + task_ledger_plan_update_prompt="custom_task_ledger_plan_prompt {{$team}}", + ) + + facts, prompt = await manager.create_facts_and_plan( + ChatHistory(), + ChatMessageContent(role="user", content="test_message"), + {"agent_a": "test_agent_a", "agent_b": "test_agent_b"}, + old_facts=ChatMessageContent(role="user", content="old_facts"), + ) + + assert isinstance(facts, ChatMessageContent) and facts.content == "mock_response" + assert isinstance(prompt, ChatMessageContent) and prompt.content == "mock_response" + + assert mock_get_chat_message_content.call_count == 2 + assert ( + mock_get_chat_message_content.call_args_list[0][0][0].messages[0].content + == "custom_task_ledger_facts_prompt old_facts" + ) + assert ( + mock_get_chat_message_content.call_args_list[1][0][0].messages[2].content + == "custom_task_ledger_plan_prompt {'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}" + ) + + +async def test_magentic_manager_create_task_ledger(): + """Test the create_task_ledger method of the MagenticManager.""" + + chat_completion_service = MockChatCompletionService(ai_model_id="test") + prompt_execution_settings = PromptExecutionSettings() + + manager = MagenticManager( + chat_completion_service=chat_completion_service, + prompt_execution_settings=prompt_execution_settings, + ) + + task = ChatMessageContent(role="user", content=uuid4().hex) + facts = ChatMessageContent(role="user", content=uuid4().hex) + plan = ChatMessageContent(role="user", content=uuid4().hex) + participants = {"agent_a": "test_agent_a", "agent_b": "test_agent_b"} + + task_ledger = await manager.create_task_ledger(task, facts, plan, participants) + + assert task.content in task_ledger + assert facts.content in task_ledger + assert plan.content in task_ledger + assert "{'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}" in task_ledger + + +async def test_magentic_manager_create_progress_ledger(): + """Test the create_progress_ledger method of the MagenticManager.""" + + mock_progress_ledger = ProgressLedger( + is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + is_progress_being_made=ProgressLedgerItem(answer=False, reason="mock_reasoning"), + next_speaker=ProgressLedgerItem(answer="agent_a", reason="mock_reasoning"), + instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"), + ) + + task = ChatMessageContent(role="user", content=uuid4().hex) + participants = {"agent_a": "test_agent_a", "agent_b": "test_agent_b"} + + with patch.object( + MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock + ) as mock_get_chat_message_content: + mock_get_chat_message_content.return_value = ChatMessageContent( + role="assistant", content=mock_progress_ledger.model_dump_json() + ) + + chat_completion_service = MockChatCompletionService(ai_model_id="test") + prompt_execution_settings = PromptExecutionSettings() + + manager = MagenticManager( + chat_completion_service=chat_completion_service, + prompt_execution_settings=prompt_execution_settings, + ) + + chat_history = ChatHistory() + progress_ledger = await manager.create_progress_ledger(chat_history, task, participants) + + assert isinstance(progress_ledger, ProgressLedger) + assert progress_ledger == mock_progress_ledger + + assert task.content in chat_history.messages[0].content + assert "{'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}" in chat_history.messages[0].content + assert "agent_a, agent_b" in chat_history.messages[0].content + assert ( + chat_history.messages[0].content + == mock_get_chat_message_content.call_args_list[0][0][0].messages[0].content + ) + assert mock_get_chat_message_content.call_args_list[0][0][1].extension_data["response_format"] == ProgressLedger + + +# endregion MagenticManager From ca839710644699d08d1b923f4754165cd7edd869 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 15 May 2025 15:48:18 -0700 Subject: [PATCH 2/9] Refine magentic manager base --- .../step5_magentic.py | 12 +- .../agents/orchestration/magentic.py | 408 ++++++++++-------- .../agents/orchestration/test_magentic.py | 259 ++++++++--- 3 files changed, 430 insertions(+), 249 deletions(-) diff --git a/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py b/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py index 66ac9e35c042..bdc7e253955d 100644 --- a/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py +++ b/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft. All rights reserved. import asyncio +import logging from semantic_kernel.agents import Agent, ChatCompletionAgent, MagenticOrchestration, OpenAIAssistantAgent from semantic_kernel.agents.orchestration.magentic import StandardMagenticManager @@ -22,6 +23,9 @@ The Magentic manager requires a chat completion model that supports structured output. """ +# Set up logging to see the invocation process +logging.basicConfig(level=logging.WARNING) # Set default level to WARNING +logging.getLogger("semantic_kernel.agents.orchestration.magentic").setLevel(logging.DEBUG) async def agents() -> list[Agent]: @@ -32,7 +36,7 @@ async def agents() -> list[Agent]: research_agent = ChatCompletionAgent( name="ResearchAgent", description="A helpful assistant with access to web search. Ask it to perform web searches.", - instructions=("You are a Researcher. You find information."), + instructions=("You are a Researcher. You find information and provide it as it is without any processing."), service=OpenAIChatCompletion(ai_model_id="gpt-4o-search-preview"), ) @@ -80,9 +84,9 @@ async def main(): # 3. Invoke the orchestration with a task and the runtime orchestration_result = await magentic_orchestration.invoke( task=( - "What are the 50 tallest buildings in the world? Create a table with their names" - " and heights grouped by country with a column of the average height of the buildings" - " in each country." + "What are the 50 tallest buildings in the world? Create a table showing the average height " + "of the buildings in each country, in descending order, along with the names of the tallest " + "buildings in that country and the counts of buildings that make top 50 in that country." ), runtime=runtime, ) diff --git a/python/semantic_kernel/agents/orchestration/magentic.py b/python/semantic_kernel/agents/orchestration/magentic.py index af03fa5a2987..edd74f23cffa 100644 --- a/python/semantic_kernel/agents/orchestration/magentic.py +++ b/python/semantic_kernel/agents/orchestration/magentic.py @@ -5,6 +5,9 @@ import sys from abc import ABC, abstractmethod from collections.abc import Awaitable, Callable +from typing import Annotated + +from pydantic import Field from semantic_kernel.agents.agent import Agent from semantic_kernel.agents.orchestration.agent_actor_base import ActorBase, AgentActorBase @@ -87,6 +90,31 @@ class ProgressLedger(KernelBaseModel): instruction_or_question: ProgressLedgerItem +class MagenticContext(KernelBaseModel): + """Context for the Magentic manager.""" + + task: Annotated[ChatMessageContent, Field(description="The task to be completed.")] + chat_history: Annotated[ + ChatHistory, Field(description="The chat history to be used to generate the facts and plan.") + ] = ChatHistory() + participant_descriptions: Annotated[ + dict[str, str], Field(description="The descriptions of the participants in the group.") + ] + round_count: Annotated[int, Field(description="The number of rounds completed.")] = 0 + stall_count: Annotated[int, Field(description="The number of stalls detected.")] = 0 + reset_count: Annotated[int, Field(description="The number of resets detected.")] = 0 + + def reset(self) -> None: + """Reset the context. + + This will clear the chat history and reset the stall count. + This won't reset the task, round count, or participant descriptions. + """ + self.chat_history.clear() + self.stall_count = 0 + self.reset_count += 1 + + # endregion Messages and Types # region MagenticManager @@ -95,62 +123,46 @@ class ProgressLedger(KernelBaseModel): class MagenticManager(KernelBaseModel, ABC): """Base class for the Magentic One manager.""" + max_stall_count: Annotated[int, Field(description="The maximum number of stalls allowed before a reset.", ge=0)] = 3 + max_reset_count: Annotated[int | None, Field(description="The maximum number of resets allowed.", ge=0)] = None + max_round_count: Annotated[ + int | None, Field(description="The maximum number of rounds (agent responses) allowed.", gt=0) + ] = None + @abstractmethod - async def create_facts_and_plan( - self, - chat_history: ChatHistory, - task: ChatMessageContent, - participant_descriptions: dict[str, str], - old_facts: ChatMessageContent | None = None, - ) -> tuple[ChatMessageContent, ChatMessageContent]: - """Create facts and plan for the task. + async def plan(self, magentic_context: MagenticContext) -> ChatMessageContent: + """Create a plan for the task. + + This is called when the task is first started. Args: - chat_history (ChatHistory): The chat history. This chat history will be modified by the function. - task (ChatMessageContent): The task. - participant_descriptions (dict[str, str]): The participant descriptions. - old_facts (ChatMessageContent | None): The old facts. If provided, the facts and plan update - prompts will be used. + magentic_context (MagenticContext): The context for the Magentic manager. Returns: - tuple[ChatMessageContent, ChatMessageContent]: The facts and plan. + ChatMessageContent: The task ledger. """ ... @abstractmethod - async def create_task_ledger( - self, - task: ChatMessageContent, - facts: ChatMessageContent, - plan: ChatMessageContent, - participant_descriptions: dict[str, str], - ) -> str: - """Create a task ledger. + async def replan(self, magentic_context: MagenticContext) -> ChatMessageContent: + """Replan for the task. + + This is called when the task is stalled or looping. Args: - task (ChatMessageContent): The task. - facts (ChatMessageContent): The facts. - plan (ChatMessageContent): The plan. - participant_descriptions (dict[str, str]): The participant descriptions. + magentic_context (MagenticContext): The context for the Magentic manager. Returns: - str: The task ledger. + ChatMessageContent: The updated task ledger. """ ... @abstractmethod - async def create_progress_ledger( - self, - chat_history: ChatHistory, - task: ChatMessageContent, - participant_descriptions: dict[str, str], - ) -> ProgressLedger: + async def create_progress_ledger(self, magentic_context: MagenticContext) -> ProgressLedger: """Create a progress ledger. Args: - chat_history (ChatHistory): The chat history. This chat history will be modified by the function. - task (ChatMessageContent): The task. - participant_descriptions (dict[str, str]): The participant descriptions. + magentic_context (MagenticContext): The context for the Magentic manager. Returns: ProgressLedger: The progress ledger. @@ -158,12 +170,11 @@ async def create_progress_ledger( ... @abstractmethod - async def prepare_final_answer(self, chat_history: ChatHistory, task: ChatMessageContent) -> ChatMessageContent: + async def prepare_final_answer(self, magentic_context: MagenticContext) -> ChatMessageContent: """Prepare the final answer. Args: - chat_history (ChatHistory): The chat history. This chat history will be modified by the function. - task (ChatMessageContent): The task. + magentic_context (MagenticContext): The context for the Magentic manager. Returns: ChatMessageContent: The final answer. @@ -172,13 +183,17 @@ async def prepare_final_answer(self, chat_history: ChatHistory, task: ChatMessag class StandardMagenticManager(MagenticManager): - """Container for the Magentic pattern.""" + """Standard Magentic manager implementation. + + This is the default implementation of the Magentic manager. + It uses the task ledger to keep track of the facts and plan for the task. + + This implementation is requires structured outputs. + """ chat_completion_service: ChatCompletionClientBase prompt_execution_settings: PromptExecutionSettings - max_stall_count: int = 3 - task_ledger_facts_prompt: str = ORCHESTRATOR_TASK_LEDGER_FACTS_PROMPT task_ledger_plan_prompt: str = ORCHESTRATOR_TASK_LEDGER_PLAN_PROMPT task_ledger_full_prompt: str = ORCHESTRATOR_TASK_LEDGER_FULL_PROMPT @@ -187,119 +202,153 @@ class StandardMagenticManager(MagenticManager): progress_ledger_prompt: str = ORCHESTRATOR_PROGRESS_LEDGER_PROMPT final_answer_prompt: str = ORCHESTRATOR_FINAL_ANSWER_PROMPT + class TaskLedger(KernelBaseModel): + """Task ledger for the Standard Magentic manager.""" + + facts: Annotated[ChatMessageContent, Field(description="The facts about the task.")] + plan: Annotated[ChatMessageContent, Field(description="The plan for the task.")] + + task_ledger: TaskLedger | None = None + @override - async def create_facts_and_plan( - self, - chat_history: ChatHistory, - task: ChatMessageContent, - participant_descriptions: dict[str, str], - old_facts: ChatMessageContent | None = None, - ) -> tuple[ChatMessageContent, ChatMessageContent]: - """Create facts and plan for the task. + async def plan(self, magentic_context: MagenticContext) -> ChatMessageContent: + """Plan the task. Args: - chat_history (ChatHistory): The chat history. This chat history will be modified by the function. - task (ChatMessageContent): The task. - participant_descriptions (dict[str, str]): The participant descriptions. - old_facts (ChatMessageContent | None): The old facts. If provided, the facts and plan update - prompts will be used. + magentic_context (MagenticContext): The context for the Magentic manager. Returns: - tuple[ChatMessageContent, ChatMessageContent]: The facts and plan. + ChatMessageContent: The task ledger. """ - # 1. Update the facts + # 1. Gather the facts prompt_template = KernelPromptTemplate( - prompt_template_config=PromptTemplateConfig( - template=self.task_ledger_facts_update_prompt if old_facts else self.task_ledger_facts_prompt + prompt_template_config=PromptTemplateConfig(template=self.task_ledger_facts_prompt) + ) + magentic_context.chat_history.add_message( + ChatMessageContent( + role=AuthorRole.USER, + content=await prompt_template.render(Kernel(), KernelArguments(task=magentic_context.task.content)), ) ) - chat_history.add_message( + facts = await self.chat_completion_service.get_chat_message_content( + magentic_context.chat_history, + self.prompt_execution_settings, + ) + assert facts is not None # nosec B101 + magentic_context.chat_history.add_message(facts) + + # 2. Create the plan + prompt_template = KernelPromptTemplate( + prompt_template_config=PromptTemplateConfig(template=self.task_ledger_plan_prompt) + ) + magentic_context.chat_history.add_message( ChatMessageContent( role=AuthorRole.USER, content=await prompt_template.render( Kernel(), - KernelArguments(task=task.content, old_facts=old_facts.content) - if old_facts - else KernelArguments(task=task.content), + KernelArguments(team=magentic_context.participant_descriptions), + ), + ) + ) + plan = await self.chat_completion_service.get_chat_message_content( + magentic_context.chat_history, + self.prompt_execution_settings, + ) + assert plan is not None # nosec B101 + + self.task_ledger = self.TaskLedger(facts=facts, plan=plan) + return await self._render_task_ledger(magentic_context) + + @override + async def replan(self, magentic_context: MagenticContext) -> ChatMessageContent: + """Replan the task. + + Args: + magentic_context (MagenticContext): The context for the Magentic manager. + + Returns: + ChatMessageContent: The updated task ledger. + """ + if self.task_ledger is None: + raise RuntimeError("The task ledger is not initialized. Planning needs to happen first.") + + # 1. Update the facts + prompt_template = KernelPromptTemplate( + prompt_template_config=PromptTemplateConfig(template=self.task_ledger_facts_update_prompt) + ) + magentic_context.chat_history.add_message( + ChatMessageContent( + role=AuthorRole.USER, + content=await prompt_template.render( + Kernel(), + KernelArguments(task=magentic_context.task.content, old_facts=self.task_ledger.facts.content), ), ) ) facts = await self.chat_completion_service.get_chat_message_content( - chat_history, + magentic_context.chat_history, self.prompt_execution_settings, ) assert facts is not None # nosec B101 - chat_history.add_message(facts) + magentic_context.chat_history.add_message(facts) # 2. Update the plan prompt_template = KernelPromptTemplate( - prompt_template_config=PromptTemplateConfig( - template=self.task_ledger_plan_update_prompt if old_facts else self.task_ledger_plan_prompt - ) + prompt_template_config=PromptTemplateConfig(template=self.task_ledger_plan_update_prompt) ) - chat_history.add_message( + magentic_context.chat_history.add_message( ChatMessageContent( role=AuthorRole.USER, content=await prompt_template.render( Kernel(), - KernelArguments(team=participant_descriptions), + KernelArguments(team=magentic_context.participant_descriptions), ), ) ) plan = await self.chat_completion_service.get_chat_message_content( - chat_history, + magentic_context.chat_history, self.prompt_execution_settings, ) assert plan is not None # nosec B101 - return facts, plan + self.task_ledger.facts = facts + self.task_ledger.plan = plan + return await self._render_task_ledger(magentic_context) - @override - async def create_task_ledger( - self, - task: ChatMessageContent, - facts: ChatMessageContent, - plan: ChatMessageContent, - participant_descriptions: dict[str, str], - ) -> str: - """Create a task ledger. + async def _render_task_ledger(self, magentic_context: MagenticContext) -> ChatMessageContent: + """Render the task ledger to a string. Args: - task (ChatMessageContent): The task. - facts (ChatMessageContent): The facts. - plan (ChatMessageContent): The plan. - participant_descriptions (dict[str, str]): The participant descriptions. + magentic_context (MagenticContext): The context for the Magentic manager. Returns: - str: The task ledger. + ChatMessageContent: The rendered task ledger. """ + if self.task_ledger is None: + raise RuntimeError("The task ledger is not initialized. Planning needs to happen first.") + prompt_template = KernelPromptTemplate( prompt_template_config=PromptTemplateConfig(template=self.task_ledger_full_prompt) ) - return await prompt_template.render( + rendered_task_ledger = await prompt_template.render( Kernel(), KernelArguments( - task=task.content, - team=participant_descriptions, - facts=facts.content, - plan=plan.content, + task=magentic_context.task.content, + team=magentic_context.participant_descriptions, + facts=self.task_ledger.facts.content, + plan=self.task_ledger.plan.content, ), ) + return ChatMessageContent(role=AuthorRole.ASSISTANT, content=rendered_task_ledger) + @override - async def create_progress_ledger( - self, - chat_history: ChatHistory, - task: ChatMessageContent, - participant_descriptions: dict[str, str], - ) -> ProgressLedger: + async def create_progress_ledger(self, magentic_context: MagenticContext) -> ProgressLedger: """Create a progress ledger. Args: - chat_history (ChatHistory): The chat history. This chat history will be modified by the function. - task (ChatMessageContent): The task. - participant_descriptions (dict[str, str]): The participant descriptions. + magentic_context (MagenticContext): The context for the Magentic manager. Returns: ProgressLedger: The progress ledger. @@ -310,12 +359,14 @@ async def create_progress_ledger( progress_ledger_prompt = await prompt_template.render( Kernel(), KernelArguments( - task=task.content, - team=participant_descriptions, - names=", ".join(participant_descriptions.keys()), + task=magentic_context.task.content, + team=magentic_context.participant_descriptions, + names=", ".join(magentic_context.participant_descriptions.keys()), ), ) - chat_history.add_message(ChatMessageContent(role=AuthorRole.USER, content=progress_ledger_prompt)) + magentic_context.chat_history.add_message( + ChatMessageContent(role=AuthorRole.USER, content=progress_ledger_prompt) + ) prompt_execution_settings_clone = PromptExecutionSettings.from_prompt_execution_settings( self.prompt_execution_settings @@ -326,7 +377,7 @@ async def create_progress_ledger( ) response = await self.chat_completion_service.get_chat_message_content( - chat_history, + magentic_context.chat_history, prompt_execution_settings_clone, ) assert response is not None # nosec B101 @@ -334,12 +385,11 @@ async def create_progress_ledger( return ProgressLedger.model_validate_json(response.content) @override - async def prepare_final_answer(self, chat_history: ChatHistory, task: ChatMessageContent) -> ChatMessageContent: + async def prepare_final_answer(self, magentic_context: MagenticContext) -> ChatMessageContent: """Prepare the final answer. Args: - chat_history (ChatHistory): The chat history. This chat history will be modified by the function. - task (ChatMessageContent): The task. + magentic_context (MagenticContext): The context for the Magentic manager. Returns: ChatMessageContent: The final answer. @@ -347,15 +397,15 @@ async def prepare_final_answer(self, chat_history: ChatHistory, task: ChatMessag prompt_template = KernelPromptTemplate( prompt_template_config=PromptTemplateConfig(template=self.final_answer_prompt) ) - chat_history.add_message( + magentic_context.chat_history.add_message( ChatMessageContent( role=AuthorRole.USER, - content=await prompt_template.render(Kernel(), KernelArguments(task=task)), + content=await prompt_template.render(Kernel(), KernelArguments(task=magentic_context.task)), ) ) response = await self.chat_completion_service.get_chat_message_content( - chat_history, + magentic_context.chat_history, self.prompt_execution_settings, ) assert response is not None # nosec B101 @@ -388,11 +438,10 @@ def __init__( """ self._manager = manager self._internal_topic_type = internal_topic_type - self._chat_history = ChatHistory() - self._participant_descriptions = participant_descriptions self._result_callback = result_callback - self._round_count = 0 - self._stall_count = 0 + self._participant_descriptions = participant_descriptions + self._context: MagenticContext | None = None + self._task_ledger: ChatMessageContent | None = None super().__init__(description="Magentic One Manager") @@ -400,71 +449,74 @@ def __init__( async def _handle_start_message(self, message: MagenticStartMessage, ctx: MessageContext) -> None: """Handle the start message for the Magentic One manager.""" logger.debug(f"{self.id}: Received Magentic One start message.") - self._task = message.body - self._facts, self._plan = await self._manager.create_facts_and_plan( - self._chat_history.model_copy(deep=True), - self._task, - self._participant_descriptions, + + self._context = MagenticContext( + task=message.body, + participant_descriptions=self._participant_descriptions, ) + # Initial planning + self._task_ledger = await self._manager.plan(self._context.model_copy(deep=True)) + await self._run_outer_loop(ctx.cancellation_token) @message_handler async def _handle_response_message(self, message: MagenticResponseMessage, ctx: MessageContext) -> None: + """Handle the response message for the Magentic One manager.""" + if self._context is None or self._task_ledger is None: + raise RuntimeError("The Magentic manager is not started yet. Make sure to send a start message first.") + if message.body.role != AuthorRole.USER: - self._chat_history.add_message( + self._context.chat_history.add_message( ChatMessageContent( role=AuthorRole.USER, content=f"Transferred to {message.body.name}", ) ) - self._chat_history.add_message(message.body) + self._context.chat_history.add_message(message.body) logger.debug(f"{self.id}: Running inner loop.") await self._run_inner_loop(ctx.cancellation_token) async def _run_outer_loop(self, cancellation_token: CancellationToken) -> None: - # 1. Create a task ledger. - task_ledger = await self._manager.create_task_ledger( - self._task, - self._facts, - self._plan, - self._participant_descriptions, - ) + if self._context is None or self._task_ledger is None: + raise RuntimeError("The Magentic manager is not started yet. Make sure to send a start message first.") - # 2. Publish the task ledger to the group chat. + # 1. Publish the rendered task ledger to the group chat. # Need to add the task ledger to the orchestrator's chat history # since the publisher won't receive the message it sends even though # the publisher also subscribes to the topic. - self._chat_history.add_message( + self._context.chat_history.add_message( ChatMessageContent( role=AuthorRole.ASSISTANT, - content=task_ledger, + content=self._task_ledger.content, name=self.__class__.__name__, ) ) - logger.debug(f"Initial task ledger:\n{task_ledger}") + logger.debug(f"Initial task ledger:\n{self._task_ledger.content}") await self.publish_message( MagenticResponseMessage( - body=self._chat_history.messages[-1], + body=self._context.chat_history.messages[-1], ), TopicId(self._internal_topic_type, self.id.key), cancellation_token=cancellation_token, ) - # 3. Start the inner loop. + # 2. Start the inner loop. await self._run_inner_loop(cancellation_token) async def _run_inner_loop(self, cancellation_token: CancellationToken) -> None: - self._round_count += 1 + if self._context is None or self._task_ledger is None: + raise RuntimeError("The Magentic manager is not started yet. Make sure to send a start message first.") + + within_limits = await self._check_within_limits() + if not within_limits: + return + self._context.round_count += 1 # 1. Create a progress ledger - current_progress_ledger = await self._manager.create_progress_ledger( - self._chat_history.model_copy(deep=True), - self._task, - self._participant_descriptions, - ) + current_progress_ledger = await self._manager.create_progress_ledger(self._context.model_copy(deep=True)) logger.debug(f"Current progress ledger:\n{current_progress_ledger.model_dump_json(indent=2)}") # 2. Process the progress ledger @@ -475,18 +527,13 @@ async def _run_inner_loop(self, cancellation_token: CancellationToken) -> None: return # 2.2 Check for stalling or looping if not current_progress_ledger.is_progress_being_made.answer or current_progress_ledger.is_in_loop.answer: - self._stall_count += 1 + self._context.stall_count += 1 else: - self._stall_count = max(0, self._stall_count - 1) + self._context.stall_count = max(0, self._context.stall_count - 1) - if self._stall_count > self._manager.max_stall_count: + if self._context.stall_count > self._manager.max_stall_count: logger.debug("Stalling detected. Resetting the task.") - self._facts, self._plan = await self._manager.create_facts_and_plan( - self._chat_history.model_copy(deep=True), - self._task, - self._participant_descriptions, - old_facts=self._facts, - ) + self._task_ledger = await self._manager.replan(self._context.model_copy(deep=True)) await self._reset_for_outer_loop(cancellation_token) logger.debug("Restarting outer loop.") await self._run_outer_loop(cancellation_token) @@ -494,7 +541,7 @@ async def _run_inner_loop(self, cancellation_token: CancellationToken) -> None: # 2.3 Publish for next step next_step = current_progress_ledger.instruction_or_question.answer - self._chat_history.add_message( + self._context.chat_history.add_message( ChatMessageContent( role=AuthorRole.ASSISTANT, content=next_step if isinstance(next_step, str) else str(next_step), @@ -503,7 +550,7 @@ async def _run_inner_loop(self, cancellation_token: CancellationToken) -> None: ) await self.publish_message( MagenticResponseMessage( - body=self._chat_history.messages[-1], + body=self._context.chat_history.messages[-1], ), TopicId(self._internal_topic_type, self.id.key), cancellation_token=cancellation_token, @@ -523,23 +570,49 @@ async def _run_inner_loop(self, cancellation_token: CancellationToken) -> None: ) async def _reset_for_outer_loop(self, cancellation_token: CancellationToken) -> None: + """Reset the context for the outer loop.""" + if self._context is None: + raise RuntimeError("The Magentic manager is not started yet. Make sure to send a start message first.") + await self.publish_message( MagenticResetMessage(), TopicId(self._internal_topic_type, self.id.key), cancellation_token=cancellation_token, ) - self._chat_history.clear() - self._stall_count = 0 + self._context.reset() async def _prepare_final_answer(self) -> None: - final_answer = await self._manager.prepare_final_answer( - self._chat_history.model_copy(deep=True), - self._task, - ) + """Prepare the final answer and send it to the result callback.""" + if self._context is None: + raise RuntimeError("The Magentic manager is not started yet. Make sure to send a start message first.") + + final_answer = await self._manager.prepare_final_answer(self._context.model_copy(deep=True)) if self._result_callback: await self._result_callback(final_answer) + async def _check_within_limits(self) -> bool: + """Check if the manager is within the limits.""" + if self._context is None: + raise RuntimeError("The Magentic manager is not started yet. Make sure to send a start message first.") + + if ( + self._manager.max_round_count is not None and self._context.round_count >= self._manager.max_round_count + ) or (self._manager.max_reset_count is not None and self._context.reset_count > self._manager.max_reset_count): + message = ( + "Max round count reached." + if self._manager.max_round_count and self._context.round_count >= self._manager.max_round_count + else "Max reset count reached." + ) + logger.debug(message) + if self._result_callback: + await self._result_callback( + ChatMessageContent(role=AuthorRole.ASSISTANT, content=message, name=self.__class__.__name__) + ) + return False + + return True + # endregion MagenticManagerActor @@ -667,28 +740,11 @@ async def _start( internal_topic_type: str, cancellation_token: CancellationToken, ) -> None: - """Start the Magentic One pattern. - - This ensures that all initial messages are sent to the individual actors - and processed before the group chat begins. It's important because if the - manager actor processes its start message too quickly (or other actors are - too slow), it might send a request to the next agent before the other actors - have the necessary context. - """ + """Start the Magentic pattern.""" if not isinstance(task, ChatMessageContent): # Magentic One only supports ChatMessageContent as input. raise ValueError("The task must be a ChatMessageContent object.") - async def send_start_message(agent: Agent) -> None: - target_actor_id = await runtime.get(self._get_agent_actor_type(agent, internal_topic_type)) - await runtime.send_message( - MagenticStartMessage(body=task), - target_actor_id, - cancellation_token=cancellation_token, - ) - - await asyncio.gather(*[send_start_message(agent) for agent in self._members]) - target_actor_id = await runtime.get(self._get_manager_actor_type(internal_topic_type)) await runtime.send_message( MagenticStartMessage(body=task), diff --git a/python/tests/unit/agents/orchestration/test_magentic.py b/python/tests/unit/agents/orchestration/test_magentic.py index d354d1b4d13e..935469cbe216 100644 --- a/python/tests/unit/agents/orchestration/test_magentic.py +++ b/python/tests/unit/agents/orchestration/test_magentic.py @@ -1,15 +1,16 @@ # Copyright (c) Microsoft. All rights reserved. +import sys from unittest.mock import AsyncMock, patch -from uuid import uuid4 import pytest from semantic_kernel.agents.orchestration.magentic import ( - MagenticManager, + MagenticContext, MagenticOrchestration, ProgressLedger, ProgressLedgerItem, + StandardMagenticManager, ) from semantic_kernel.agents.orchestration.orchestration_base import DefaultTypeAlias, OrchestrationResult from semantic_kernel.agents.orchestration.prompts._magentic_prompts import ( @@ -47,7 +48,7 @@ async def test_init_member_without_description_throws(): with pytest.raises(ValueError): MagenticOrchestration( members=[agent_a, agent_b], - manager=MagenticManager( + manager=StandardMagenticManager( chat_completion_service=MockChatCompletionService(ai_model_id="test"), prompt_execution_settings=PromptExecutionSettings(), ), @@ -70,7 +71,7 @@ async def test_prepare(): ): orchestration = MagenticOrchestration( members=[agent_a, agent_b], - manager=MagenticManager( + manager=StandardMagenticManager( chat_completion_service=MockChatCompletionService(ai_model_id="test"), prompt_execution_settings=PromptExecutionSettings(), ), @@ -155,6 +156,10 @@ async def test_prepare(): ] +@pytest.mark.skipif( + sys.version_info < (3, 11), + reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.", +) async def test_invoke(): """Test the invoke method of the MagenticOrchestration.""" with ( @@ -163,14 +168,14 @@ async def test_invoke(): MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock ) as mock_get_chat_message_content, patch.object( - MagenticManager, "create_progress_ledger", new_callable=AsyncMock, side_effect=ManagerProgressList + StandardMagenticManager, "create_progress_ledger", new_callable=AsyncMock, side_effect=ManagerProgressList ), ): mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response") chat_completion_service = MockChatCompletionService(ai_model_id="test") prompt_execution_settings = PromptExecutionSettings() - manager = MagenticManager( + manager = StandardMagenticManager( chat_completion_service=chat_completion_service, prompt_execution_settings=prompt_execution_settings, ) @@ -202,7 +207,7 @@ async def test_invoke_with_list_error(): chat_completion_service = MockChatCompletionService(ai_model_id="test") prompt_execution_settings = PromptExecutionSettings() - manager = MagenticManager( + manager = StandardMagenticManager( chat_completion_service=chat_completion_service, prompt_execution_settings=prompt_execution_settings, ) @@ -229,6 +234,10 @@ async def test_invoke_with_list_error(): await orchestration_result.get() +@pytest.mark.skipif( + sys.version_info < (3, 11), + reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.", +) async def test_invoke_with_response_callback(): """Test the invoke method of the MagenticOrchestration with a response callback.""" @@ -242,7 +251,7 @@ async def test_invoke_with_response_callback(): MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock ) as mock_get_chat_message_content, patch.object( - MagenticManager, "create_progress_ledger", new_callable=AsyncMock, side_effect=ManagerProgressList + StandardMagenticManager, "create_progress_ledger", new_callable=AsyncMock, side_effect=ManagerProgressList ), ): mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response") @@ -253,7 +262,7 @@ async def test_invoke_with_response_callback(): try: orchestration = MagenticOrchestration( members=[agent_a, agent_b], - manager=MagenticManager( + manager=StandardMagenticManager( chat_completion_service=MockChatCompletionService(ai_model_id="test"), prompt_execution_settings=PromptExecutionSettings(), ), @@ -269,6 +278,10 @@ async def test_invoke_with_response_callback(): assert all(item.content == "mock_response" for item in responses) +@pytest.mark.skipif( + sys.version_info < (3, 11), + reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.", +) async def test_invoke_with_max_stall_count_exceeded(): """ "Test the invoke method of the MagenticOrchestration with max stall count exceeded.""" runtime = InProcessRuntime() @@ -280,7 +293,7 @@ async def test_invoke_with_max_stall_count_exceeded(): MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock ) as mock_get_chat_message_content, patch.object( - MagenticManager, + StandardMagenticManager, "create_progress_ledger", new_callable=AsyncMock, side_effect=ManagerProgressListStalling, @@ -294,7 +307,7 @@ async def test_invoke_with_max_stall_count_exceeded(): try: orchestration = MagenticOrchestration( members=[agent_a, agent_b], - manager=MagenticManager( + manager=StandardMagenticManager( chat_completion_service=MockChatCompletionService(ai_model_id="test"), prompt_execution_settings=PromptExecutionSettings(), max_stall_count=1, @@ -311,6 +324,103 @@ async def test_invoke_with_max_stall_count_exceeded(): assert mock_get_chat_message_content.call_count == 5 +@pytest.mark.skipif( + sys.version_info < (3, 11), + reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.", +) +async def test_invoke_with_max_round_count_exceeded(): + """ "Test the invoke method of the MagenticOrchestration with max round count exceeded.""" + runtime = InProcessRuntime() + runtime.start() + + with ( + patch.object(MockAgent, "get_response", wraps=MockAgent.get_response, autospec=True) as mock_get_response, + patch.object( + MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock + ) as mock_get_chat_message_content, + patch.object( + StandardMagenticManager, + "create_progress_ledger", + new_callable=AsyncMock, + side_effect=ManagerProgressListStalling, + ), + ): + mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response") + + agent_a = MockAgent(name="agent_a", description="test agent") + agent_b = MockAgent(name="agent_b", description="test agent") + + try: + orchestration = MagenticOrchestration( + members=[agent_a, agent_b], + manager=StandardMagenticManager( + chat_completion_service=MockChatCompletionService(ai_model_id="test"), + prompt_execution_settings=PromptExecutionSettings(), + max_round_count=1, + ), + ) + orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime) + result = await orchestration_result.get(1.0) + finally: + await runtime.stop_when_idle() + + assert result.content == "Max round count reached." + assert mock_get_response.call_count == 1 + # Planning will be called once, so the facts and plan will be created once. + assert mock_get_chat_message_content.call_count == 2 + + +@pytest.mark.skipif( + sys.version_info < (3, 11), + reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.", +) +async def test_invoke_with_max_reset_count_exceeded(): + """ "Test the invoke method of the MagenticOrchestration with max reset count exceeded.""" + runtime = InProcessRuntime() + runtime.start() + + with ( + patch.object(MockAgent, "get_response", wraps=MockAgent.get_response, autospec=True) as mock_get_response, + patch.object( + MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock + ) as mock_get_chat_message_content, + patch.object( + StandardMagenticManager, + "create_progress_ledger", + new_callable=AsyncMock, + side_effect=ManagerProgressListStalling, + ), + ): + mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response") + + agent_a = MockAgent(name="agent_a", description="test agent") + agent_b = MockAgent(name="agent_b", description="test agent") + + try: + orchestration = MagenticOrchestration( + members=[agent_a, agent_b], + manager=StandardMagenticManager( + chat_completion_service=MockChatCompletionService(ai_model_id="test"), + prompt_execution_settings=PromptExecutionSettings(), + max_stall_count=0, # No stall allowed + max_reset_count=0, # No reset allowed + ), + ) + orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime) + result = await orchestration_result.get(1.0) + finally: + await runtime.stop_when_idle() + + assert result.content == "Max reset count reached." + assert mock_get_response.call_count == 1 + # Planning and replanning will be each called once, so the facts and plan will be created twice. + assert mock_get_chat_message_content.call_count == 4 + + +@pytest.mark.skipif( + sys.version_info < (3, 11), + reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.", +) async def test_invoke_with_unknown_speaker(): """Test the invoke method of the MagenticOrchestration with an unknown speaker.""" runtime = InProcessRuntime() @@ -322,7 +432,7 @@ async def test_invoke_with_unknown_speaker(): MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock ) as mock_get_chat_message_content, patch.object( - MagenticManager, + StandardMagenticManager, "create_progress_ledger", new_callable=AsyncMock, side_effect=ManagerProgressListUnknownSpeaker, @@ -337,7 +447,7 @@ async def test_invoke_with_unknown_speaker(): try: orchestration = MagenticOrchestration( members=[agent_a, agent_b], - manager=MagenticManager( + manager=StandardMagenticManager( chat_completion_service=MockChatCompletionService(ai_model_id="test"), prompt_execution_settings=PromptExecutionSettings(), ), @@ -350,20 +460,22 @@ async def test_invoke_with_unknown_speaker(): # endregion MagenticOrchestration -# region MagenticManager +# region StandardMagenticManager -def test_magentic_manager_init(): - """Test the initialization of the MagenticManager.""" +def test_standard_magentic_manager_init(): + """Test the initialization of the StandardMagenticManager.""" chat_completion_service = MockChatCompletionService(ai_model_id="test") prompt_execution_settings = PromptExecutionSettings() - manager = MagenticManager( + manager = StandardMagenticManager( chat_completion_service=chat_completion_service, prompt_execution_settings=prompt_execution_settings, ) assert manager.max_stall_count > 0 + assert manager.max_reset_count is None + assert manager.max_round_count is None assert ( manager.task_ledger_facts_prompt is not None and manager.task_ledger_facts_prompt == ORCHESTRATOR_TASK_LEDGER_FACTS_PROMPT @@ -391,12 +503,12 @@ def test_magentic_manager_init(): assert manager.final_answer_prompt is not None and manager.final_answer_prompt == ORCHESTRATOR_FINAL_ANSWER_PROMPT -def test_magentic_manager_init_with_custom_prompts(): - """Test the initialization of the MagenticManager with custom prompts.""" +def test_standard_magentic_manager_init_with_custom_prompts(): + """Test the initialization of the StandardMagenticManager with custom prompts.""" chat_completion_service = MockChatCompletionService(ai_model_id="test") prompt_execution_settings = PromptExecutionSettings() - manager = MagenticManager( + manager = StandardMagenticManager( chat_completion_service=chat_completion_service, prompt_execution_settings=prompt_execution_settings, task_ledger_facts_prompt="custom_task_ledger_facts_prompt", @@ -417,8 +529,8 @@ def test_magentic_manager_init_with_custom_prompts(): assert manager.final_answer_prompt == "custom_final_answer_prompt" -async def test_magentic_manager_create_facts_and_prompt(): - """Test the create_facts_and_prompt method of the MagenticManager.""" +async def test_standard_magentic_manager_plan(): + """Test the plan method of the StandardMagenticManager.""" with patch.object( MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock @@ -427,21 +539,25 @@ async def test_magentic_manager_create_facts_and_prompt(): chat_completion_service = MockChatCompletionService(ai_model_id="test") prompt_execution_settings = PromptExecutionSettings() - manager = MagenticManager( + manager = StandardMagenticManager( chat_completion_service=chat_completion_service, prompt_execution_settings=prompt_execution_settings, task_ledger_facts_prompt="custom_task_ledger_facts_prompt", task_ledger_plan_prompt="custom_task_ledger_plan_prompt {{$team}}", ) - facts, prompt = await manager.create_facts_and_plan( - ChatHistory(), - ChatMessageContent(role="user", content="test_message"), - {"agent_a": "test_agent_a", "agent_b": "test_agent_b"}, + magentic_context = MagenticContext( + chat_history=ChatHistory(), + task=ChatMessageContent(role="user", content="test_message"), + participant_descriptions={"agent_a": "test_agent_a", "agent_b": "test_agent_b"}, ) - assert isinstance(facts, ChatMessageContent) and facts.content == "mock_response" - assert isinstance(prompt, ChatMessageContent) and prompt.content == "mock_response" + task_ledger = await manager.plan(magentic_context.model_copy(deep=True)) + + assert isinstance(task_ledger, ChatMessageContent) + assert task_ledger.content.count("mock_response") == 2 + assert "test_message" in task_ledger.content + assert "{'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}" in task_ledger.content assert mock_get_chat_message_content.call_count == 2 assert ( @@ -454,8 +570,8 @@ async def test_magentic_manager_create_facts_and_prompt(): ) -async def test_magentic_manager_create_facts_and_prompt_with_old_facts(): - """Test the create_facts_and_prompt method of the MagenticManager with old facts.""" +async def test_standard_magentic_manager_replan(): + """Test the replan method of the StandardMagenticManager.""" with patch.object( MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock @@ -465,60 +581,62 @@ async def test_magentic_manager_create_facts_and_prompt_with_old_facts(): chat_completion_service = MockChatCompletionService(ai_model_id="test") prompt_execution_settings = PromptExecutionSettings() - manager = MagenticManager( + manager = StandardMagenticManager( chat_completion_service=chat_completion_service, prompt_execution_settings=prompt_execution_settings, task_ledger_facts_update_prompt="custom_task_ledger_facts_prompt {{$old_facts}}", task_ledger_plan_update_prompt="custom_task_ledger_plan_prompt {{$team}}", ) - facts, prompt = await manager.create_facts_and_plan( - ChatHistory(), - ChatMessageContent(role="user", content="test_message"), - {"agent_a": "test_agent_a", "agent_b": "test_agent_b"}, - old_facts=ChatMessageContent(role="user", content="old_facts"), + magentic_context = MagenticContext( + chat_history=ChatHistory(), + task=ChatMessageContent(role="user", content="test_message"), + participant_descriptions={"agent_a": "test_agent_a", "agent_b": "test_agent_b"}, ) - assert isinstance(facts, ChatMessageContent) and facts.content == "mock_response" - assert isinstance(prompt, ChatMessageContent) and prompt.content == "mock_response" + # Need to plan before replanning + _ = await manager.plan(magentic_context.model_copy(deep=True)) + task_ledger = await manager.replan(magentic_context.model_copy(deep=True)) - assert mock_get_chat_message_content.call_count == 2 + assert isinstance(task_ledger, ChatMessageContent) + assert task_ledger.content.count("mock_response") == 2 + assert "test_message" in task_ledger.content + assert "{'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}" in task_ledger.content + + assert mock_get_chat_message_content.call_count == 4 assert ( - mock_get_chat_message_content.call_args_list[0][0][0].messages[0].content - == "custom_task_ledger_facts_prompt old_facts" + mock_get_chat_message_content.call_args_list[2][0][0].messages[0].content + == "custom_task_ledger_facts_prompt mock_response" ) assert ( - mock_get_chat_message_content.call_args_list[1][0][0].messages[2].content + mock_get_chat_message_content.call_args_list[3][0][0].messages[2].content == "custom_task_ledger_plan_prompt {'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}" ) -async def test_magentic_manager_create_task_ledger(): - """Test the create_task_ledger method of the MagenticManager.""" +async def test_standard_magentic_manager_replan_without_plan(): + """Test the replan method of the StandardMagenticManager.""" chat_completion_service = MockChatCompletionService(ai_model_id="test") prompt_execution_settings = PromptExecutionSettings() - manager = MagenticManager( + manager = StandardMagenticManager( chat_completion_service=chat_completion_service, prompt_execution_settings=prompt_execution_settings, ) - task = ChatMessageContent(role="user", content=uuid4().hex) - facts = ChatMessageContent(role="user", content=uuid4().hex) - plan = ChatMessageContent(role="user", content=uuid4().hex) - participants = {"agent_a": "test_agent_a", "agent_b": "test_agent_b"} - - task_ledger = await manager.create_task_ledger(task, facts, plan, participants) + magentic_context = MagenticContext( + chat_history=ChatHistory(), + task=ChatMessageContent(role="user", content="test_message"), + participant_descriptions={"agent_a": "test_agent_a", "agent_b": "test_agent_b"}, + ) - assert task.content in task_ledger - assert facts.content in task_ledger - assert plan.content in task_ledger - assert "{'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}" in task_ledger + with pytest.raises(RuntimeError): + _ = await manager.replan(magentic_context.model_copy(deep=True)) -async def test_magentic_manager_create_progress_ledger(): - """Test the create_progress_ledger method of the MagenticManager.""" +async def test_standard_magentic_manager_create_progress_ledger(): + """Test the create_progress_ledger method of the StandardMagenticManager.""" mock_progress_ledger = ProgressLedger( is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"), @@ -528,9 +646,6 @@ async def test_magentic_manager_create_progress_ledger(): instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"), ) - task = ChatMessageContent(role="user", content=uuid4().hex) - participants = {"agent_a": "test_agent_a", "agent_b": "test_agent_b"} - with patch.object( MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock ) as mock_get_chat_message_content: @@ -541,23 +656,29 @@ async def test_magentic_manager_create_progress_ledger(): chat_completion_service = MockChatCompletionService(ai_model_id="test") prompt_execution_settings = PromptExecutionSettings() - manager = MagenticManager( + manager = StandardMagenticManager( chat_completion_service=chat_completion_service, prompt_execution_settings=prompt_execution_settings, ) - chat_history = ChatHistory() - progress_ledger = await manager.create_progress_ledger(chat_history, task, participants) + magentic_context = MagenticContext( + chat_history=ChatHistory(), + task=ChatMessageContent(role="user", content="test_message"), + participant_descriptions={"agent_a": "test_agent_a", "agent_b": "test_agent_b"}, + ) + + progress_ledger = await manager.create_progress_ledger(magentic_context.model_copy(deep=True)) assert isinstance(progress_ledger, ProgressLedger) assert progress_ledger == mock_progress_ledger - assert task.content in chat_history.messages[0].content - assert "{'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}" in chat_history.messages[0].content - assert "agent_a, agent_b" in chat_history.messages[0].content assert ( - chat_history.messages[0].content - == mock_get_chat_message_content.call_args_list[0][0][0].messages[0].content + "{'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}" + in mock_get_chat_message_content.call_args_list[0][0][0].messages[0].content + ) + assert "agent_a, agent_b" in mock_get_chat_message_content.call_args_list[0][0][0].messages[0].content + assert ( + magentic_context.task.content in mock_get_chat_message_content.call_args_list[0][0][0].messages[0].content ) assert mock_get_chat_message_content.call_args_list[0][0][1].extension_data["response_format"] == ProgressLedger From 7c01aa944ac3b1de8c463a97f311d16a7eef9bce Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 15 May 2025 17:24:19 -0700 Subject: [PATCH 3/9] Tune magentic sample --- .../step5_magentic.py | 165 ++++++------------ 1 file changed, 56 insertions(+), 109 deletions(-) diff --git a/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py b/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py index bdc7e253955d..2e5c677cdd6f 100644 --- a/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py +++ b/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft. All rights reserved. import asyncio -import logging from semantic_kernel.agents import Agent, ChatCompletionAgent, MagenticOrchestration, OpenAIAssistantAgent from semantic_kernel.agents.orchestration.magentic import StandardMagenticManager @@ -23,9 +22,6 @@ The Magentic manager requires a chat completion model that supports structured output. """ -# Set up logging to see the invocation process -logging.basicConfig(level=logging.WARNING) # Set default level to WARNING -logging.getLogger("semantic_kernel.agents.orchestration.magentic").setLevel(logging.DEBUG) async def agents() -> list[Agent]: @@ -36,7 +32,10 @@ async def agents() -> list[Agent]: research_agent = ChatCompletionAgent( name="ResearchAgent", description="A helpful assistant with access to web search. Ask it to perform web searches.", - instructions=("You are a Researcher. You find information and provide it as it is without any processing."), + instructions=( + "You are a Researcher. You find information without additional computation or quantitative analysis." + ), + # This agent requires the gpt-4o-search-preview model to perform web searches. service=OpenAIChatCompletion(ai_model_id="gpt-4o-search-preview"), ) @@ -47,7 +46,7 @@ async def agents() -> list[Agent]: model=model, name="CoderAgent", description="A helpful assistant with code interpreter capability.", - instructions="You solve questions using code.", + instructions="You solve questions using code. Please provide detailed analysis and computation process.", tools=code_interpreter_tool, tool_resources=code_interpreter_tool_resources, ) @@ -84,16 +83,22 @@ async def main(): # 3. Invoke the orchestration with a task and the runtime orchestration_result = await magentic_orchestration.invoke( task=( - "What are the 50 tallest buildings in the world? Create a table showing the average height " - "of the buildings in each country, in descending order, along with the names of the tallest " - "buildings in that country and the counts of buildings that make top 50 in that country." + "The 2025 trade war between the US and other countries has had a significant impact " + "on the global economy. I am a business owner in the US that import household goods " + "such as bed sheets and holiday decorations from south-east Asia. I want " + "to know the impact of the tariffs on my business given that my current profit " + "margin is 20%. And If I were to increase the price of my products by 10%, " + "how would that affect my customer behavior and profit margin? Base on the analysis, " + "find similar cases in the past to cross-reference the results. Provide a detailed " + "report and recommendations on how to adapt to the changing market conditions at the end." ), runtime=runtime, ) # 4. Wait for the results value = await orchestration_result.get() - print(value) + + print(f"\nFinal result:\n{value}") # 5. Stop the runtime when idle await runtime.stop_when_idle() @@ -101,106 +106,48 @@ async def main(): """ Sample output: **ResearchAgent** - Based on the available information, here is a list of the 50 tallest buildings in the world, including their names, - heights, and countries: - - | Rank | Building Name | Height (m) | Country | - |------|---------------------------------------|------------|-------------------------| - | 1 | Burj Khalifa | 828 | United Arab Emirates | - | 2 | Merdeka 118 | 679 | Malaysia | - | 3 | Shanghai Tower | 632 | China | - | 4 | Makkah Royal Clock Tower | 601 | Saudi Arabia | - | 5 | Ping An Finance Center | 599 | China | - | 6 | Lotte World Tower | 555 | South Korea | - | 7 | One World Trade Center | 541 | United States | - | 8 | Guangzhou CTF Finance Centre | 530 | China | - | 9 | Tianjin CTF Finance Centre | 530 | China | - | 10 | CITIC Tower | 528 | China | - | 11 | TAIPEI 101 | 508 | Taiwan | - | 12 | Shanghai World Financial Center | 492 | China | - | 13 | International Commerce Centre | 484 | Hong Kong | - | 14 | Wuhan Greenland Center | 476 | China | - | 15 | Central Park Tower | 472 | United States | - | 16 | Lakhta Center | 462 | Russia | - | 17 | Vincom Landmark 81 | 461 | Vietnam | - | 18 | Changsha IFS Tower T1 | 452 | China | - | 19 | Petronas Tower 1 | 452 | Malaysia | - | 20 | Petronas Tower 2 | 452 | Malaysia | - | 21 | Suzhou IFS | 450 | China | - | 22 | Zifeng Tower | 450 | China | - | 23 | The Exchange 106 | 445 | Malaysia | - | 24 | Wuhan Center Tower | 443 | China | - | 25 | Willis Tower | 442 | United States | - | 26 | KK100 | 442 | China | - | 27 | Guangzhou International Finance Center| 438 | China | - | 28 | Wuhan Greenland Center | 438 | China | - | 29 | 432 Park Avenue | 425 | United States | - | 30 | Marina 101 | 425 | United Arab Emirates | - | 31 | Trump International Hotel & Tower | 423 | United States | - | 32 | Jin Mao Tower | 421 | China | - | 33 | Princess Tower | 414 | United Arab Emirates | - | 34 | Al Hamra Tower | 413 | Kuwait | - | 35 | Two International Finance Centre | 412 | Hong Kong | - | 36 | 23 Marina | 392 | United Arab Emirates | - | 37 | CITIC Plaza | 391 | China | - | 38 | Shun Hing Square | 384 | China | - | 39 | Eton Place Dalian Tower 1 | 383 | China | - | 40 | Empire State Building | 381 | United States | - | 41 | Burj Mohammed Bin Rashid | 381 | United Arab Emirates | - | 42 | Elite Residence | 380 | United Arab Emirates | - | 43 | The Address Boulevard | 370 | United Arab Emirates | - | 44 | Bank of China Tower | 367 | Hong Kong | - | 45 | Bank of America Tower | 366 | United States | - | 46 | St. Regis Chicago | 363 | United States | - | 47 | Almas Tower | 360 | United Arab Emirates | - | 48 | Hanking Center | 359 | China | - | 49 | Guangzhou Chow Tai Fook Finance Centre| 530 | China | - | 50 | Tianjin Chow Tai Fook Binhai Center | 530 | China | - - *Note: The heights are measured to the architectural top, including spires but excluding antennas, signage, flag - poles, or other functional or technical equipment.* - - This information is compiled from various sources, including the Council on Tall Buildings and Urban Habitat - (CTBUH) and other reputable architectural databases. + The 2025 trade war has led to significant tariffs imposed by the United States on imports from Southeast Asian + countries, directly affecting industries such as household goods. For instance, Cambodia faces a 49% tariff, + Vietnam 46%, and Thailand 36% on their exports to the U.S. + ([thailandinfo.se](https://www.thailandinfo.se/en/usa-tariffs-southeast-asia-2025/?utm_source=openai)) + + ... + **CoderAgent** + Here's the analysis based on your scenario: + + 1. **Initial Scenario:** + - Initial Selling Price: $125.00 (to achieve a 20% profit margin) + + 2. **After Applying Tariffs:** + - New Cost Price: $145.00 (after a 45% tariff on the initial $100 cost) + + 3. **With a 10% Price Increase:** + - New Selling Price: $137.50 + + 4. **Profit Margin and Volume Impact:** + ... + **ResearchAgent** + In response to increased tariffs during trade wars, various companies have implemented strategic measures to + mitigate financial impacts and maintain competitiveness. Notable examples include: + + **1. Supply Chain Diversification:** + + - **Steven Madden Ltd.:** Faced with a 10% tariff on handbags imported from China, the company relocated + production to Cambodia to circumvent the tariffs.([money.usnews.com](https://money.usnews.com/money/blogs/... **CoderAgent** - Here's the table of the 50 tallest buildings in the world, grouped by country with the buildings' names, heights, - and the average height for each country: - - | Country | Number of Buildings | Average Height (m) | Building Names & Heights | - |-----------------------|---------------------|---------------------|---------------------------------------------| - | China | 21 | 471.33 | Shanghai Tower (632m), Ping An Finance ... | - | Hong Kong | 3 | 421.00 | International Commerce Centre (484m), T ... | - | Kuwait | 1 | 413.00 | Al Hamra Tower (413m) ... | - | Malaysia | 4 | 507.00 | Merdeka 118 (679m), Petronas Tower 1 (4 ... | - | Russia | 1 | 462.00 | Lakhta Center (462m) ... | - | Saudi Arabia | 1 | 601.00 | Makkah Royal Clock Tower (601m) ... | - | South Korea | 1 | 555.00 | Lotte World Tower (555m) ... | - | Taiwan | 1 | 508.00 | TAIPEI 101 (508m) ... | - | United Arab Emirates | 8 | 443.75 | Burj Khalifa (828m), Marina 101 (425m), ... | - | United States | 8 | 426.63 | One World Trade Center (541m), Central ... | - | Vietnam | 1 | 461.00 | Vincom Landmark 81 (461m) ... | - - This table presents a clear summary of tallest building distributions across various countries, along with the - average height of skyscrapers present in each region. - Here's the information you requested regarding the 50 tallest buildings in the world, organized by country. The - table below includes the names and heights of these buildings, along with the average height for each country: - - | Country | Number of Buildings | Average Height (m) | Building Names & Heights | - |-----------------------|---------------------|---------------------|---------------------------------------------| - | China | 21 | 471.33 | Shanghai Tower (632m), Ping An Finance ... | - | Hong Kong | 3 | 421.00 | International Commerce Centre (484m), T ... | - | Kuwait | 1 | 413.00 | Al Hamra Tower (413m) ... | - | Malaysia | 4 | 507.00 | Merdeka 118 (679m), Petronas Tower 1 (4 ... | - | Russia | 1 | 462.00 | Lakhta Center (462m) ... | - | Saudi Arabia | 1 | 601.00 | Makkah Royal Clock Tower (601m) ... | - | South Korea | 1 | 555.00 | Lotte World Tower (555m) ... | - | Taiwan | 1 | 508.00 | TAIPEI 101 (508m) ... | - | United Arab Emirates | 8 | 443.75 | Burj Khalifa (828m), Marina 101 (425m), ... | - | United States | 8 | 426.63 | One World Trade Center (541m), Central ... | - | Vietnam | 1 | 461.00 | Vincom Landmark 81 (461m) ... | - - This comprehensive list should give you a clear view of the tallest skyscrapers, showcasing their significant - presence across the globe. If you have any more questions or need further details, feel free to ask! + Here's a detailed simulated report on the potential business impact due to tariffs and price adjustments, + along with strategic recommendations: + + ### Financial Impact Summary: + + 1. **New Cost Price after Tariffs:** $145.00 + 2. **New Selling Price after 10% Increase:** $137.50 + 3. **New Profit Margin:** -5.45% + 4. **Estimated Sales Volume Change:** Decrease to 95.0% of original + 5. **New Estimated Profit per Unit:** Negative $7.12 + + ### Strategies from Historical Cases: + ... """ From b508bb62f7ff01e7967d2021c821aa19bd584469 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 15 May 2025 17:31:09 -0700 Subject: [PATCH 4/9] Update sample readme --- .../samples/getting_started_with_agents/README.md | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/python/samples/getting_started_with_agents/README.md b/python/samples/getting_started_with_agents/README.md index 33c71a1adad0..fdc7e8aa5996 100644 --- a/python/samples/getting_started_with_agents/README.md +++ b/python/samples/getting_started_with_agents/README.md @@ -49,20 +49,6 @@ Example|Description _Note: For details on configuring an Azure AI Agent, please see [here](../getting_started_with_agents/azure_ai_agent/README.md)._ -## Multi Agent Orchestration - -Example|Description ----|--- -[step1_concurrent](../getting_started_with_agents/multi_agent_orchestration/step1_concurrent.py)|How to run multiple agents concurrently and manage their output. -[step1a_concurrent_structure_output](../getting_started_with_agents/multi_agent_orchestration/step1a_concurrent_structure_output.py)|How to run concurrent agents that return structured outputs. -[step2_sequential](../getting_started_with_agents/multi_agent_orchestration/step2_sequential.py)|How to run agents sequentially where each one depends on the previous. -[step2a_sequential_cancellation_token](../getting_started_with_agents/multi_agent_orchestration/step2a_sequential_cancellation_token.py)|How to use cancellation tokens in a sequential agent flow. -[step3_group_chat](../getting_started_with_agents/multi_agent_orchestration/step3_group_chat.py)|How to create a group chat with multiple agents interacting together. -[step3a_group_chat_human_in_the_loop](../getting_started_with_agents/multi_agent_orchestration/step3a_group_chat_human_in_the_loop.py)|How to include a human participant in a group chat with agents. -[step3b_group_chat_with_chat_completion_manager](../getting_started_with_agents/multi_agent_orchestration/step3b_group_chat_with_chat_completion_manager.py)|How to manage a group chat with agents using a chat completion manager. -[step4_handoff](../getting_started_with_agents/multi_agent_orchestration/step4_handoff.py)|How to hand off conversation or tasks from one agent to another. -[step4a_handoff_structured_inputs](../getting_started_with_agents/multi_agent_orchestration/step4a_handoff_structured_inputs.py)|How to perform structured inputs handoffs between agents. - ## OpenAI Assistant Agent @@ -87,6 +73,7 @@ Example|Description [step7_responses_agent_structured_outputs](../getting_started_with_agents/openai_responses/step7_responses_agent_structured_outputs.py)|How to use have an OpenAI Responses agent use structured outputs. ## Multi-Agent Orchestration + Example|Description ---|--- [step1_concurrent](../getting_started_with_agents/multi_agent_orchestration/step1_concurrent.py)|How to run agents in parallel on the same task. From f0cdd3bfaa67459586105d004605e02844ed3935 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 15 May 2025 20:08:56 -0700 Subject: [PATCH 5/9] Tune magentic sample --- .../step5_magentic.py | 105 ++++++++++++------ 1 file changed, 70 insertions(+), 35 deletions(-) diff --git a/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py b/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py index 2e5c677cdd6f..1d04a17f9b45 100644 --- a/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py +++ b/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py @@ -83,14 +83,12 @@ async def main(): # 3. Invoke the orchestration with a task and the runtime orchestration_result = await magentic_orchestration.invoke( task=( - "The 2025 trade war between the US and other countries has had a significant impact " - "on the global economy. I am a business owner in the US that import household goods " - "such as bed sheets and holiday decorations from south-east Asia. I want " - "to know the impact of the tariffs on my business given that my current profit " - "margin is 20%. And If I were to increase the price of my products by 10%, " - "how would that affect my customer behavior and profit margin? Base on the analysis, " - "find similar cases in the past to cross-reference the results. Provide a detailed " - "report and recommendations on how to adapt to the changing market conditions at the end." + "I am preparing a report on the energy efficiency of different machine learning model architectures. " + "Compare the estimated training and inference energy consumption of ResNet-50, BERT-base, and GPT-2 " + "on standard datasets (e.g., ImageNet for ResNet, GLUE for BERT, WebText for GPT-2). " + "Then, estimate the CO2 emissions associated with each, assuming training on an Azure Standard_NC6s_v3 VM " + "for 24 hours. Provide tables for clarity, and recommend the most energy-efficient model " + "per task type (image classification, text classification, and text generation)." ), runtime=runtime, ) @@ -106,47 +104,84 @@ async def main(): """ Sample output: **ResearchAgent** - The 2025 trade war has led to significant tariffs imposed by the United States on imports from Southeast Asian - countries, directly affecting industries such as household goods. For instance, Cambodia faces a 49% tariff, - Vietnam 46%, and Thailand 36% on their exports to the U.S. - ([thailandinfo.se](https://www.thailandinfo.se/en/usa-tariffs-southeast-asia-2025/?utm_source=openai)) + Estimating the energy consumption and associated CO₂ emissions for training and inference of ResNet-50, BERT-base... - ... **CoderAgent** - Here's the analysis based on your scenario: + Here is the comparison of energy consumption and CO₂ emissions for each model (ResNet-50, BERT-base, and GPT-2) + over a 24-hour period: + + | Model | Training Energy (kWh) | Inference Energy (kWh) | Total Energy (kWh) | CO₂ Emissions (kg) | + |-----------|------------------------|------------------------|---------------------|---------------------| + | ResNet-50 | 21.11 | 0.08232 | 21.19232 | 19.50 | + | BERT-base | 0.048 | 0.23736 | 0.28536 | 0.26 | + | GPT-2 | 42.22 | 0.35604 | 42.57604 | 39.17 | - 1. **Initial Scenario:** - - Initial Selling Price: $125.00 (to achieve a 20% profit margin) + ### Recommendations: + ... - 2. **After Applying Tariffs:** - - New Cost Price: $145.00 (after a 45% tariff on the initial $100 cost) + **CoderAgent** + Here are the recalibrated results for energy consumption and CO₂ emissions, assuming a more conservative approach + for models like GPT-2: - 3. **With a 10% Price Increase:** - - New Selling Price: $137.50 + | Model | Training Energy (kWh) | Inference Energy (kWh) | Total Energy (kWh) | CO₂ Emissions (kg) | + |------------------|------------------------|------------------------|---------------------|---------------------| + | ResNet-50 | 21.11 | 0.08232 | 21.19232 | 19.50 | + | BERT-base | 0.048 | 0.23736 | 0.28536 | 0.26 | + | GPT-2 (Adjusted) | 42.22 | 0.35604 | 42.57604 | 39.17 | - 4. **Profit Margin and Volume Impact:** ... + + **ResearchAgent** + Estimating the energy consumption and associated CO₂ emissions for training and inference of machine learning ... + + **ResearchAgent** + Estimating the energy consumption and CO₂ emissions of training and inference for ResNet-50, BERT-base, and ... + + **CoderAgent** + Here is the estimated energy use and CO₂ emissions for a full day of operation for each model on an Azure ... + **ResearchAgent** - In response to increased tariffs during trade wars, various companies have implemented strategic measures to - mitigate financial impacts and maintain competitiveness. Notable examples include: + Recent analyses have highlighted the substantial energy consumption and carbon emissions associated with ... - **1. Supply Chain Diversification:** + **CoderAgent** + Here's the refined estimation for the energy use and CO₂ emissions for optimized models on an Azure ... - - **Steven Madden Ltd.:** Faced with a 10% tariff on handbags imported from China, the company relocated - production to Cambodia to circumvent the tariffs.([money.usnews.com](https://money.usnews.com/money/blogs/... **CoderAgent** - Here's a detailed simulated report on the potential business impact due to tariffs and price adjustments, - along with strategic recommendations: + To provide precise estimates for CO₂ emissions based on Azure's regional data centers' carbon intensity, we need ... + + **ResearchAgent** + To refine the CO₂ emission estimates for training and inference of ResNet-50, BERT-base, and GPT-2 on an Azure ... + + **CoderAgent** + Here's the refined comparative table for energy consumption and CO₂ emissions for ResNet-50, BERT-base, and GPT-2, + taking into account carbon intensity data for Azure's West Europe and Sweden Central regions: + + | Model | Energy (kWh) | CO₂ Emissions West Europe (kg) | CO₂ Emissions Sweden Central (kg) | + |------------|--------------|--------------------------------|-----------------------------------| + | ResNet-50 | 5.76 | 0.639 | 0.086 | + | BERT-base | 9.18 | 1.019 | 0.138 | + | GPT-2 | 12.96 | 1.439 | 0.194 | + + **Refined Recommendations:** + + ... + + Final result: + Here is the comprehensive report on energy efficiency and CO₂ emissions for ResNet-50, BERT-base, and GPT-2 models + when trained and inferred on an Azure Standard_NC6s_v3 VM for 24 hours. + + ### Energy Consumption and CO₂ Emissions: + + Based on refined analyses, here are the estimated energy consumption and CO₂ emissions for each model: - ### Financial Impact Summary: + | Model | Energy (kWh) | CO₂ Emissions West Europe (kg) | CO₂ Emissions Sweden Central (kg) | + |------------|--------------|--------------------------------|-----------------------------------| + | ResNet-50 | 5.76 | 0.639 | 0.086 | + | BERT-base | 9.18 | 1.019 | 0.138 | + | GPT-2 | 12.96 | 1.439 | 0.194 | - 1. **New Cost Price after Tariffs:** $145.00 - 2. **New Selling Price after 10% Increase:** $137.50 - 3. **New Profit Margin:** -5.45% - 4. **Estimated Sales Volume Change:** Decrease to 95.0% of original - 5. **New Estimated Profit per Unit:** Negative $7.12 + ### Recommendations for Energy Efficiency: - ### Strategies from Historical Cases: ... """ From 954e46cfb717557f7a958574fa07b4a7f022f284 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 15 May 2025 21:14:56 -0700 Subject: [PATCH 6/9] Address comments --- .../step5_magentic.py | 18 +++++-- .../agents/orchestration/magentic.py | 31 ++++++++++- .../agents/orchestration/test_magentic.py | 52 +++++++++++++------ 3 files changed, 79 insertions(+), 22 deletions(-) diff --git a/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py b/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py index 1d04a17f9b45..20e893782c91 100644 --- a/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py +++ b/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py @@ -2,10 +2,18 @@ import asyncio -from semantic_kernel.agents import Agent, ChatCompletionAgent, MagenticOrchestration, OpenAIAssistantAgent -from semantic_kernel.agents.orchestration.magentic import StandardMagenticManager +from semantic_kernel.agents import ( + Agent, + ChatCompletionAgent, + MagenticOrchestration, + OpenAIAssistantAgent, + StandardMagenticManager, +) from semantic_kernel.agents.runtime import InProcessRuntime -from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion, OpenAIPromptExecutionSettings +from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion +from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import ( + OpenAIChatPromptExecutionSettings, +) from semantic_kernel.contents import ChatMessageContent """ @@ -66,12 +74,12 @@ def agent_response_callback(message: ChatMessageContent) -> None: async def main(): """Main function to run the agents.""" # 1. Create a Magentic orchestration with two agents and a Magentic manager - # Note, the Magentic manager accepts custom prompts for advanced users and scenarios. + # Note, the Standard Magentic manager accepts custom prompts for advanced users and scenarios. magentic_orchestration = MagenticOrchestration( members=await agents(), manager=StandardMagenticManager( chat_completion_service=OpenAIChatCompletion(), - prompt_execution_settings=OpenAIPromptExecutionSettings(), + prompt_execution_settings=OpenAIChatPromptExecutionSettings(), ), agent_response_callback=agent_response_callback, ) diff --git a/python/semantic_kernel/agents/orchestration/magentic.py b/python/semantic_kernel/agents/orchestration/magentic.py index edd74f23cffa..bb649761ea1d 100644 --- a/python/semantic_kernel/agents/orchestration/magentic.py +++ b/python/semantic_kernel/agents/orchestration/magentic.py @@ -7,7 +7,7 @@ from collections.abc import Awaitable, Callable from typing import Annotated -from pydantic import Field +from pydantic import Field, field_validator from semantic_kernel.agents.agent import Agent from semantic_kernel.agents.orchestration.agent_actor_base import ActorBase, AgentActorBase @@ -37,6 +37,7 @@ from semantic_kernel.kernel_pydantic import KernelBaseModel from semantic_kernel.prompt_template.kernel_prompt_template import KernelPromptTemplate from semantic_kernel.prompt_template.prompt_template_config import PromptTemplateConfig +from semantic_kernel.utils.feature_stage_decorator import experimental if sys.version_info >= (3, 12): from typing import override # pragma: no cover @@ -49,30 +50,35 @@ # region Messages and Types +@experimental class MagenticStartMessage(KernelBaseModel): """A message to start a magentic group chat.""" body: ChatMessageContent +@experimental class MagenticRequestMessage(KernelBaseModel): """A request message type for agents in a magentic group chat.""" agent_name: str +@experimental class MagenticResponseMessage(KernelBaseModel): """A response message type from agents in a magentic group chat.""" body: ChatMessageContent +@experimental class MagenticResetMessage(KernelBaseModel): """A message to reset a participant's chat history in a magentic group chat.""" pass +@experimental class ProgressLedgerItem(KernelBaseModel): """A progress ledger item.""" @@ -80,6 +86,7 @@ class ProgressLedgerItem(KernelBaseModel): answer: str | bool +@experimental class ProgressLedger(KernelBaseModel): """A progress ledger.""" @@ -90,6 +97,7 @@ class ProgressLedger(KernelBaseModel): instruction_or_question: ProgressLedgerItem +@experimental class MagenticContext(KernelBaseModel): """Context for the Magentic manager.""" @@ -120,6 +128,7 @@ def reset(self) -> None: # region MagenticManager +@experimental class MagenticManager(KernelBaseModel, ABC): """Base class for the Magentic One manager.""" @@ -182,6 +191,7 @@ async def prepare_final_answer(self, magentic_context: MagenticContext) -> ChatM ... +@experimental class StandardMagenticManager(MagenticManager): """Standard Magentic manager implementation. @@ -210,6 +220,21 @@ class TaskLedger(KernelBaseModel): task_ledger: TaskLedger | None = None + @field_validator("prompt_execution_settings", mode="after") + def _validate_prompt_execution_settings(cls, v: PromptExecutionSettings) -> PromptExecutionSettings: + """Validate the prompt execution settings to ensure the service supports structured output. + + Args: + v (PromptExecutionSettings): The prompt execution settings. + + Returns: + PromptExecutionSettings: The validated prompt execution settings. + """ + if not hasattr(v, "response_format"): + raise ValueError("The service must support structured output.") + + return v + @override async def plan(self, magentic_context: MagenticContext) -> ChatMessageContent: """Plan the task. @@ -372,7 +397,6 @@ async def create_progress_ledger(self, magentic_context: MagenticContext) -> Pro self.prompt_execution_settings ) prompt_execution_settings_clone.update_from_prompt_execution_settings( - # TODO(@taochen): Double check how to make sure the service support json output. PromptExecutionSettings(extension_data={"response_format": ProgressLedger}) ) @@ -418,6 +442,7 @@ async def prepare_final_answer(self, magentic_context: MagenticContext) -> ChatM # region MagenticManagerActor +@experimental class MagenticManagerActor(ActorBase): """Actor for the Magentic One manager.""" @@ -619,6 +644,7 @@ async def _check_within_limits(self) -> bool: # region MagenticAgentActor +@experimental class MagenticAgentActor(AgentActorBase): """An agent actor that process messages in a Magentic One group chat.""" @@ -692,6 +718,7 @@ async def _handle_reset_message(self, message: MagenticResetMessage, ctx: Messag # region MagenticOrchestration +@experimental class MagenticOrchestration(OrchestrationBase[TIn, TOut]): """The Magentic One pattern orchestration.""" diff --git a/python/tests/unit/agents/orchestration/test_magentic.py b/python/tests/unit/agents/orchestration/test_magentic.py index 935469cbe216..eb5a4c032886 100644 --- a/python/tests/unit/agents/orchestration/test_magentic.py +++ b/python/tests/unit/agents/orchestration/test_magentic.py @@ -1,9 +1,11 @@ # Copyright (c) Microsoft. All rights reserved. import sys +from typing import Any, Literal from unittest.mock import AsyncMock, patch import pytest +from pydantic import BaseModel from semantic_kernel.agents.orchestration.magentic import ( MagenticContext, @@ -37,6 +39,14 @@ class MockChatCompletionService(ChatCompletionClientBase): pass +class MockPromptExecutionSettings(PromptExecutionSettings): + """A mock prompt execution settings class for testing purposes.""" + + response_format: ( + dict[Literal["type"], Literal["text", "json_object"]] | dict[str, Any] | type[BaseModel] | type | None + ) = None + + # region MagenticOrchestration @@ -50,7 +60,7 @@ async def test_init_member_without_description_throws(): members=[agent_a, agent_b], manager=StandardMagenticManager( chat_completion_service=MockChatCompletionService(ai_model_id="test"), - prompt_execution_settings=PromptExecutionSettings(), + prompt_execution_settings=MockPromptExecutionSettings(), ), ) @@ -73,7 +83,7 @@ async def test_prepare(): members=[agent_a, agent_b], manager=StandardMagenticManager( chat_completion_service=MockChatCompletionService(ai_model_id="test"), - prompt_execution_settings=PromptExecutionSettings(), + prompt_execution_settings=MockPromptExecutionSettings(), ), ) await orchestration.invoke(task="test_message", runtime=runtime) @@ -173,7 +183,7 @@ async def test_invoke(): ): mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response") chat_completion_service = MockChatCompletionService(ai_model_id="test") - prompt_execution_settings = PromptExecutionSettings() + prompt_execution_settings = MockPromptExecutionSettings() manager = StandardMagenticManager( chat_completion_service=chat_completion_service, @@ -205,7 +215,7 @@ async def test_invoke(): async def test_invoke_with_list_error(): """Test the invoke method of the MagenticOrchestration with a list of messages which raises an error.""" chat_completion_service = MockChatCompletionService(ai_model_id="test") - prompt_execution_settings = PromptExecutionSettings() + prompt_execution_settings = MockPromptExecutionSettings() manager = StandardMagenticManager( chat_completion_service=chat_completion_service, @@ -264,7 +274,7 @@ async def test_invoke_with_response_callback(): members=[agent_a, agent_b], manager=StandardMagenticManager( chat_completion_service=MockChatCompletionService(ai_model_id="test"), - prompt_execution_settings=PromptExecutionSettings(), + prompt_execution_settings=MockPromptExecutionSettings(), ), agent_response_callback=lambda x: responses.append(x), ) @@ -309,7 +319,7 @@ async def test_invoke_with_max_stall_count_exceeded(): members=[agent_a, agent_b], manager=StandardMagenticManager( chat_completion_service=MockChatCompletionService(ai_model_id="test"), - prompt_execution_settings=PromptExecutionSettings(), + prompt_execution_settings=MockPromptExecutionSettings(), max_stall_count=1, ), ) @@ -355,7 +365,7 @@ async def test_invoke_with_max_round_count_exceeded(): members=[agent_a, agent_b], manager=StandardMagenticManager( chat_completion_service=MockChatCompletionService(ai_model_id="test"), - prompt_execution_settings=PromptExecutionSettings(), + prompt_execution_settings=MockPromptExecutionSettings(), max_round_count=1, ), ) @@ -401,7 +411,7 @@ async def test_invoke_with_max_reset_count_exceeded(): members=[agent_a, agent_b], manager=StandardMagenticManager( chat_completion_service=MockChatCompletionService(ai_model_id="test"), - prompt_execution_settings=PromptExecutionSettings(), + prompt_execution_settings=MockPromptExecutionSettings(), max_stall_count=0, # No stall allowed max_reset_count=0, # No reset allowed ), @@ -449,7 +459,7 @@ async def test_invoke_with_unknown_speaker(): members=[agent_a, agent_b], manager=StandardMagenticManager( chat_completion_service=MockChatCompletionService(ai_model_id="test"), - prompt_execution_settings=PromptExecutionSettings(), + prompt_execution_settings=MockPromptExecutionSettings(), ), ) orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime) @@ -466,7 +476,7 @@ async def test_invoke_with_unknown_speaker(): def test_standard_magentic_manager_init(): """Test the initialization of the StandardMagenticManager.""" chat_completion_service = MockChatCompletionService(ai_model_id="test") - prompt_execution_settings = PromptExecutionSettings() + prompt_execution_settings = MockPromptExecutionSettings() manager = StandardMagenticManager( chat_completion_service=chat_completion_service, @@ -506,7 +516,7 @@ def test_standard_magentic_manager_init(): def test_standard_magentic_manager_init_with_custom_prompts(): """Test the initialization of the StandardMagenticManager with custom prompts.""" chat_completion_service = MockChatCompletionService(ai_model_id="test") - prompt_execution_settings = PromptExecutionSettings() + prompt_execution_settings = MockPromptExecutionSettings() manager = StandardMagenticManager( chat_completion_service=chat_completion_service, @@ -529,6 +539,18 @@ def test_standard_magentic_manager_init_with_custom_prompts(): assert manager.final_answer_prompt == "custom_final_answer_prompt" +def test_standard_magentic_manager_init_with_invalid_prompt_execution_settings(): + """Test the initialization of the StandardMagenticManager with invalid prompt execution settings.""" + chat_completion_service = MockChatCompletionService(ai_model_id="test") + prompt_execution_settings = PromptExecutionSettings() + + with pytest.raises(ValueError): + StandardMagenticManager( + chat_completion_service=chat_completion_service, + prompt_execution_settings=prompt_execution_settings, + ) + + async def test_standard_magentic_manager_plan(): """Test the plan method of the StandardMagenticManager.""" @@ -537,7 +559,7 @@ async def test_standard_magentic_manager_plan(): ) as mock_get_chat_message_content: mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response") chat_completion_service = MockChatCompletionService(ai_model_id="test") - prompt_execution_settings = PromptExecutionSettings() + prompt_execution_settings = MockPromptExecutionSettings() manager = StandardMagenticManager( chat_completion_service=chat_completion_service, @@ -579,7 +601,7 @@ async def test_standard_magentic_manager_replan(): mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response") chat_completion_service = MockChatCompletionService(ai_model_id="test") - prompt_execution_settings = PromptExecutionSettings() + prompt_execution_settings = MockPromptExecutionSettings() manager = StandardMagenticManager( chat_completion_service=chat_completion_service, @@ -618,7 +640,7 @@ async def test_standard_magentic_manager_replan_without_plan(): """Test the replan method of the StandardMagenticManager.""" chat_completion_service = MockChatCompletionService(ai_model_id="test") - prompt_execution_settings = PromptExecutionSettings() + prompt_execution_settings = MockPromptExecutionSettings() manager = StandardMagenticManager( chat_completion_service=chat_completion_service, @@ -654,7 +676,7 @@ async def test_standard_magentic_manager_create_progress_ledger(): ) chat_completion_service = MockChatCompletionService(ai_model_id="test") - prompt_execution_settings = PromptExecutionSettings() + prompt_execution_settings = MockPromptExecutionSettings() manager = StandardMagenticManager( chat_completion_service=chat_completion_service, From b5d52b6190b90f2d1cf7fb527ea423ee2a694cef Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Fri, 16 May 2025 08:46:04 -0700 Subject: [PATCH 7/9] Address comments 2 --- .../multi_agent_orchestration/README.md | 15 ---- .../step5_magentic.py | 17 ++-- .../agents/orchestration/magentic.py | 79 ++++++++++++------- .../agents/orchestration/test_magentic.py | 10 +++ 4 files changed, 71 insertions(+), 50 deletions(-) diff --git a/python/samples/getting_started_with_agents/multi_agent_orchestration/README.md b/python/samples/getting_started_with_agents/multi_agent_orchestration/README.md index 37c8fd05526f..2ce05547b4b0 100644 --- a/python/samples/getting_started_with_agents/multi_agent_orchestration/README.md +++ b/python/samples/getting_started_with_agents/multi_agent_orchestration/README.md @@ -31,18 +31,3 @@ Refer to [here](../../concepts/setup/README.md) on how to set up the environment | **Handoff** | Useful for tasks that are dynamic in nature and don't have a well-defined step-by-step approach. | | **GroupChat** | Useful for tasks that will benefit from inputs from multiple agents and a highly configurable conversation flow. | | **Magentic** | GroupChat like with a planner based manager. Inspired by [Magentic One](https://www.microsoft.com/en-us/research/articles/magentic-one-a-generalist-multi-agent-system-for-solving-complex-tasks/). | - -## Samples - -| Sample | Description | -|-----------------------------------------------------------------------------|--------------| -| [step1_concurrent](step1_concurrent.py) | Run agents in parallel on the same task. | -| [step1a_concurrent_structure_output](step1a_concurrent_structure_output.py) | Run agents in parallel on the same task and return structured output. | -| [step2_sequential](step2_sequential.py) | Run agents in sequence to complete a task. | -| [step2a_sequential_cancellation_token](step2a_sequential_cancellation_token.py) | Cancel an invocation while it is in progress. | -| [step3_group_chat](step3_group_chat.py) | Run agents in a group chat to complete a task. | -| [step3a_group_chat_human_in_the_loop](step3a_group_chat_human_in_the_loop.py) | Run agents in a group chat with human in the loop. | -| [step3b_group_chat_with_chat_completion_manager](step3b_group_chat_with_chat_completion_manager.py) | Run agents in a group chat with a more dynamic manager. | -| [step4_handoff](step4_handoff.py) | Run agents in a handoff orchestration to complete a task. | -| [step4a_handoff_structure_input](step4a_handoff_structure_input.py) | Run agents in a handoff orchestration to complete a task with structured input. | -| [step5_magentic](step5_magentic.py) | Run agents in a Magentic orchestration to complete a task. | diff --git a/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py b/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py index 20e893782c91..98e43c27d422 100644 --- a/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py +++ b/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py @@ -11,9 +11,6 @@ ) from semantic_kernel.agents.runtime import InProcessRuntime from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion -from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import ( - OpenAIChatPromptExecutionSettings, -) from semantic_kernel.contents import ChatMessageContent """ @@ -44,6 +41,8 @@ async def agents() -> list[Agent]: "You are a Researcher. You find information without additional computation or quantitative analysis." ), # This agent requires the gpt-4o-search-preview model to perform web searches. + # Feel free to explore with other agents that support web search, for example, + # the `OpenAIResponseAgent` or `AzureAIAgent` with bing grounding. service=OpenAIChatCompletion(ai_model_id="gpt-4o-search-preview"), ) @@ -74,13 +73,15 @@ def agent_response_callback(message: ChatMessageContent) -> None: async def main(): """Main function to run the agents.""" # 1. Create a Magentic orchestration with two agents and a Magentic manager - # Note, the Standard Magentic manager accepts custom prompts for advanced users and scenarios. + # Note, the Standard Magentic manager uses prompts that have been tuned very + # carefully but it accepts custom prompts for advanced users and scenarios. + # For even more advanced scenarios, you can subclass the MagenticManagerBase + # and implement your own manager logic. + # The standard manager also requires a chat completion model that supports + # structured output. magentic_orchestration = MagenticOrchestration( members=await agents(), - manager=StandardMagenticManager( - chat_completion_service=OpenAIChatCompletion(), - prompt_execution_settings=OpenAIChatPromptExecutionSettings(), - ), + manager=StandardMagenticManager(chat_completion_service=OpenAIChatCompletion()), agent_response_callback=agent_response_callback, ) diff --git a/python/semantic_kernel/agents/orchestration/magentic.py b/python/semantic_kernel/agents/orchestration/magentic.py index bb649761ea1d..1608e5a0827e 100644 --- a/python/semantic_kernel/agents/orchestration/magentic.py +++ b/python/semantic_kernel/agents/orchestration/magentic.py @@ -7,7 +7,7 @@ from collections.abc import Awaitable, Callable from typing import Annotated -from pydantic import Field, field_validator +from pydantic import Field from semantic_kernel.agents.agent import Agent from semantic_kernel.agents.orchestration.agent_actor_base import ActorBase, AgentActorBase @@ -129,7 +129,7 @@ def reset(self) -> None: @experimental -class MagenticManager(KernelBaseModel, ABC): +class MagenticManagerBase(KernelBaseModel, ABC): """Base class for the Magentic One manager.""" max_stall_count: Annotated[int, Field(description="The maximum number of stalls allowed before a reset.", ge=0)] = 3 @@ -192,13 +192,21 @@ async def prepare_final_answer(self, magentic_context: MagenticContext) -> ChatM @experimental -class StandardMagenticManager(MagenticManager): +class _TaskLedger(KernelBaseModel): + """Task ledger for the Standard Magentic manager.""" + + facts: Annotated[ChatMessageContent, Field(description="The facts about the task.")] + plan: Annotated[ChatMessageContent, Field(description="The plan for the task.")] + + +@experimental +class StandardMagenticManager(MagenticManagerBase): """Standard Magentic manager implementation. This is the default implementation of the Magentic manager. It uses the task ledger to keep track of the facts and plan for the task. - This implementation is requires structured outputs. + This implementation requires a chat completion model with structured outputs. """ chat_completion_service: ChatCompletionClientBase @@ -212,28 +220,45 @@ class StandardMagenticManager(MagenticManager): progress_ledger_prompt: str = ORCHESTRATOR_PROGRESS_LEDGER_PROMPT final_answer_prompt: str = ORCHESTRATOR_FINAL_ANSWER_PROMPT - class TaskLedger(KernelBaseModel): - """Task ledger for the Standard Magentic manager.""" + task_ledger: _TaskLedger | None = None - facts: Annotated[ChatMessageContent, Field(description="The facts about the task.")] - plan: Annotated[ChatMessageContent, Field(description="The plan for the task.")] - - task_ledger: TaskLedger | None = None - - @field_validator("prompt_execution_settings", mode="after") - def _validate_prompt_execution_settings(cls, v: PromptExecutionSettings) -> PromptExecutionSettings: - """Validate the prompt execution settings to ensure the service supports structured output. + def __init__( + self, + chat_completion_service: ChatCompletionClientBase, + prompt_execution_settings: PromptExecutionSettings | None = None, + **kwargs, + ) -> None: + """Initialize the Standard Magentic manager. Args: - v (PromptExecutionSettings): The prompt execution settings. - - Returns: - PromptExecutionSettings: The validated prompt execution settings. + chat_completion_service (ChatCompletionClientBase): The chat completion service to use. + prompt_execution_settings (PromptExecutionSettings | None): The prompt execution settings to use. + **kwargs: Additional keyword arguments for prompts: + - task_ledger_facts_prompt: The prompt to use for the task ledger facts. + - task_ledger_plan_prompt: The prompt to use for the task ledger plan. + - task_ledger_full_prompt: The prompt to use for the full task ledger. + - task_ledger_facts_update_prompt: The prompt to use for the task ledger facts update. + - task_ledger_plan_update_prompt: The prompt to use for the task ledger plan update. + - progress_ledger_prompt: The prompt to use for the progress ledger. + - final_answer_prompt: The prompt to use for the final answer. """ - if not hasattr(v, "response_format"): - raise ValueError("The service must support structured output.") + # Bast effort to make sure the service supports structured output. Even if the service supports + # structured output, the model may not support it, in which case there is no good way to check. + if prompt_execution_settings is None: + prompt_execution_settings = chat_completion_service.instantiate_prompt_execution_settings() + if not hasattr(prompt_execution_settings, "response_format"): + raise ValueError("The service must support structured output.") + else: + if not hasattr(prompt_execution_settings, "response_format"): + raise ValueError("The service must support structured output.") + if prompt_execution_settings.response_format is not None: + raise ValueError("The prompt execution settings must not have a response format set.") - return v + super().__init__( + chat_completion_service=chat_completion_service, + prompt_execution_settings=prompt_execution_settings, + **kwargs, + ) @override async def plan(self, magentic_context: MagenticContext) -> ChatMessageContent: @@ -281,7 +306,7 @@ async def plan(self, magentic_context: MagenticContext) -> ChatMessageContent: ) assert plan is not None # nosec B101 - self.task_ledger = self.TaskLedger(facts=facts, plan=plan) + self.task_ledger = _TaskLedger(facts=facts, plan=plan) return await self._render_task_ledger(magentic_context) @override @@ -448,7 +473,7 @@ class MagenticManagerActor(ActorBase): def __init__( self, - manager: MagenticManager, + manager: MagenticManagerBase, internal_topic_type: str, participant_descriptions: dict[str, str], result_callback: Callable[[DefaultTypeAlias], Awaitable[None]] | None = None, @@ -456,7 +481,7 @@ def __init__( """Initialize the Magentic One manager actor. Args: - manager (MagenticManager): The Magentic One manager. + manager (MagenticManagerBase): The Magentic One manager. internal_topic_type (str): The internal topic type. participant_descriptions (dict[str, str]): The participant descriptions. result_callback (Callable | None): A callback function to handle the final answer. @@ -725,7 +750,7 @@ class MagenticOrchestration(OrchestrationBase[TIn, TOut]): def __init__( self, members: list[Agent], - manager: MagenticManager, + manager: MagenticManagerBase, name: str | None = None, description: str | None = None, input_transform: Callable[[TIn], Awaitable[DefaultTypeAlias] | DefaultTypeAlias] | None = None, @@ -735,8 +760,8 @@ def __init__( """Initialize the Magentic One orchestration. Args: - members (list[Agent | OrchestrationBase]): A list of agents or orchestration bases. - manager (MagenticManager): The manager for the Magentic One pattern. + members (list[Agent]): A list of agents. + manager (MagenticManagerBase): The manager for the Magentic One pattern. name (str | None): The name of the orchestration. description (str | None): The description of the orchestration. input_transform (Callable | None): A function that transforms the external input message. diff --git a/python/tests/unit/agents/orchestration/test_magentic.py b/python/tests/unit/agents/orchestration/test_magentic.py index eb5a4c032886..bb416b327622 100644 --- a/python/tests/unit/agents/orchestration/test_magentic.py +++ b/python/tests/unit/agents/orchestration/test_magentic.py @@ -551,6 +551,16 @@ def test_standard_magentic_manager_init_with_invalid_prompt_execution_settings() ) +def test_standard_magentic_manager_init_without_prompt_execution_settings(): + """Test the initialization of the StandardMagenticManager without prompt execution settings.""" + # The default prompt execution settings of the mock chat completion service + # does not support structured output. + chat_completion_service = MockChatCompletionService(ai_model_id="test") + + with pytest.raises(ValueError): + StandardMagenticManager(chat_completion_service=chat_completion_service) + + async def test_standard_magentic_manager_plan(): """Test the plan method of the StandardMagenticManager.""" From 122f55a594945cf0db4da73c19cff6882e3b79fa Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Fri, 16 May 2025 08:54:46 -0700 Subject: [PATCH 8/9] Fix mypy --- python/semantic_kernel/agents/__init__.py | 2 +- python/semantic_kernel/agents/__init__.pyi | 4 ++-- python/semantic_kernel/agents/orchestration/magentic.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/semantic_kernel/agents/__init__.py b/python/semantic_kernel/agents/__init__.py index 8f5f326a6a00..b00316dec715 100644 --- a/python/semantic_kernel/agents/__init__.py +++ b/python/semantic_kernel/agents/__init__.py @@ -47,7 +47,7 @@ "GroupChatManager": ".orchestration.group_chat", "MagenticOrchestration": ".orchestration.magentic", "ProgressLedger": ".orchestration.magentic", - "MagenticManager": ".orchestration.magentic", + "MagenticManagerBase": ".orchestration.magentic", "StandardMagenticManager": ".orchestration.magentic", } diff --git a/python/semantic_kernel/agents/__init__.pyi b/python/semantic_kernel/agents/__init__.pyi index 5de2b5f9a17d..60ed72badb98 100644 --- a/python/semantic_kernel/agents/__init__.pyi +++ b/python/semantic_kernel/agents/__init__.pyi @@ -36,7 +36,7 @@ from .orchestration.group_chat import ( StringResult, ) from .orchestration.handoffs import HandoffOrchestration, OrchestrationHandoffs -from .orchestration.magentic import MagenticManager, MagenticOrchestration, ProgressLedger, StandardMagenticManager +from .orchestration.magentic import MagenticManagerBase, MagenticOrchestration, ProgressLedger, StandardMagenticManager from .orchestration.sequential import SequentialOrchestration __all__ = [ @@ -69,7 +69,7 @@ __all__ = [ "GroupChatManager", "GroupChatOrchestration", "HandoffOrchestration", - "MagenticManager", + "MagenticManagerBase", "MagenticOrchestration", "MessageResult", "ModelConnection", diff --git a/python/semantic_kernel/agents/orchestration/magentic.py b/python/semantic_kernel/agents/orchestration/magentic.py index 1608e5a0827e..9970ae3506d7 100644 --- a/python/semantic_kernel/agents/orchestration/magentic.py +++ b/python/semantic_kernel/agents/orchestration/magentic.py @@ -251,7 +251,7 @@ def __init__( else: if not hasattr(prompt_execution_settings, "response_format"): raise ValueError("The service must support structured output.") - if prompt_execution_settings.response_format is not None: + if getattr(prompt_execution_settings, "response_format", None) is not None: raise ValueError("The prompt execution settings must not have a response format set.") super().__init__( From 577101e316e73fb95bc22328445e6d12e1c3e5e9 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Fri, 16 May 2025 13:43:12 -0700 Subject: [PATCH 9/9] Update agent description --- .../multi_agent_orchestration/step5_magentic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py b/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py index 98e43c27d422..cfe6bbcaeb21 100644 --- a/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py +++ b/python/samples/getting_started_with_agents/multi_agent_orchestration/step5_magentic.py @@ -52,7 +52,7 @@ async def agents() -> list[Agent]: definition = await client.beta.assistants.create( model=model, name="CoderAgent", - description="A helpful assistant with code interpreter capability.", + description="A helpful assistant that writes and executes code to process and analyze data.", instructions="You solve questions using code. Please provide detailed analysis and computation process.", tools=code_interpreter_tool, tool_resources=code_interpreter_tool_resources,