feat(agent): Add summary assistant checking (#1053)

Co-authored-by: yhjun1026 <[email protected]> Co-authored-by: csunny <[email protected]> Co-authored-by: Fangyin Cheng <[email protected]>
eosphoros-ai · Jan 10, 2024 · 2706e27 · 2706e27
1 parent fa8b5b1
commit 2706e27
Show file tree

Hide file tree

Showing 2 changed files with 71 additions and 7 deletions.
diff --git a/dbgpt/agent/agents/expand/summary_assistant_agent.py b/dbgpt/agent/agents/expand/summary_assistant_agent.py
@@ -20,9 +20,12 @@ class SummaryAssistantAgent(ConversableAgent):
     DEFAULT_SYSTEM_MESSAGE = """You are a great summary writter to summarize the provided text content according to user questions.
            Please complete this task step by step following instructions below:
            1. You need to first detect user's question that you need to answer with your summarization.
-           2. Output the extracted user's question with the format - The User's Question: user's question.
-           3. Then you need to summarize the provided messages.
+           2. Extract the provided text content used for summarization.
+           3. Then you need to summarize the extracted text content.
            4. Output the content of summarization ONLY related to user's question. The output language must be the same to user's question language.
+
+           ####Important Notice####
+           If you think the provided text content is not related to user questions at all, ONLY output "NO RELATIONSHIP.TERMINATE."!!.
         """
 
     DEFAULT_DESCRIBE = """Summarize provided text content according to user's questions and output the summaraization."""
@@ -34,7 +37,10 @@ def __init__(
         memory: GptsMemory,
         agent_context: AgentContext,
         describe: Optional[str] = DEFAULT_DESCRIBE,
-        is_termination_msg: Optional[Callable[[Dict], bool]] = None,
+        is_termination_msg: Optional[Callable[[Dict], bool]] = lambda x: isinstance(
+            x, dict
+        )
+        and "TERMINATE" in str(x).upper(),
         max_consecutive_auto_reply: Optional[int] = None,
         human_input_mode: Optional[str] = "NEVER",
         **kwargs,
@@ -71,10 +77,14 @@ async def generate_summary_reply(
             response_success = False
         else:
             try:
-                content = message
-                view = content
+                if "NO RELATIONSHIP.TERMINATE." in message:
+                    fail_reason = f"Return summarization error, the provided text content has no relationship to user's question. TERMINATE."
+                    response_success = False
+                else:
+                    content = message
+                    view = content
             except Exception as e:
-                fail_reason += f"Return summarization error，{str(e)}"
+                fail_reason = f"Return summarization error, {str(e)}"
                 response_success = False
 
         if not response_success:

diff --git a/examples/agents/single_summary_agent_dialogue_example.py b/examples/agents/single_summary_agent_dialogue_example.py
@@ -23,7 +23,8 @@
 from dbgpt.agent.memory.gpts_memory import GptsMemory
 from dbgpt.core.interface.llm import ModelMetadata
 
-if __name__ == "__main__":
+
+def summary_example_with_success():
     from dbgpt.model import OpenAILLMClient
 
     llm_client = OpenAILLMClient()
@@ -72,3 +73,56 @@
 
     ## dbgpt-vis message infos
     print(asyncio.run(default_memory.one_plan_chat_competions("summarize")))
+
+
+def summary_example_with_faliure():
+    from dbgpt.model import OpenAILLMClient
+
+    llm_client = OpenAILLMClient()
+    context: AgentContext = AgentContext(conv_id="summarize", llm_provider=llm_client)
+    context.llm_models = [ModelMetadata(model="gpt-3.5-turbo")]
+
+    default_memory = GptsMemory()
+    summarizer = SummaryAssistantAgent(memory=default_memory, agent_context=context)
+
+    user_proxy = UserProxyAgent(memory=default_memory, agent_context=context)
+
+    # Test the failure example
+    asyncio.run(
+        user_proxy.a_initiate_chat(
+            recipient=summarizer,
+            reviewer=user_proxy,
+            message="""I want to summarize advantages of Nuclear Power according to the following content.
+
+            Taylor Swift is an American singer-songwriter and actress who is one of the most prominent and successful figures in the music industry. She was born on December 13, 1989, in Reading, Pennsylvania, USA. Taylor Swift gained widespread recognition for her narrative songwriting style, which often draws from her personal experiences and relationships.
+
+            Swift's career began in country music, and her self-titled debut album was released in 2006. She quickly became a sensation in the country music scene with hits like "Tim McGraw" and "Teardrops on My Guitar." However, it was her transition to pop music with albums like "Fearless," "Speak Now," and "Red" that catapulted her to international superstardom.
+
+            Throughout her career, Taylor Swift has won numerous awards, including multiple Grammy Awards. Her albums consistently top charts, and her songs resonate with a wide audience due to their relatable lyrics and catchy melodies. Some of her most famous songs include "Love Story," "Blank Space," "Shake It Off," "Bad Blood," and "Lover."
+
+            Beyond music, Taylor Swift has ventured into acting with roles in movies like "Valentine's Day" and "The Giver." She is also known for her philanthropic efforts and her willingness to use her platform to advocate for various causes.
+
+            Taylor Swift is not only a successful artist but also an influential cultural icon known for her evolving musical style, storytelling abilities, and her impact on the entertainment industry.
+            """,
+        )
+    )
+
+    print(asyncio.run(default_memory.one_plan_chat_competions("summarize")))
+
+
+if __name__ == "__main__":
+    print(
+        "\033[92m=======================Start The Summary Assistant with Successful Results==================\033[0m"
+    )
+    summary_example_with_success()
+    print(
+        "\033[92m=======================The Summary Assistant with Successful Results Ended==================\n\n\033[91m"
+    )
+
+    print(
+        "\033[91m=======================Start The Summary Assistant with Fail Results==================\033[91m"
+    )
+    summary_example_with_faliure()
+    print(
+        "\033[91m=======================The Summary Assistant with Fail Results Ended==================\033[91m"
+    )