Make sure to log thoughts if verbosity_level is set to high (huggingface#964)

Author: aymeric-roucher

src/smolagents/agents.py

@@ -1054,15 +1054,24 @@ def step(self, memory_step: ActionStep) -> Union[None, Any]:
                 stop_sequences=["Observation:"],
             )
             memory_step.model_output_message = model_message
-            if model_message.tool_calls is None or len(model_message.tool_calls) == 0:
-                raise Exception("Model did not call any tools. Call `final_answer` tool to return a final answer.")
-            tool_call = model_message.tool_calls[0]
-            tool_name, tool_call_id = tool_call.function.name, tool_call.id
-            tool_arguments = tool_call.function.arguments
-
         except Exception as e:
             raise AgentGenerationError(f"Error in generating tool call with model:\n{e}", self.logger) from e
 
+        self.logger.log_markdown(
+            content=str(model_message.raw),
+            title="Output message of the LLM:",
+            level=LogLevel.DEBUG,
+        )
+
+        if model_message.tool_calls is None or len(model_message.tool_calls) == 0:
+            raise AgentParsingError(
+                "Model did not call any tools. Call `final_answer` tool to return a final answer.", self.logger
+            )
+
+        tool_call = model_message.tool_calls[0]
+        tool_name, tool_call_id = tool_call.function.name, tool_call.id
+        tool_arguments = tool_call.function.arguments
+
         memory_step.tool_calls = [ToolCall(name=tool_name, arguments=tool_arguments, id=tool_call_id)]
 
         # Execute

tests/test_agents.py

@@ -387,26 +387,6 @@ def test_reset_conversations(self):
         assert output == 7.2904
         assert len(agent.memory.steps) == 3
 
-    def test_code_agent_code_errors_show_offending_line_and_error(self):
-        agent = CodeAgent(tools=[PythonInterpreterTool()], model=fake_code_model_error)
-        output = agent.run("What is 2 multiplied by 3.6452?")
-        assert isinstance(output, AgentText)
-        assert output == "got an error"
-        assert "Code execution failed at line 'error_function()'" in str(agent.memory.steps[1].error)
-        assert "ValueError" in str(agent.memory.steps)
-
-    def test_code_agent_code_error_saves_previous_print_outputs(self):
-        agent = CodeAgent(tools=[PythonInterpreterTool()], model=fake_code_model_error, verbosity_level=10)
-        agent.run("What is 2 multiplied by 3.6452?")
-        assert "Flag!" in str(agent.memory.steps[1].observations)
-
-    def test_code_agent_syntax_error_show_offending_lines(self):
-        agent = CodeAgent(tools=[PythonInterpreterTool()], model=fake_code_model_syntax_error)
-        output = agent.run("What is 2 multiplied by 3.6452?")
-        assert isinstance(output, AgentText)
-        assert output == "got an error"
-        assert '    print("Failing due to unexpected indent")' in str(agent.memory.steps)
-
     def test_setup_agent_with_empty_toolbox(self):
         ToolCallingAgent(model=FakeToolCallModel(), tools=[])
 
@@ -496,15 +476,6 @@ def test_agent_description_gets_correctly_inserted_in_system_prompt(self):
         assert "{{managed_agents_descriptions}}" not in managed_agent.system_prompt
         assert "You can also give tasks to team members." in manager_agent.system_prompt
 
-    def test_code_agent_missing_import_triggers_advice_in_error_log(self):
-        # Set explicit verbosity level to 1 to override the default verbosity level of -1 set in CI fixture
-        agent = CodeAgent(tools=[], model=fake_code_model_import, verbosity_level=1)
-
-        with agent.logger.console.capture() as capture:
-            agent.run("Count to 3")
-        str_output = capture.get()
-        assert "`additional_authorized_imports`" in str_output.replace("\n", "")
-
     def test_replay_shows_logs(self):
         agent = CodeAgent(
             tools=[], model=fake_code_model_import, verbosity_level=0, additional_authorized_imports=["numpy"]
@@ -630,6 +601,31 @@ def test_instantiation_with_final_answer_tool(self, tools, expected_final_answer
         assert "final_answer" in agent.tools
         assert isinstance(agent.tools["final_answer"], expected_final_answer_tool)
 
+    def test_logs_display_thoughts_even_if_error(self):
+        def fake_json_model_no_call(messages, stop_sequences=None, tools_to_call_from=None):
+            return ChatMessage(
+                role="assistant",
+                content="""I don't want to call tools today""",
+                tool_calls=None,
+                raw="""I don't want to call tools today""",
+            )
+
+        agent_toolcalling = ToolCallingAgent(model=fake_json_model_no_call, tools=[], max_steps=1, verbosity_level=10)
+        with agent_toolcalling.logger.console.capture() as capture:
+            agent_toolcalling.run("Dummy task")
+        assert "don't" in capture.get() and "want" in capture.get()
+
+        def fake_code_model_no_call(messages, stop_sequences=None):
+            return ChatMessage(
+                role="assistant",
+                content="""I don't want to write an action today""",
+            )
+
+        agent_code = CodeAgent(model=fake_code_model_no_call, tools=[], max_steps=1, verbosity_level=10)
+        with agent_code.logger.console.capture() as capture:
+            agent_code.run("Dummy task")
+        assert "don't" in capture.get() and "want" in capture.get()
+
     def test_step_number(self):
         fake_model = MagicMock()
         fake_model.last_input_token_count = 10
@@ -892,6 +888,35 @@ def fake_code_model(messages, stop_sequences=None, grammar=None) -> str:
             agent.run("Test request")
         assert "secret\\\\" in repr(capture.get())
 
+    def test_missing_import_triggers_advice_in_error_log(self):
+        # Set explicit verbosity level to 1 to override the default verbosity level of -1 set in CI fixture
+        agent = CodeAgent(tools=[], model=fake_code_model_import, verbosity_level=1)
+
+        with agent.logger.console.capture() as capture:
+            agent.run("Count to 3")
+        str_output = capture.get()
+        assert "`additional_authorized_imports`" in str_output.replace("\n", "")
+
+    def test_errors_show_offending_line_and_error(self):
+        agent = CodeAgent(tools=[PythonInterpreterTool()], model=fake_code_model_error)
+        output = agent.run("What is 2 multiplied by 3.6452?")
+        assert isinstance(output, AgentText)
+        assert output == "got an error"
+        assert "Code execution failed at line 'error_function()'" in str(agent.memory.steps[1].error)
+        assert "ValueError" in str(agent.memory.steps)
+
+    def test_error_saves_previous_print_outputs(self):
+        agent = CodeAgent(tools=[PythonInterpreterTool()], model=fake_code_model_error, verbosity_level=10)
+        agent.run("What is 2 multiplied by 3.6452?")
+        assert "Flag!" in str(agent.memory.steps[1].observations)
+
+    def test_syntax_error_show_offending_lines(self):
+        agent = CodeAgent(tools=[PythonInterpreterTool()], model=fake_code_model_syntax_error)
+        output = agent.run("What is 2 multiplied by 3.6452?")
+        assert isinstance(output, AgentText)
+        assert output == "got an error"
+        assert '    print("Failing due to unexpected indent")' in str(agent.memory.steps)
+
     def test_change_tools_after_init(self):
         from smolagents import tool