Add E2B code interpreter 🥳

Author: aymeric-roucher

.gitignore

@@ -36,6 +36,7 @@ sdist/
 var/
 wheels/
 share/python-wheels/
+node_modules/
 *.egg-info/
 .installed.cfg
 *.egg
@@ -156,4 +157,8 @@ dmypy.json
 cython_debug/
 
 # PyCharm
-#.idea/
+#.idea/
+
+# Archive
+archive/
+savedir/

Dockerfile

@@ -1,14 +1,12 @@
 # Base Python image
-FROM python:3.9-slim
+FROM python:3.12-slim
 
 # Set working directory
 WORKDIR /app
 
 # Install build dependencies
 RUN apt-get update && apt-get install -y \
     build-essential \
-    gcc \
-    g++ \
     zlib1g-dev \
     libjpeg-dev \
     libpng-dev \

e2b.Dockerfile

@@ -0,0 +1,5 @@
+# You can use most Debian-based base images
+FROM e2bdev/code-interpreter:latest 
+
+# Install dependencies and customize sandbox
+RUN pip install git+https://github.com/huggingface/agents.git

e2b.toml

@@ -0,0 +1,16 @@
+# This is a config for E2B sandbox template.
+# You can use template ID (qywp2ctmu2q7jzprcf4j) to create a sandbox:
+
+# Python SDK
+# from e2b import Sandbox, AsyncSandbox
+# sandbox = Sandbox("qywp2ctmu2q7jzprcf4j") # Sync sandbox
+# sandbox = await AsyncSandbox.create("qywp2ctmu2q7jzprcf4j") # Async sandbox
+
+# JS SDK
+# import { Sandbox } from 'e2b'
+# const sandbox = await Sandbox.create('qywp2ctmu2q7jzprcf4j')
+
+team_id = "f8776d3a-df2f-4a1d-af48-68c2e13b3b87"
+start_cmd = "/root/.jupyter/start-up.sh"
+dockerfile = "e2b.Dockerfile"
+template_id = "qywp2ctmu2q7jzprcf4j"

examples/docker_example.py

@@ -1,8 +1,8 @@
-from agents.tools.search import DuckDuckGoSearchTool
+from agents.default_tools.search import DuckDuckGoSearchTool
 from agents.docker_alternative import DockerPythonInterpreter
 
 
-from agents.tool import Tool
+from agents.tools import Tool
 
 class DummyTool(Tool):
     name = "echo"

examples/dummytool.py

@@ -1,4 +1,4 @@
-from agents.tool import Tool
+from agents.tools import Tool
 
 
 class DummyTool(Tool):

examples/e2b_example.py

@@ -0,0 +1,44 @@
+from agents import Tool, CodeAgent
+from agents.default_tools.search import VisitWebpageTool
+from dotenv import load_dotenv
+load_dotenv()
+
+LAUNCH_GRADIO = False
+
+class GetCatImageTool(Tool):
+    name="get_cat_image"
+    description = "Get a cat image"
+    inputs = {}
+    output_type = "image"
+
+    def __init__(self):
+        super().__init__()
+        self.url = "https://em-content.zobj.net/source/twitter/53/robot-face_1f916.png"
+
+    def forward(self):
+        from PIL import Image
+        import requests
+        from io import BytesIO
+
+        response = requests.get(self.url)
+
+        return Image.open(BytesIO(response.content))
+
+get_cat_image = GetCatImageTool()
+
+
+agent = CodeAgent(
+    tools = [get_cat_image, VisitWebpageTool()],
+    additional_authorized_imports=["Pillow", "requests", "markdownify"], # "duckduckgo-search", 
+    use_e2b_executor=False
+)
+
+if LAUNCH_GRADIO:
+    from agents.gradio_ui import GradioUI
+
+    GradioUI(agent).launch()
+else:
+    agent.run(
+        "Return me an image of Lincoln's preferred pet",
+        additional_context="Here is a webpage about US presidents and pets: https://www.9lives.com/blog/a-history-of-cats-in-the-white-house/"
+    )

src/agents/__init__.py

@@ -24,22 +24,28 @@
 
 if TYPE_CHECKING:
     from .agents import *
-    from .default_tools import *
+    from .default_tools.base import *
+    from .default_tools.search import *
     from .gradio_ui import *
     from .llm_engines import *
     from .local_python_executor import *
     from .monitoring import *
     from .prompts import *
-    from .tools.search import *
-    from .tool import *
+    from .tools import *
     from .types import *
     from .utils import *
+    from .default_tools.search import *
 
 
 else:
     import sys
-
     _file = globals()["__file__"]
+    import_structure = define_import_structure(_file)
+    import_structure[""]= {"__version__": __version__}
     sys.modules[__name__] = _LazyModule(
-        __name__, _file, define_import_structure(_file), module_spec=__spec__
+        __name__,
+        _file,
+        import_structure,
+        module_spec=__spec__,
+        extra_objects={"__version__": __version__}
     )

src/agents/agents.py

@@ -27,7 +27,7 @@
 
 from .utils import console, parse_code_blob, parse_json_tool_call, truncate_content
 from .types import AgentAudio, AgentImage
-from .default_tools import BASE_PYTHON_TOOLS, FinalAnswerTool
+from .default_tools.base import FinalAnswerTool
 from .llm_engines import HfApiEngine, MessageRole
 from .monitoring import Monitor
 from .prompts import (
@@ -42,8 +42,9 @@
     SYSTEM_PROMPT_PLAN_UPDATE,
     SYSTEM_PROMPT_PLAN,
 )
-from .local_python_executor import LIST_SAFE_MODULES, evaluate_python_code
-from .tool import (
+from .local_python_executor import BASE_BUILTIN_MODULES, LocalPythonExecutor
+from .e2b_executor import E2BExecutor
+from .tools import (
     DEFAULT_TOOL_DESCRIPTION_TEMPLATE,
     Tool,
     get_tool_description_with_args,
@@ -169,17 +170,6 @@ def format_prompt_with_managed_agents_descriptions(
     else:
         return prompt_template.replace(agent_descriptions_placeholder, "")
 
-
-def format_prompt_with_imports(
-    prompt_template: str, authorized_imports: List[str]
-) -> str:
-    if "<<authorized_imports>>" not in prompt_template:
-        raise AgentError(
-            "Tag '<<authorized_imports>>' should be provided in the prompt."
-        )
-    return prompt_template.replace("<<authorized_imports>>", str(authorized_imports))
-
-
 class BaseAgent:
     def __init__(
         self,
@@ -264,11 +254,6 @@ def initialize_system_prompt(self):
         self.system_prompt = format_prompt_with_managed_agents_descriptions(
             self.system_prompt, self.managed_agents
         )
-        if hasattr(self, "authorized_imports"):
-            self.system_prompt = format_prompt_with_imports(
-                self.system_prompt,
-                list(set(LIST_SAFE_MODULES) | set(getattr(self, "authorized_imports"))),
-            )
 
         return self.system_prompt
 
@@ -439,9 +424,7 @@ def execute_tool_call(self, tool_name: str, arguments: Dict[str, str]) -> Any:
             tool_name (`str`): Name of the Tool to execute (should be one from self.toolbox).
             arguments (Dict[str, str]): Arguments passed to the Tool.
         """
-        available_tools = self.toolbox.tools
-        if self.managed_agents is not None:
-            available_tools = {**available_tools, **self.managed_agents}
+        available_tools = {**self.toolbox.tools, **self.managed_agents}
         if tool_name not in available_tools:
             error_msg = f"Error: unknown tool {tool_name}, should be instead one of {list(available_tools.keys())}."
             console.print(f"[bold red]{error_msg}")
@@ -674,8 +657,6 @@ def planning_step(self, task, is_first_step: bool, iteration: int):
                     ),
                     managed_agents_descriptions=(
                         show_agents_descriptions(self.managed_agents)
-                        if self.managed_agents is not None
-                        else ""
                     ),
                     answer_facts=answer_facts,
                 ),
@@ -729,8 +710,6 @@ def planning_step(self, task, is_first_step: bool, iteration: int):
                     ),
                     managed_agents_descriptions=(
                         show_agents_descriptions(self.managed_agents)
-                        if self.managed_agents is not None
-                        else ""
                     ),
                     facts_update=facts_update,
                     remaining_steps=(self.max_iterations - iteration),
@@ -891,6 +870,7 @@ def __init__(
         grammar: Optional[Dict[str, str]] = None,
         additional_authorized_imports: Optional[List[str]] = None,
         planning_interval: Optional[int] = None,
+        use_e2b_executor: bool = False,
         **kwargs,
     ):
         if llm_engine is None:
@@ -909,17 +889,24 @@ def __init__(
             **kwargs,
         )
 
-        self.python_evaluator = evaluate_python_code
         self.additional_authorized_imports = (
             additional_authorized_imports if additional_authorized_imports else []
         )
+        all_tools = {**self.toolbox.tools, **self.managed_agents}
+        if use_e2b_executor:
+            self.python_executor = E2BExecutor(self.additional_authorized_imports, list(all_tools.values()))
+        else:
+            self.python_executor = LocalPythonExecutor(self.additional_authorized_imports, all_tools)
         self.authorized_imports = list(
-            set(LIST_SAFE_MODULES) | set(self.additional_authorized_imports)
+            set(BASE_BUILTIN_MODULES) | set(self.additional_authorized_imports)
         )
+        if "{{authorized_imports}}" not in self.system_prompt:
+            raise AgentError(
+                "Tag '{{authorized_imports}}' should be provided in the prompt."
+            )
         self.system_prompt = self.system_prompt.replace(
             "{{authorized_imports}}", str(self.authorized_imports)
         )
-        self.custom_tools = {}
 
     def step(self, log_entry: ActionStep) -> Union[None, Any]:
         """
@@ -991,22 +978,12 @@ def step(self, log_entry: ActionStep) -> Union[None, Any]:
         )
 
         try:
-            static_tools = {
-                **BASE_PYTHON_TOOLS.copy(),
-                **self.toolbox.tools,
-            }
-            if self.managed_agents is not None:
-                static_tools = {**static_tools, **self.managed_agents}
-            output = self.python_evaluator(
+            output, execution_logs = self.python_executor(
                 code_action,
-                static_tools=static_tools,
-                custom_tools=self.custom_tools,
-                state=self.state,
-                authorized_imports=self.authorized_imports,
             )
-            if len(self.state["print_outputs"]) > 0:
-                console.print(Group(Text("Print outputs:", style="bold"), Text(self.state["print_outputs"])))
-            observation = "Print outputs:\n" + self.state["print_outputs"]
+            if len(execution_logs) > 0:
+                console.print(Group(Text("Execution logs:", style="bold"), Text(execution_logs)))
+            observation = "Execution logs:\n" + execution_logs
             if output is not None:
                 truncated_output = truncate_content(
                     str(output)
@@ -1026,7 +1003,7 @@ def step(self, log_entry: ActionStep) -> Union[None, Any]:
                 console.print(Group(Text("Final answer:", style="bold"), Text(str(output), style="bold green")))
                 log_entry.action_output = output
                 return output
-        return None
+
 
 
 class ManagedAgent:

src/agents/tools/__init__.py src/agents/default_tools/__init__.py

@@ -15,75 +15,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import json
-import math
 from dataclasses import dataclass
-from math import sqrt
 from typing import Dict
 
 from huggingface_hub import hf_hub_download, list_spaces
 
 from transformers.utils import is_offline_mode
-from .local_python_executor import LIST_SAFE_MODULES, evaluate_python_code
-from .tool import TOOL_CONFIG_FILE, Tool
-
-
-def custom_print(*args):
-    return None
-
-
-BASE_PYTHON_TOOLS = {
-    "print": custom_print,
-    "isinstance": isinstance,
-    "range": range,
-    "float": float,
-    "int": int,
-    "bool": bool,
-    "str": str,
-    "set": set,
-    "list": list,
-    "dict": dict,
-    "tuple": tuple,
-    "round": round,
-    "ceil": math.ceil,
-    "floor": math.floor,
-    "log": math.log,
-    "exp": math.exp,
-    "sin": math.sin,
-    "cos": math.cos,
-    "tan": math.tan,
-    "asin": math.asin,
-    "acos": math.acos,
-    "atan": math.atan,
-    "atan2": math.atan2,
-    "degrees": math.degrees,
-    "radians": math.radians,
-    "pow": math.pow,
-    "sqrt": sqrt,
-    "len": len,
-    "sum": sum,
-    "max": max,
-    "min": min,
-    "abs": abs,
-    "enumerate": enumerate,
-    "zip": zip,
-    "reversed": reversed,
-    "sorted": sorted,
-    "all": all,
-    "any": any,
-    "map": map,
-    "filter": filter,
-    "ord": ord,
-    "chr": chr,
-    "next": next,
-    "iter": iter,
-    "divmod": divmod,
-    "callable": callable,
-    "getattr": getattr,
-    "hasattr": hasattr,
-    "setattr": setattr,
-    "issubclass": issubclass,
-    "type": type,
-}
+from ..local_python_executor import BASE_BUILTIN_MODULES, BASE_PYTHON_TOOLS, evaluate_python_code
+from ..tools import TOOL_CONFIG_FILE, Tool
 
 
 @dataclass
@@ -136,10 +75,10 @@ class PythonInterpreterTool(Tool):
 
     def __init__(self, *args, authorized_imports=None, **kwargs):
         if authorized_imports is None:
-            self.authorized_imports = list(set(LIST_SAFE_MODULES))
+            self.authorized_imports = list(set(BASE_BUILTIN_MODULES))
         else:
             self.authorized_imports = list(
-                set(LIST_SAFE_MODULES) | set(authorized_imports)
+                set(BASE_BUILTIN_MODULES) | set(authorized_imports)
             )
         self.inputs = {
             "code": {