diff --git a/src/google/adk/cli/fast_api.py b/src/google/adk/cli/fast_api.py
index 3d214ba9b..8d6ca2e46 100644
--- a/src/google/adk/cli/fast_api.py
+++ b/src/google/adk/cli/fast_api.py
@@ -523,7 +523,11 @@ def create_eval_set(
   )
   def list_eval_sets(app_name: str) -> list[str]:
     """Lists all eval sets for the given app."""
-    return eval_sets_manager.list_eval_sets(app_name)
+    try:
+      return eval_sets_manager.list_eval_sets(app_name)
+    except NotFoundError as e:
+      logger.warning(e)
+      return []
 
   @app.post(
       "/apps/{app_name}/eval_sets/{eval_set_id}/add_session",
diff --git a/src/google/adk/evaluation/local_eval_sets_manager.py b/src/google/adk/evaluation/local_eval_sets_manager.py
index 0e93b9201..b5669d538 100644
--- a/src/google/adk/evaluation/local_eval_sets_manager.py
+++ b/src/google/adk/evaluation/local_eval_sets_manager.py
@@ -27,6 +27,7 @@
 from pydantic import ValidationError
 from typing_extensions import override
 
+from ..errors.not_found_error import NotFoundError
 from ._eval_sets_manager_utils import add_eval_case_to_eval_set
 from ._eval_sets_manager_utils import delete_eval_case_from_eval_set
 from ._eval_sets_manager_utils import get_eval_case_from_eval_set
@@ -226,16 +227,30 @@ def create_eval_set(self, app_name: str, eval_set_id: str):
 
   @override
   def list_eval_sets(self, app_name: str) -> list[str]:
-    """Returns a list of EvalSets that belong to the given app_name."""
+    """Returns a list of EvalSets that belong to the given app_name.
+
+    Args:
+      app_name: The app name to list the eval sets for.
+
+    Returns:
+      A list of EvalSet ids.
+
+    Raises:
+      NotFoundError: If the eval directory for the app is not found.
+    """
     eval_set_file_path = os.path.join(self._agents_dir, app_name)
     eval_sets = []
-    for file in os.listdir(eval_set_file_path):
-      if file.endswith(_EVAL_SET_FILE_EXTENSION):
-        eval_sets.append(
-            os.path.basename(file).removesuffix(_EVAL_SET_FILE_EXTENSION)
-        )
-
-    return sorted(eval_sets)
+    try:
+      for file in os.listdir(eval_set_file_path):
+        if file.endswith(_EVAL_SET_FILE_EXTENSION):
+          eval_sets.append(
+              os.path.basename(file).removesuffix(_EVAL_SET_FILE_EXTENSION)
+          )
+      return sorted(eval_sets)
+    except FileNotFoundError as e:
+      raise NotFoundError(
+          f"Eval directory for app `{app_name}` not found."
+      ) from e
 
   @override
   def get_eval_case(
diff --git a/tests/unittests/evaluation/test_local_eval_sets_manager.py b/tests/unittests/evaluation/test_local_eval_sets_manager.py
index de00659e2..20159c66a 100644
--- a/tests/unittests/evaluation/test_local_eval_sets_manager.py
+++ b/tests/unittests/evaluation/test_local_eval_sets_manager.py
@@ -407,6 +407,15 @@ def test_local_eval_sets_manager_list_eval_sets_success(
 
     assert eval_sets == ["eval_set_1", "eval_set_2"]
 
+  def test_local_eval_sets_manager_list_eval_sets_not_found(
+      self, local_eval_sets_manager, mocker
+  ):
+    app_name = "test_app"
+    mocker.patch("os.listdir", side_effect=FileNotFoundError)
+
+    with pytest.raises(NotFoundError):
+      local_eval_sets_manager.list_eval_sets(app_name)
+
   def test_local_eval_sets_manager_add_eval_case_success(
       self, local_eval_sets_manager, mocker
   ):