Protect against engine initialization failure (#70)

* recover from engine init error * remove unused structlog import * fix
octoml · Nov 20, 2023 · 4d75375 · 4d75375
1 parent 43c8ebb
commit 4d75375
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 6 deletions.
diff --git a/serve/mlc_serve/engine/staging_engine.py b/serve/mlc_serve/engine/staging_engine.py
@@ -65,11 +65,14 @@ def __init__(
         )
 
     def start(self):
-        self.worker_process.start()
-        if not self.ready_event.wait(timeout=180):
-            raise RuntimeError(
-                "StagingInferenceEngine worker is not ready before timeout."
-            )
+        try:
+            self.worker_process.start()
+            if not self.ready_event.wait(timeout=90):
+                raise RuntimeError(
+                    "StagingInferenceEngine worker is not ready before timeout."
+                )
+        except:
+            raise RuntimeError("Failed to start StagingInferenceEngine worker process.")
 
     def stop(self):
         self.command_queue.put(ShutdownCommand())

diff --git a/serve/mlc_serve/engine/staging_engine_worker.py b/serve/mlc_serve/engine/staging_engine_worker.py
@@ -11,7 +11,7 @@
 
 from .base import FinishReason, RequestId, RequestState
 from .model_module import DecodeRequest, ModelModule, PrefillRequest, SequenceId
-import structlog
+
 
 logger = logging.getLogger(__name__)