enlarge num_instance

Author: lvhan028

lmdeploy/pytorch/engine/mp_engine/base_worker.py

@@ -20,7 +20,8 @@ class EngineInstancePool:
     def __init__(self, engine):
         from lmdeploy.pytorch.engine import Engine
         self.engine: Engine = engine
-        self.num_instance = self.engine.engine_config.max_batch_size
+        # enlarge `num_instance`, otherwise an sequence cannot be stopped in time
+        self.num_instance = self.engine.engine_config.max_batch_size * 2
         self.pool = None
 
     def create_instance_pool(self, num_instance: int):

lmdeploy/serve/async_engine.py

@@ -448,12 +448,14 @@ async def stop_all_session(self):
         """Stop all running sessions."""
         logger.info('stop all sessions')
         tasks = []
+        session_ids = []
         for session_id in list(self.id2inst.keys()):
             generator = self.id2inst.get(session_id)
             if generator:
+                session_ids.append(session_id)
                 tasks.append(generator.async_cancel(session_id))
         await asyncio.gather(*tasks)
-        logger.info('all sessions stopped')
+        logger.info(f'all {len(session_ids)} sessions stopped')
 
     async def stop_session(self, session_id: int):
         """Stop a session by a session_id."""