fix access None req_metrics when sending abort_request

Author: lvhan028

lmdeploy/metrics/metrics_processor.py

@@ -122,7 +122,9 @@ async def _run_metrics_handler(self):
                 outputs, req_state, iteration_stats = update_data
 
                 # update request state according the engine events
-                req_state.update_from_events(outputs.req_metrics.engine_events)
+                if outputs and outputs.req_metrics:
+                    # when users visit "/abort_request" endpoint, `req_metrics` might be None
+                    req_state.update_from_events(outputs.req_metrics.engine_events)
 
                 # update iteration stats based on outputs and request state.
                 # some attributes of req_state will also be updated, e.g., lastest_token_time

lmdeploy/metrics/stats.py

@@ -198,6 +198,9 @@ def update_from_output(self, outputs: EngineOutput, req_state: RequestState):
             outputs (EngineOutput): The output from the engine containing information about the current iteration.
             req_state (RequestState): The state of the request, including timestamps and token counts.
         """
+        if outputs.req_metrics is None:
+            # when users visit "/abort_request" endpoint, `req_metrics` might be None
+            return
         new_generation_tokens = len(outputs.token_ids)
         if new_generation_tokens == 0:
             return

lmdeploy/serve/openai/api_server.py

@@ -955,18 +955,8 @@ async def generate(request: GenerateReqInput, raw_request: Request = None):
         do_preprocess=False,
     )
 
-    def create_finish_reason(finish_reason):
-        # TODO: add detail info
-        if not finish_reason:
-            return None
-        if finish_reason == 'length':
-            return dict(type='length')
-        if finish_reason == 'stop':
-            return dict(type='stop')
-        return dict(type='abort')
-
     def create_generate_response_json(res, text, output_ids, logprobs, finish_reason):
-        meta = GenerateReqMetaOutput(finish_reason=create_finish_reason(finish_reason),
+        meta = GenerateReqMetaOutput(finish_reason=dict(type=finish_reason) if finish_reason else None,
                                      output_token_logprobs=logprobs or None,
                                      prompt_tokens=res.input_token_len,
                                      completion_tokens=res.generate_token_len)
@@ -1005,7 +995,7 @@ async def _inner_call():
                 for tok, tok_logprobs in zip(res.token_ids, res.logprobs):
                     logprobs.append((tok_logprobs[tok], tok))
         nonlocal response
-        meta = GenerateReqMetaOutput(finish_reason=create_finish_reason(res.finish_reason),
+        meta = GenerateReqMetaOutput(finish_reason=dict(type=res.finish_reason) if res.finish_reason else None,
                                      output_token_logprobs=logprobs or None,
                                      prompt_tokens=res.input_token_len,
                                      completion_tokens=res.generate_token_len)

lmdeploy/serve/openai/protocol.py

@@ -256,7 +256,7 @@ class ChatCompletionResponseStreamChoice(BaseModel):
     index: int
     delta: DeltaMessage
     logprobs: Optional[ChoiceLogprobs] = None
-    finish_reason: Optional[Literal['stop', 'length', 'tool_calls', 'error']] = None
+    finish_reason: Optional[Literal['stop', 'length', 'tool_calls', 'error', 'abort']] = None
 
 
 class ChatCompletionStreamResponse(BaseModel):
@@ -314,7 +314,7 @@ class CompletionResponseChoice(BaseModel):
     text: str
     logprobs: Optional[LogProbs] = None
     gen_tokens: Optional[List[int]] = None
-    finish_reason: Optional[Literal['stop', 'length', 'tool_calls', 'error']] = None
+    finish_reason: Optional[Literal['stop', 'length', 'tool_calls', 'error', 'abort']] = None
 
 
 class CompletionResponse(BaseModel):
@@ -430,7 +430,7 @@ class GenerateResponse(BaseModel):
     tokens: int
     input_tokens: int
     history_tokens: int
-    finish_reason: Optional[Literal['stop', 'length', 'tool_calls', 'error']] = None
+    finish_reason: Optional[Literal['stop', 'length', 'tool_calls', 'error', 'abort']] = None
 
 
 class UpdateParamsRequest(BaseModel):