seed

Author: lizexu123

fastdeploy/config.py

@@ -116,6 +116,7 @@ def __init__(
         self.enable_redundant_experts = False
         self.redundant_experts_num = 0
         self.quantization = None
+        self.seed = 0
         for key, value in args.items():
             if hasattr(self, key):
                 setattr(self, key, value)

fastdeploy/engine/args_utils.py

@@ -313,7 +313,7 @@ class EngineArgs:
     Must be explicitly enabled via the `--enable-logprob` startup parameter to output logprob values.
     """
 
-    seed : Optional[int] = None
+    seed: Optional[int] = None
     """
     Random seed to use for initialization. If not set, a random seed is used.
     """
@@ -469,15 +469,12 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
             default=EngineArgs.enable_logprob,
             help="Enable output of token-level log probabilities.",
         )
-        model_group.add_argument("--enable-logprob",
-                                 action="store_true",
-                                 default=EngineArgs.enable_logprob,
-                                 help="Enable output of token-level log probabilities."
-                                 )
-        model_group.add_argument("--seed",
-                                type=int,
-                                defualt=None,
-                                help="Random seed for initialization. If not specified, a random seed will be used.")
+        model_group.add_argument(
+            "--seed",
+            type=int,
+            default=None,
+            help="Random seed for initialization. If not specified, a random seed will be used.",
+        )
         # Parallel processing parameters group
         parallel_group = parser.add_argument_group("Parallel Configuration")
         parallel_group.add_argument(

fastdeploy/model_executor/layers/sample/meta_data.py

@@ -27,13 +27,11 @@ class SamplingMetadata:
     """
 
     temperature: paddle.Tensor
-
     pre_token_ids: paddle.Tensor
     eos_token_ids: paddle.Tensor
     frequency_penalties: paddle.Tensor
     presence_penalties: paddle.Tensor
     repetition_penalties: paddle.Tensor
-
     min_dec_lens: paddle.Tensor
 
     bad_words_token_ids: paddle.Tensor

fastdeploy/utils.py

@@ -29,6 +29,8 @@
 from pathlib import Path
 from typing import Literal, TypeVar, Union
 
+import numpy as np
+import paddle
 import requests
 import yaml
 from aistudio_sdk.snapshot_download import snapshot_download as aistudio_download
@@ -291,6 +293,13 @@ def extract_tar(tar_path, output_dir):
         raise RuntimeError(f"Extraction failed: {e!s}")
 
 
+def set_random_seed(seed: int) -> None:
+    if seed is not None:
+        random.seed(seed)
+        np.random.seed(seed)
+        paddle.seed(seed)
+
+
 def download_model(url, output_dir, temp_tar):
     """
     下载模型，并将其解压到指定目录。

fastdeploy/worker/gpu_model_runner.py

@@ -78,14 +78,11 @@ def __init__(
         self.enable_mm = self.model_config.enable_mm
         self.rank = rank
         self.local_rank = local_rank
-        self.generators={}
         self.device_id = device_id
         self.speculative_method = self.fd_config.speculative_config.method
         self.speculative_decoding = self.speculative_method is not None
         self.enable_logprob = fd_config.model_config.enable_logprob
 
-        self.gene
-
         self.guided_backend = None
         if self.fd_config.parallel_config.guided_decoding_backend != "off":
             self.guided_backend = get_guided_backend(fd_config=self.fd_config)
@@ -130,8 +127,8 @@ def __init__(
         # Initialize share inputs
         self._init_share_inputs(self.parallel_config.max_num_seqs)
         self.infer_seed_increment = paddle.full(
-            shape=[self.parallel_config.max_num_seqs, 1],
-            dtype="int64")
+            shape=[self.parallel_config.max_num_seqs, 1], fill_value=4, dtype="int64"
+        )
         self.restore_chunked_prefill_request = dict()
 
         # Initialize attention Backend
@@ -270,6 +267,7 @@ def insert_tasks_v1(self, req_dicts: List[Request]):
             self.share_inputs["first_token_ids"][idx : idx + 1] = self.share_inputs["input_ids"][idx : idx + 1, :1]
             self.share_inputs["ori_seq_lens_encoder"][idx : idx + 1] = length
 
+            print("seed", request.get("seed"))
             if request.get("seed") is not None:
                 self.share_inputs["infer_seed"][idx : idx + 1] = request.get("seed")
 

fastdeploy/worker/gpu_worker.py

@@ -26,7 +26,7 @@
 from fastdeploy.config import FDConfig
 from fastdeploy.engine.request import Request
 from fastdeploy.platforms import current_platform
-from fastdeploy.utils import get_logger
+from fastdeploy.utils import get_logger, set_random_seed
 from fastdeploy.worker.gpu_model_runner import GPUModelRunner
 from fastdeploy.worker.output import ModelRunnerOutput
 from fastdeploy.worker.worker_base import WorkerBase
@@ -69,6 +69,7 @@ def init_device(self):
         else:
             raise RuntimeError(f"Not support device type: {self.device_config.device}")
 
+        set_random_seed(self.fd_config.model_config.seed)
         # Construct model runner
         self.model_runner: GPUModelRunner = GPUModelRunner(
             fd_config=self.fd_config,
@@ -123,6 +124,7 @@ def determine_available_memory(self) -> int:
 
         # 2. Profile run
         self.model_runner.profile_run()
+        set_random_seed(self.fd_config.model_config.seed)
 
         # 3. Statistical memory information
         paddle_reserved_mem_after_run = paddle.device.cuda.max_memory_reserved(local_rank)