Skip to content

Commit 7cbe6b2

Browse files
authored
[FDConfig] Turn on the CUDAGraph + Speculative Decoding switch (#4511)
1 parent 153f15d commit 7cbe6b2

1 file changed

Lines changed: 2 additions & 7 deletions

File tree

fastdeploy/config.py

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1511,14 +1511,13 @@ def postprocess(self):
15111511

15121512
# Adjustment GraphOptConfig
15131513
if (
1514-
(self.speculative_config is not None and self.speculative_config.method is not None)
1514+
(self.scheduler_config.splitwise_role != "mixed")
15151515
or (self.model_config is not None and self.model_config.enable_mm is True)
15161516
or (self.load_config is not None and self.load_config.dynamic_load_weight is True)
1517-
or (self.scheduler_config.splitwise_role != "mixed")
15181517
):
15191518
self.graph_opt_config.use_cudagraph = False
15201519
logger.info(
1521-
"CUDAGraph does not support to be started together with SpeculativeDecode and MultiModel temporarily, but has been automatically closed!"
1520+
"CUDAGraph does not support to be started together with MultiModel temporarily, but has been automatically closed!"
15221521
)
15231522
if self.load_config is not None and self.load_config.dynamic_load_weight is True:
15241523
self.graph_opt_config.graph_opt_level = 0
@@ -1634,10 +1633,6 @@ def check(self):
16341633

16351634
# Check graph optimization config
16361635
if self.graph_opt_config.use_cudagraph:
1637-
if self.speculative_config is not None:
1638-
assert (
1639-
self.speculative_config.method is None
1640-
), "CUDAGraph does not support the simultaneous use of Speculative Decoding"
16411636
if self.model_config is not None:
16421637
assert (
16431638
self.model_config.enable_mm is not True

0 commit comments

Comments
 (0)