File tree Expand file tree Collapse file tree 1 file changed +6
-3
lines changed
components/backends/trtllm/src/dynamo/trtllm/utils Expand file tree Collapse file tree 1 file changed +6
-3
lines changed Original file line number Diff line number Diff line change 2
2
# SPDX-License-Identifier: Apache-2.0
3
3
4
4
import argparse
5
+ import os
5
6
from typing import Optional
6
7
7
8
from tensorrt_llm .llmapi import BuildConfig
13
14
DisaggregationStrategy ,
14
15
)
15
16
17
+ DYN_NAMESPACE = os .environ .get ("DYN_NAMESPACE" , "dynamo" )
18
+
16
19
# Default endpoint for the next worker.
17
- DEFAULT_ENDPOINT = "dyn://dynamo .tensorrt_llm.generate"
20
+ DEFAULT_ENDPOINT = f "dyn://{ DYN_NAMESPACE } .tensorrt_llm.generate"
18
21
DEFAULT_MODEL_PATH = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
19
- DEFAULT_NEXT_ENDPOINT = "dyn://dynamo .tensorrt_llm_next.generate"
20
- DEFAULT_ENCODE_ENDPOINT = "dyn://dynamo .tensorrt_llm_encode.generate"
22
+ DEFAULT_NEXT_ENDPOINT = f "dyn://{ DYN_NAMESPACE } .tensorrt_llm_next.generate"
23
+ DEFAULT_ENCODE_ENDPOINT = f "dyn://{ DYN_NAMESPACE } .tensorrt_llm_encode.generate"
21
24
DEFAULT_DISAGGREGATION_STRATEGY = DisaggregationStrategy .DECODE_FIRST
22
25
DEFAULT_DISAGGREGATION_MODE = DisaggregationMode .AGGREGATED
23
26
You can’t perform that action at this time.
0 commit comments