diff --git a/.github/workflows/macOS_test.yml b/.github/workflows/macOS_test.yml index 28a7d3803..9a3a16b1c 100644 --- a/.github/workflows/macOS_test.yml +++ b/.github/workflows/macOS_test.yml @@ -180,6 +180,59 @@ jobs: git submodule init LazyLLM-Env git submodule update LazyLLM-Env + - name: Create Output and Cache Directories + run: | + mkdir -p github_cache + mkdir -p github_cache/output + mkdir -p github_cache/paid_llm_cache + echo "LAZYLLM_TEMP_DIR=github_cache/output" >> $GITHUB_ENV + echo "LAZYLLM_CACHE_DIR=github_cache/paid_llm_cache" >> $GITHUB_ENV + + - name: Set Cache Mode Based on Run Context + run: | + echo "GITHUB_RUN_ATTEMPT=$GITHUB_RUN_ATTEMPT" + echo "LAZYLLM_CACHE_ONLINE_MODULE=True" >> $GITHUB_ENV + echo "LAZYLLM_CACHE_STRATEGY=sqlite" >> $GITHUB_ENV + echo "LAZYLLM_CACHE_MODE=RO" >> $GITHUB_ENV + + - name: Show ENV + run: | + echo "LAZYLLM_TEMP_DIR: $LAZYLLM_TEMP_DIR" + echo "LAZYLLM_CACHE_DIR: $LAZYLLM_CACHE_DIR" + echo "LAZYLLM_CACHE_MODE: $LAZYLLM_CACHE_MODE" + echo "LAZYLLM_CACHE_STRATEGY: $LAZYLLM_CACHE_STRATEGY" + echo "LAZYLLM_CACHE_ONLINE_MODULE: $LAZYLLM_CACHE_ONLINE_MODULE" + + - name: Get SHAs + run: | + CURRENT_SHA=$(git rev-parse HEAD) + if git rev-parse HEAD^ >/dev/null 2>&1; then + PREVIOUS_SHA=$(git rev-parse HEAD^) + else + PREVIOUS_SHA=$CURRENT_SHA + fi + echo "PREVIOUS_SHA=$PREVIOUS_SHA" >> $GITHUB_ENV + if [ "$GITHUB_EVENT_NAME" = "pull_request_target" ]; then + BASE_SHA="${{ github.event.pull_request.base.sha }}" + echo "BASE_SHA=$BASE_SHA" >> $GITHUB_ENV + fi + + - name: Restore Cache (Push) + if: ${{ github.event_name == 'push' }} + uses: lwj-st/cache/restore@v4.2.4 + with: + path: github_cache/ + key: python-result-cache-${{ env.PREVIOUS_SHA }} + enableCrossOsArchive: true + + - name: Restore Cache (Pull Request) + if: ${{ github.event_name == 'pull_request_target' }} + uses: lwj-st/cache/restore@v4.2.4 + with: + path: github_cache/ + key: python-result-cache-${{ env.BASE_SHA }} + enableCrossOsArchive: true + - name: Merge PR branch into main if: github.event_name == 'pull_request_target' run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0ee94a44a..9363bf8bb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,7 @@ on: - "docs/assets/**" - "docs/**" env: - CI_PATH: '/home/mnt/platform_ci/GitHub/${{ github.repository }}/${GITHUB_RUN_NUMBER}' + CI_PATH: '/home/mnt/platform_ci/GitHub/${{ github.repository }}/${{ github.run_number }}' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -64,6 +64,8 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Build doc run: | @@ -88,9 +90,10 @@ jobs: - name: Move code to custom directory run: | set -ex - mv $GITHUB_WORKSPACE/* ${{ env.CI_PATH }}/ + cp -r $GITHUB_WORKSPACE/. ${{ env.CI_PATH }}/ BasicTests: + if: contains(github.event.head_commit.message, 'temp_close') runs-on: tps_sco_nv needs: [Clone] steps: @@ -111,6 +114,7 @@ jobs: python -m pytest --lf --last-failed-no-failures=all -m "not skip_on_linux" --durations=0 --reruns=2 -v --cov=lazyllm --cov-append --cov-report=html tests/basic_tests/ AdvancedStandardTests: + if: contains(github.event.head_commit.message, 'temp_close') runs-on: tps_sco_nv needs: [Clone] steps: @@ -134,6 +138,7 @@ jobs: fi AdvancedFullTests: + if: contains(github.event.head_commit.message, 'temp_close') runs-on: tps_sco_nv needs: [Clone] steps: @@ -158,7 +163,75 @@ jobs: runs-on: tps_sco_nv needs: [ Clone ] steps: + - name: Create Output and Cache Directories + run: | + cd ${{ env.CI_PATH }} + mkdir -p github_cache + mkdir -p github_cache/output + mkdir -p github_cache/paid_llm_cache + echo "LAZYLLM_TEMP_DIR=github_cache/output" >> $GITHUB_ENV + echo "LAZYLLM_CACHE_DIR=github_cache/paid_llm_cache" >> $GITHUB_ENV + + - name: Set Cache Mode Based on Run Context + run: | + echo "GITHUB_RUN_ATTEMPT=$GITHUB_RUN_ATTEMPT" + echo "LAZYLLM_CACHE_ONLINE_MODULE=True" >> $GITHUB_ENV + echo "LAZYLLM_CACHE_STRATEGY=sqlite" >> $GITHUB_ENV + if [ "$GITHUB_RUN_ATTEMPT" = "1" ] && [ "$GITHUB_EVENT_NAME" = "push" ]; then + echo "This is the first run, and push to main branch." + echo "LAZYLLM_CACHE_MODE=RW" >> $GITHUB_ENV + elif [ "$GITHUB_RUN_ATTEMPT" != "1" ] && [ "$GITHUB_EVENT_NAME" = "push" ]; then + echo "This is a re-run! (GITHUB_RUN_ATTEMPT=$GITHUB_RUN_ATTEMPT), and push to main branch." + echo "LAZYLLM_CACHE_MODE=WO" >> $GITHUB_ENV + elif [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + echo "This is a pull request from other branch to main." + echo "LAZYLLM_CACHE_MODE=RO" >> $GITHUB_ENV + fi + + - name: Show ENV + run: | + echo "LAZYLLM_TEMP_DIR: $LAZYLLM_TEMP_DIR" + echo "LAZYLLM_CACHE_DIR: $LAZYLLM_CACHE_DIR" + echo "LAZYLLM_CACHE_MODE: $LAZYLLM_CACHE_MODE" + echo "LAZYLLM_CACHE_STRATEGY: $LAZYLLM_CACHE_STRATEGY" + echo "LAZYLLM_CACHE_ONLINE_MODULE: $LAZYLLM_CACHE_ONLINE_MODULE" + + - name: Get SHAs + run: | + cd ${{ env.CI_PATH }} + CURRENT_SHA=$(git rev-parse HEAD) + if git rev-parse HEAD^ >/dev/null 2>&1; then + PREVIOUS_SHA=$(git rev-parse HEAD^) + else + PREVIOUS_SHA=$CURRENT_SHA + fi + echo "CURRENT_SHA=$CURRENT_SHA" >> $GITHUB_ENV + echo "PREVIOUS_SHA=$PREVIOUS_SHA" >> $GITHUB_ENV + if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + BASE_SHA="${{ github.event.pull_request.base.sha }}" + echo "BASE_SHA=$BASE_SHA" >> $GITHUB_ENV + fi + + - name: Restore Cache (Push) + if: ${{ github.event_name == 'push' }} + uses: lwj-st/cache/restore@v4.2.4 + with: + path: github_cache/ + key: python-result-cache-${{ env.PREVIOUS_SHA }} + enableCrossOsArchive: true + ci_path: ${{ env.CI_PATH }} + + - name: Restore Cache (Pull Request) + if: ${{ github.event_name == 'pull_request' }} + uses: lwj-st/cache/restore@v4.2.4 + with: + path: github_cache/ + key: python-result-cache-${{ env.BASE_SHA }} + enableCrossOsArchive: true + ci_path: ${{ env.CI_PATH }} + - name: RunTests + id: exe_ci run: | cd ${{ env.CI_PATH }} pip install -r tests/requirements.txt @@ -168,13 +241,24 @@ jobs: export LAZYLLM_HOME="${{ env.CI_PATH }}/${{ github.run_id }}-${{ github.job }}" mkdir -p $LAZYLLM_HOME source ~/ENV/env.sh + find ./github_cache/ -print | sed -e 's;[^/]*/;|____;g;s;____|; |;g' if [ -f tests/charge_tests/.pytest-cache/v/cache/lastfailed ]; then python -m pytest --lf --last-failed-no-failures=none -m "not skip_on_linux" --durations=0 --reruns=2 -v --cov=lazyllm --cov-append --cov-report=html tests/charge_tests else python -m pytest -m "not skip_on_linux" --durations=0 --reruns=2 -v --cov=lazyllm --cov-append --cov-report=html tests/charge_tests fi + - name: Success On Main Branch + if: ${{ success() && steps.exe_ci.conclusion == 'success' && github.event_name == 'push' }} + uses: lwj-st/cache/save@v4.2.4 + with: + path: github_cache/ + key: python-result-cache-${{ env.CURRENT_SHA }} + enableCrossOsArchive: true + ci_path: ${{ env.CI_PATH }} + Coverage_linux: + if: contains(github.event.head_commit.message, 'temp_close') runs-on: ubuntu-latest needs: [ BasicTests, AdvancedStandardTests, AdvancedFullTests, ChargeTests ] steps: @@ -187,6 +271,7 @@ jobs: echo "[http://10.210.0.49:8088/${GITHUB_RUN_NUMBER}/htmlcov](http://10.210.0.49:8088/${GITHUB_RUN_NUMBER}/htmlcov)" >> $GITHUB_STEP_SUMMARY DocCheck: + if: contains(github.event.head_commit.message, 'temp_close') runs-on: tps_sco_nv needs: [ Clone ] steps: diff --git a/.github/workflows/win_test.yml b/.github/workflows/win_test.yml index d10edf947..d0f5adce8 100644 --- a/.github/workflows/win_test.yml +++ b/.github/workflows/win_test.yml @@ -193,6 +193,59 @@ jobs: git submodule init LazyLLM-Env git submodule update LazyLLM-Env + - name: Create Output and Cache Directories + shell: bash + run: | + mkdir -p github_cache + mkdir -p github_cache/output + mkdir -p github_cache/paid_llm_cache + echo "LAZYLLM_TEMP_DIR=github_cache/output" >> $GITHUB_ENV + echo "LAZYLLM_CACHE_DIR=github_cache/paid_llm_cache" >> $GITHUB_ENV + + - name: Set Cache Mode Based on Run Context + shell: bash + run: | + echo "GITHUB_RUN_ATTEMPT=$GITHUB_RUN_ATTEMPT" + echo "LAZYLLM_CACHE_ONLINE_MODULE=True" >> $GITHUB_ENV + echo "LAZYLLM_CACHE_STRATEGY=sqlite" >> $GITHUB_ENV + echo "LAZYLLM_CACHE_MODE=RO" >> $GITHUB_ENV + + - name: Show ENV + shell: bash + run: | + echo "LAZYLLM_TEMP_DIR: $LAZYLLM_TEMP_DIR" + echo "LAZYLLM_CACHE_DIR: $LAZYLLM_CACHE_DIR" + echo "LAZYLLM_CACHE_MODE: $LAZYLLM_CACHE_MODE" + echo "LAZYLLM_CACHE_STRATEGY: $LAZYLLM_CACHE_STRATEGY" + echo "LAZYLLM_CACHE_ONLINE_MODULE: $LAZYLLM_CACHE_ONLINE_MODULE" + + - name: Get SHAs + shell: bash + run: | + CURRENT_SHA=$(git rev-parse HEAD) + PREVIOUS_SHA=$(git rev-parse HEAD^ 2>/dev/null || echo $CURRENT_SHA) + echo "PREVIOUS_SHA=$PREVIOUS_SHA" >> $GITHUB_ENV + if [ "$GITHUB_EVENT_NAME" = "pull_request_target" ]; then + BASE_SHA="${{ github.event.pull_request.base.sha }}" + echo "BASE_SHA=$BASE_SHA" >> $GITHUB_ENV + fi + + - name: Restore Cache (Push) + if: ${{ github.event_name == 'push' }} + uses: lwj-st/cache/restore@v4.2.4 + with: + path: github_cache/ + key: python-result-cache-${{ env.PREVIOUS_SHA }} + enableCrossOsArchive: true + + - name: Restore Cache (Pull Request) + if: ${{ github.event_name == 'pull_request_target' }} + uses: lwj-st/cache/restore@v4.2.4 + with: + path: github_cache/ + key: python-result-cache-${{ env.BASE_SHA }} + enableCrossOsArchive: true + - name: Merge PR branch into main shell: bash if: github.event_name == 'pull_request_target' diff --git a/lazyllm/components/formatter/formatterbase.py b/lazyllm/components/formatter/formatterbase.py index 97b0bd6af..afa5772ec 100644 --- a/lazyllm/components/formatter/formatterbase.py +++ b/lazyllm/components/formatter/formatterbase.py @@ -1,7 +1,10 @@ +import os import json import uuid +import hashlib from typing import Callable, Optional, List, Union, Any +import lazyllm from lazyllm.flow.flow import Pipeline from ...common import LazyLLMRegisterMetaClass, package, Finalizer @@ -229,3 +232,102 @@ def _decode_one_data(self, py_data): return encode_query_with_filepaths(**py_data) else: return py_data + + +class FileContentHash(str): + + def __hash__(self): + res = decode_query_with_filepaths(self) + if isinstance(res, str): + return hashlib.md5(self.encode()).hexdigest() + query = res['query'] + file_path_list = res['files'] + + hash_obj = hashlib.md5() + hash_obj.update(query.encode('utf-8')) + + search_paths = [ + '', + lazyllm.config['temp_dir'], + ] + for file_path in file_path_list: + if os.path.isabs(file_path): + full_path = file_path + else: + full_path = None + for base_path in search_paths: + candidate_path = os.path.join(base_path, file_path) if base_path else file_path + if os.path.exists(candidate_path) and os.path.isfile(candidate_path): + full_path = candidate_path + break + if full_path is None: + full_path = file_path + try: + with open(full_path, 'rb') as f: + while chunk := f.read(8192): + hash_obj.update(chunk) + except (FileNotFoundError, IOError): + lazyllm.LOG.debug(f'Error: File not found or cannot be read: {full_path}') + hash_obj.update(full_path.encode('utf-8')) + return int(hash_obj.hexdigest(), 16) + + +def proccess_path_recursively(value, process_func): + if isinstance(value, str): + if LAZYLLM_QUERY_PREFIX in value: + res = decode_query_with_filepaths(value) + if res['files']: + replace_path = [] + for file_path in res['files']: + replace_path.append(process_func(file_path)) + res['files'] = replace_path + return encode_query_with_filepaths(res['query'], res['files']) + else: + return value + elif isinstance(value, (list, tuple)): + process_items = [] + for item in value: + process_items.append(proccess_path_recursively(item, process_func)) + if isinstance(value, tuple): + return tuple(process_items) + return process_items + elif isinstance(value, dict): + process_dict = {} + for key, val in value.items(): + process_dict[key] = proccess_path_recursively(val, process_func) + return process_dict + elif isinstance(value, set): + process_set = set() + for item in value: + process_set.add(proccess_path_recursively(item, process_func)) + return process_set + else: + return value + +def path_relative_to_absolute(path): + if os.path.isabs(path): + return path + absolute_path = os.path.join(lazyllm.config['temp_dir'], path) + if os.path.exists(absolute_path): + return os.path.abspath(absolute_path) + else: + return path + +def path_absolute_to_relative(path): + if not os.path.isabs(path): + return path + temp_dir_abs = os.path.abspath(lazyllm.config['temp_dir']) + if path.startswith(temp_dir_abs): + relative_path = path[len(temp_dir_abs):] + if relative_path.startswith(os.sep): + relative_path = relative_path[1:] + return relative_path + else: + return path + +def transform_path(value, mode='r2a'): + assert mode in ('r2a', 'a2r') + if mode == 'r2a': + return proccess_path_recursively(value, path_relative_to_absolute) + else: + return proccess_path_recursively(value, path_absolute_to_relative) diff --git a/lazyllm/components/utils/file_operate.py b/lazyllm/components/utils/file_operate.py index 5233c0b2c..4e3fbed4b 100644 --- a/lazyllm/components/utils/file_operate.py +++ b/lazyllm/components/utils/file_operate.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Optional, Tuple, Union -from lazyllm import LOG +from lazyllm import LOG, config try: import magic @@ -157,7 +157,8 @@ def _base64_to_file(base64_str: Union[str, list[str]], target_dir: Optional[str] else: raise ValueError(f'Unsupported MIME type: {mime_type}') - target_dir = target_dir if target_dir and os.path.isdir(target_dir) else None + target_dir = target_dir if target_dir and os.path.isdir(target_dir) else config['temp_dir'] + os.makedirs(target_dir, exist_ok=True) file_path = tempfile.NamedTemporaryFile(prefix='base64_to_file_', suffix=suffix, dir=target_dir, delete=False).name @@ -194,7 +195,8 @@ def bytes_to_file(bytes_str: Union[bytes, list[bytes]], target_dir: Optional[str if isinstance(bytes_str, list): return [bytes_to_file(item, target_dir) for item in bytes_str] elif isinstance(bytes_str, bytes): - output_dir = target_dir if target_dir and os.path.isdir(target_dir) else None + output_dir = target_dir if target_dir and os.path.isdir(target_dir) else config['temp_dir'] + os.makedirs(output_dir, exist_ok=True) file_extension = infer_file_extension(bytes_str) temp_file = tempfile.NamedTemporaryFile(mode='wb', suffix=file_extension, delete=False, dir=output_dir) temp_file.write(bytes_str) diff --git a/lazyllm/module/module.py b/lazyllm/module/module.py index 2fe4c53b4..4ef270e1e 100644 --- a/lazyllm/module/module.py +++ b/lazyllm/module/module.py @@ -5,6 +5,7 @@ import lazyllm from lazyllm import FlatList, Option, kwargs, globals, colored_text, redis_client +from ..components.formatter.formatterbase import FileContentHash, transform_path from ..flow import FlowBase, Pipeline, Parallel from ..common.bind import _MetaBind import uuid @@ -198,18 +199,37 @@ def _create_strategy(self, strategy: str) -> _CacheStorageStrategy: return strategies[strategy]() def _hash(self, args, kw): - content = str(args) + str(sorted(kw.items()) if kw else '') - return hashlib.md5(content.encode()).hexdigest() + def process_value(value, hash_obj): + if isinstance(value, str): + value = FileContentHash(value) + if isinstance(value, FileContentHash): + hash_obj.update(str(value.__hash__()).encode()) + elif isinstance(value, (list, tuple)): + for item in value: + process_value(item, hash_obj) + elif isinstance(value, dict): + for k, v in sorted(value.items()): + hash_obj.update(str(k).encode()) + process_value(v, hash_obj) + else: + hash_obj.update(str(value).encode()) + hash_obj = hashlib.md5() + process_value(args, hash_obj) + if kw: + process_value(kw, hash_obj) + return hash_obj.hexdigest() def get(self, key, args, kw): if 'R' not in lazyllm.config['cache_mode']: raise CacheNotFoundError('Cannot read cache due to `LAZYLLM_CACHE_MODE = WO`') hash_key = self._hash(args, kw) - return self._strategy.get(key, hash_key) + value = self._strategy.get(key, hash_key) + return transform_path(value, mode='r2a') def set(self, key, args, kw, value): if 'W' not in lazyllm.config['cache_mode']: return hash_key = self._hash(args, kw) + value = transform_path(value, mode='a2r') self._strategy.set(key, hash_key, value) def close(self): @@ -472,6 +492,7 @@ def for_each(self, filter, action): def __cache_hash__(self): cache_hash = self.__class__.__name__ if isinstance(self._use_cache, str): cache_hash += f'@{self._use_cache}' + if hasattr(self, 'appendix_hash_key'): cache_hash += f'@{self.appendix_hash_key}' return cache_hash def use_cache(self, flag: Union[bool, str] = True): diff --git a/lazyllm/module/servermodule.py b/lazyllm/module/servermodule.py index 5c4880bf7..50ea59ca2 100644 --- a/lazyllm/module/servermodule.py +++ b/lazyllm/module/servermodule.py @@ -1,5 +1,7 @@ import re import time +import json +import hashlib import requests import pickle import codecs @@ -85,6 +87,20 @@ def __or__(self, other): return NotImplemented return self.share(format=(other if isinstance(self._formatter, EmptyFormatter) else (self._formatter | other))) + @property + def appendix_hash_key(self): + try: + prompts = self._prompt.generate_prompt('x') + except Exception: + prompts = self._prompt._instruction_template + if not isinstance(prompts, str): + try: + content = json.dumps(prompts, sort_keys=True) + except Exception: + content = str(prompts) + else: + content = prompts + return hashlib.md5(content.encode()).hexdigest() class _UrlHelper(object): @dataclass diff --git a/tests/charge_tests/test_engine.py b/tests/charge_tests/test_engine.py index 338eb93d7..7dabb6187 100644 --- a/tests/charge_tests/test_engine.py +++ b/tests/charge_tests/test_engine.py @@ -1,4 +1,6 @@ +import os import lazyllm +from lazyllm import config from lazyllm.engine import LightEngine, NodeMetaHook import pytest from .utils import SqlEgsData, get_db_init_keywords @@ -260,8 +262,9 @@ def test_stream_and_hostory(self): elif future.done(): break result = future.result() - assert '一天' in stream_result and '小时' in stream_result - assert '您好，我的答案是' in stream_result and '24' in stream_result + if not config['cache_online_module']: + assert '一天' in stream_result and '小时' in stream_result + assert '您好，我的答案是' in stream_result and '24' in stream_result assert ('蓝鲸' in result or '动物' in result) and '水' in result def test_tools_with_llm(self): @@ -301,7 +304,8 @@ def test_online_multi_module(self): nodes = [ dict(id="1", kind="TTS", name="m1", args=dict(source='qwen', type='online')), dict(id="2", kind="STT", name="m2", args=dict(source='glm', type='online')), - dict(id="3", kind="SD", name="m3", args=dict(source='qwen', type='online', target_dir='./test_online_mm')), + dict(id="3", kind="SD", name="m3", args=dict(source='qwen', type='online', + target_dir=os.path.join(lazyllm.config['temp_dir'], 'output'))), dict(id="4", kind="VQA", name="m4", args=dict(source='qwen', base_model='qwen-vl-plus', type='online')) ] engine = LightEngine() diff --git a/tests/charge_tests/test_example.py b/tests/charge_tests/test_example.py index d310c8fa3..4cabdce6a 100644 --- a/tests/charge_tests/test_example.py +++ b/tests/charge_tests/test_example.py @@ -6,6 +6,7 @@ from gradio_client import Client import lazyllm +from lazyllm import config from lazyllm.launcher import cleanup from lazyllm.components.formatter import encode_query_with_filepaths from lazyllm.components.prompter import ChatPrompter @@ -137,10 +138,11 @@ def test_rag(self): assert len(res) >= 16 # test pipeline wrapped into iterator - from lazyllm.tools.common import StreamCallHelper - flow_iterator = StreamCallHelper(ppl, interval=0.01) - res_list = list(flow_iterator(query)) - assert isinstance(res_list, list) and len(res_list) > 1 + if not config['cache_online_module']: + from lazyllm.tools.common import StreamCallHelper + flow_iterator = StreamCallHelper(ppl, interval=0.01) + res_list = list(flow_iterator(query)) + assert isinstance(res_list, list) and len(res_list) > 1 # test rag warpped in web web, client = self.warp_into_web(rag) diff --git a/tests/charge_tests/test_multimodal.py b/tests/charge_tests/test_multimodal.py index 2ea72c393..619dff50a 100644 --- a/tests/charge_tests/test_multimodal.py +++ b/tests/charge_tests/test_multimodal.py @@ -1,6 +1,7 @@ import os import lazyllm import pytest +from lazyllm import config from lazyllm.components.formatter import decode_query_with_filepaths @@ -21,6 +22,7 @@ def test_online_tts(self): result = tts(self.test_text) self._check_file_result(result, format='audio') + if config['cache_online_module']: return with pytest.raises(RuntimeError, match='cosyvoice-v1 does not support multi user, don\'t set api_key'): tts = lazyllm.OnlineMultiModalModule(source='qwen', function='tts', model='cosyvoice-v1', api_key=api_key) result = tts(self.test_text)