Test PR

.github/workflows/macOS_test.yml

@@ -180,6 +180,59 @@ jobs:
           git submodule init LazyLLM-Env
           git submodule update LazyLLM-Env
 
+      - name: Create Output and Cache Directories
+        run: |
+          mkdir -p github_cache
+          mkdir -p github_cache/output
+          mkdir -p github_cache/paid_llm_cache
+          echo "LAZYLLM_TEMP_DIR=github_cache/output" >> $GITHUB_ENV
+          echo "LAZYLLM_CACHE_DIR=github_cache/paid_llm_cache" >> $GITHUB_ENV
+
+      - name: Set Cache Mode Based on Run Context
+        run: |
+          echo "GITHUB_RUN_ATTEMPT=$GITHUB_RUN_ATTEMPT"
+          echo "LAZYLLM_CACHE_ONLINE_MODULE=True" >> $GITHUB_ENV
+          echo "LAZYLLM_CACHE_STRATEGY=sqlite" >> $GITHUB_ENV
+          echo "LAZYLLM_CACHE_MODE=RO" >> $GITHUB_ENV
+
+      - name: Show ENV
+        run: |
+          echo "LAZYLLM_TEMP_DIR: $LAZYLLM_TEMP_DIR"
+          echo "LAZYLLM_CACHE_DIR: $LAZYLLM_CACHE_DIR"
+          echo "LAZYLLM_CACHE_MODE: $LAZYLLM_CACHE_MODE"
+          echo "LAZYLLM_CACHE_STRATEGY: $LAZYLLM_CACHE_STRATEGY"
+          echo "LAZYLLM_CACHE_ONLINE_MODULE: $LAZYLLM_CACHE_ONLINE_MODULE"
+
+      - name: Get SHAs
+        run: |
+          CURRENT_SHA=$(git rev-parse HEAD)
+          if git rev-parse HEAD^ >/dev/null 2>&1; then
+            PREVIOUS_SHA=$(git rev-parse HEAD^)
+          else
+            PREVIOUS_SHA=$CURRENT_SHA
+          fi
+          echo "PREVIOUS_SHA=$PREVIOUS_SHA" >> $GITHUB_ENV
+          if [ "$GITHUB_EVENT_NAME" = "pull_request_target" ]; then
+            BASE_SHA="${{ github.event.pull_request.base.sha }}"
+            echo "BASE_SHA=$BASE_SHA" >> $GITHUB_ENV
+          fi
+
+      - name: Restore Cache (Push)
+        if: ${{ github.event_name == 'push' }}
+        uses: lwj-st/cache/[email protected]
+        with:
+          path: github_cache/
+          key: python-result-cache-${{ env.PREVIOUS_SHA }}
+          enableCrossOsArchive: true
+
+      - name: Restore Cache (Pull Request)
+        if: ${{ github.event_name == 'pull_request_target' }}
+        uses: lwj-st/cache/[email protected]
+        with:
+          path: github_cache/
+          key: python-result-cache-${{ env.BASE_SHA }}
+          enableCrossOsArchive: true
+
       - name: Merge PR branch into main
         if: github.event_name == 'pull_request_target'
         run: |

.github/workflows/main.yml

@@ -18,7 +18,7 @@ on:
       - "docs/assets/**"
       - "docs/**"
 env:
-  CI_PATH: '/home/mnt/platform_ci/GitHub/${{ github.repository }}/${GITHUB_RUN_NUMBER}'
+  CI_PATH: '/home/mnt/platform_ci/GitHub/${{ github.repository }}/${{ github.run_number }}'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -64,6 +64,8 @@ jobs:
 
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Build doc
         run: |
@@ -88,9 +90,10 @@ jobs:
       - name: Move code to custom directory
         run: |
           set -ex
-          mv $GITHUB_WORKSPACE/* ${{ env.CI_PATH }}/
+          cp -r $GITHUB_WORKSPACE/. ${{ env.CI_PATH }}/
 
   BasicTests:
+    if: contains(github.event.head_commit.message, 'temp_close')
     runs-on: tps_sco_nv
     needs: [Clone]
     steps:
@@ -111,6 +114,7 @@ jobs:
           python -m pytest --lf --last-failed-no-failures=all -m "not skip_on_linux" --durations=0 --reruns=2 -v --cov=lazyllm --cov-append --cov-report=html tests/basic_tests/
 
   AdvancedStandardTests:
+    if: contains(github.event.head_commit.message, 'temp_close')
     runs-on: tps_sco_nv
     needs: [Clone]
     steps:
@@ -134,6 +138,7 @@ jobs:
           fi
 
   AdvancedFullTests:
+    if: contains(github.event.head_commit.message, 'temp_close')
     runs-on: tps_sco_nv
     needs: [Clone]
     steps:
@@ -158,7 +163,75 @@ jobs:
     runs-on: tps_sco_nv
     needs: [ Clone ]
     steps:
+      - name: Create Output and Cache Directories
+        run: |
+          cd ${{ env.CI_PATH }}
+          mkdir -p github_cache
+          mkdir -p github_cache/output
+          mkdir -p github_cache/paid_llm_cache
+          echo "LAZYLLM_TEMP_DIR=github_cache/output" >> $GITHUB_ENV
+          echo "LAZYLLM_CACHE_DIR=github_cache/paid_llm_cache" >> $GITHUB_ENV
+
+      - name: Set Cache Mode Based on Run Context
+        run: |
+          echo "GITHUB_RUN_ATTEMPT=$GITHUB_RUN_ATTEMPT"
+          echo "LAZYLLM_CACHE_ONLINE_MODULE=True" >> $GITHUB_ENV
+          echo "LAZYLLM_CACHE_STRATEGY=sqlite" >> $GITHUB_ENV
+          if [ "$GITHUB_RUN_ATTEMPT" = "1" ] && [ "$GITHUB_EVENT_NAME" = "push" ]; then
+            echo "This is the first run, and push to main branch."
+            echo "LAZYLLM_CACHE_MODE=RW" >> $GITHUB_ENV
+          elif [ "$GITHUB_RUN_ATTEMPT" != "1" ] && [ "$GITHUB_EVENT_NAME" = "push" ]; then
+            echo "This is a re-run! (GITHUB_RUN_ATTEMPT=$GITHUB_RUN_ATTEMPT), and push to main branch."
+            echo "LAZYLLM_CACHE_MODE=WO" >> $GITHUB_ENV
+          elif [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+            echo "This is a pull request from other branch to main."
+            echo "LAZYLLM_CACHE_MODE=RO" >> $GITHUB_ENV
+          fi
+
+      - name: Show ENV
+        run: |
+          echo "LAZYLLM_TEMP_DIR: $LAZYLLM_TEMP_DIR"
+          echo "LAZYLLM_CACHE_DIR: $LAZYLLM_CACHE_DIR"
+          echo "LAZYLLM_CACHE_MODE: $LAZYLLM_CACHE_MODE"
+          echo "LAZYLLM_CACHE_STRATEGY: $LAZYLLM_CACHE_STRATEGY"
+          echo "LAZYLLM_CACHE_ONLINE_MODULE: $LAZYLLM_CACHE_ONLINE_MODULE"
+
+      - name: Get SHAs
+        run: |
+          cd ${{ env.CI_PATH }}
+          CURRENT_SHA=$(git rev-parse HEAD)
+          if git rev-parse HEAD^ >/dev/null 2>&1; then
+            PREVIOUS_SHA=$(git rev-parse HEAD^)
+          else
+            PREVIOUS_SHA=$CURRENT_SHA
+          fi
+          echo "CURRENT_SHA=$CURRENT_SHA" >> $GITHUB_ENV
+          echo "PREVIOUS_SHA=$PREVIOUS_SHA" >> $GITHUB_ENV
+          if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+            BASE_SHA="${{ github.event.pull_request.base.sha }}"
+            echo "BASE_SHA=$BASE_SHA" >> $GITHUB_ENV
+          fi
+
+      - name: Restore Cache (Push)
+        if: ${{ github.event_name == 'push' }}
+        uses: lwj-st/cache/[email protected]
+        with:
+          path: github_cache/
+          key: python-result-cache-${{ env.PREVIOUS_SHA }}
+          enableCrossOsArchive: true
+          ci_path: ${{ env.CI_PATH }}
+
+      - name: Restore Cache (Pull Request)
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: lwj-st/cache/[email protected]
+        with:
+          path: github_cache/
+          key: python-result-cache-${{ env.BASE_SHA }}
+          enableCrossOsArchive: true
+          ci_path: ${{ env.CI_PATH }}
+
       - name: RunTests
+        id: exe_ci
         run: |
           cd ${{ env.CI_PATH }}
           pip install -r tests/requirements.txt
@@ -168,13 +241,24 @@ jobs:
           export LAZYLLM_HOME="${{ env.CI_PATH }}/${{ github.run_id }}-${{ github.job }}"
           mkdir -p $LAZYLLM_HOME
           source ~/ENV/env.sh
+          find ./github_cache/ -print | sed -e 's;[^/]*/;|____;g;s;____|; |;g'
           if [ -f tests/charge_tests/.pytest-cache/v/cache/lastfailed ]; then
             python -m pytest --lf --last-failed-no-failures=none -m "not skip_on_linux" --durations=0 --reruns=2 -v --cov=lazyllm --cov-append --cov-report=html tests/charge_tests
           else
             python -m pytest -m "not skip_on_linux" --durations=0 --reruns=2 -v --cov=lazyllm --cov-append --cov-report=html tests/charge_tests
           fi
 
+      - name: Success On Main Branch
+        if: ${{ success() && steps.exe_ci.conclusion == 'success' && github.event_name == 'push' }}
+        uses: lwj-st/cache/[email protected]
+        with:
+          path: github_cache/
+          key: python-result-cache-${{ env.CURRENT_SHA }}
+          enableCrossOsArchive: true
+          ci_path: ${{ env.CI_PATH }}
+
   Coverage_linux:
+    if: contains(github.event.head_commit.message, 'temp_close')
     runs-on: ubuntu-latest
     needs: [ BasicTests, AdvancedStandardTests, AdvancedFullTests, ChargeTests ]
     steps:
@@ -187,6 +271,7 @@ jobs:
           echo "[http://10.210.0.49:8088/${GITHUB_RUN_NUMBER}/htmlcov](http://10.210.0.49:8088/${GITHUB_RUN_NUMBER}/htmlcov)" >> $GITHUB_STEP_SUMMARY
 
   DocCheck:
+    if: contains(github.event.head_commit.message, 'temp_close')
     runs-on: tps_sco_nv
     needs: [ Clone ]
     steps:

.github/workflows/win_test.yml

@@ -193,6 +193,59 @@ jobs:
           git submodule init LazyLLM-Env
           git submodule update LazyLLM-Env
 
+      - name: Create Output and Cache Directories
+        shell: bash
+        run: |
+          mkdir -p github_cache
+          mkdir -p github_cache/output
+          mkdir -p github_cache/paid_llm_cache
+          echo "LAZYLLM_TEMP_DIR=github_cache/output" >> $GITHUB_ENV
+          echo "LAZYLLM_CACHE_DIR=github_cache/paid_llm_cache" >> $GITHUB_ENV
+
+      - name: Set Cache Mode Based on Run Context
+        shell: bash
+        run: |
+          echo "GITHUB_RUN_ATTEMPT=$GITHUB_RUN_ATTEMPT"
+          echo "LAZYLLM_CACHE_ONLINE_MODULE=True" >> $GITHUB_ENV
+          echo "LAZYLLM_CACHE_STRATEGY=sqlite" >> $GITHUB_ENV
+          echo "LAZYLLM_CACHE_MODE=RO" >> $GITHUB_ENV
+
+      - name: Show ENV
+        shell: bash
+        run: |
+          echo "LAZYLLM_TEMP_DIR: $LAZYLLM_TEMP_DIR"
+          echo "LAZYLLM_CACHE_DIR: $LAZYLLM_CACHE_DIR"
+          echo "LAZYLLM_CACHE_MODE: $LAZYLLM_CACHE_MODE"
+          echo "LAZYLLM_CACHE_STRATEGY: $LAZYLLM_CACHE_STRATEGY"
+          echo "LAZYLLM_CACHE_ONLINE_MODULE: $LAZYLLM_CACHE_ONLINE_MODULE"
+
+      - name: Get SHAs
+        shell: bash
+        run: |
+          CURRENT_SHA=$(git rev-parse HEAD)
+          PREVIOUS_SHA=$(git rev-parse HEAD^ 2>/dev/null || echo $CURRENT_SHA)
+          echo "PREVIOUS_SHA=$PREVIOUS_SHA" >> $GITHUB_ENV
+          if [ "$GITHUB_EVENT_NAME" = "pull_request_target" ]; then
+            BASE_SHA="${{ github.event.pull_request.base.sha }}"
+            echo "BASE_SHA=$BASE_SHA" >> $GITHUB_ENV
+          fi
+
+      - name: Restore Cache (Push)
+        if: ${{ github.event_name == 'push' }}
+        uses: lwj-st/cache/[email protected]
+        with:
+          path: github_cache/
+          key: python-result-cache-${{ env.PREVIOUS_SHA }}
+          enableCrossOsArchive: true
+
+      - name: Restore Cache (Pull Request)
+        if: ${{ github.event_name == 'pull_request_target' }}
+        uses: lwj-st/cache/[email protected]
+        with:
+          path: github_cache/
+          key: python-result-cache-${{ env.BASE_SHA }}
+          enableCrossOsArchive: true
+
       - name: Merge PR branch into main
         shell: bash
         if: github.event_name == 'pull_request_target'

lazyllm/components/formatter/formatterbase.py

@@ -1,7 +1,10 @@
+import os
 import json
 import uuid
+import hashlib
 from typing import Callable, Optional, List, Union, Any
 
+import lazyllm
 from lazyllm.flow.flow import Pipeline
 
 from ...common import LazyLLMRegisterMetaClass, package, Finalizer
@@ -229,3 +232,102 @@ def _decode_one_data(self, py_data):
                 return encode_query_with_filepaths(**py_data)
             else:
                 return py_data
+
+
+class FileContentHash(str):
+
+    def __hash__(self):
+        res = decode_query_with_filepaths(self)
+        if isinstance(res, str):
+            return hashlib.md5(self.encode()).hexdigest()
+        query = res['query']
+        file_path_list = res['files']
+
+        hash_obj = hashlib.md5()
+        hash_obj.update(query.encode('utf-8'))
+
+        search_paths = [
+            '',
+            lazyllm.config['temp_dir'],
+        ]
+        for file_path in file_path_list:
+            if os.path.isabs(file_path):
+                full_path = file_path
+            else:
+                full_path = None
+                for base_path in search_paths:
+                    candidate_path = os.path.join(base_path, file_path) if base_path else file_path
+                    if os.path.exists(candidate_path) and os.path.isfile(candidate_path):
+                        full_path = candidate_path
+                        break
+                if full_path is None:
+                    full_path = file_path
+            try:
+                with open(full_path, 'rb') as f:
+                    while chunk := f.read(8192):
+                        hash_obj.update(chunk)
+            except (FileNotFoundError, IOError):
+                lazyllm.LOG.debug(f'Error: File not found or cannot be read: {full_path}')
+                hash_obj.update(full_path.encode('utf-8'))
+        return int(hash_obj.hexdigest(), 16)
+
+
+def proccess_path_recursively(value, process_func):
+    if isinstance(value, str):
+        if LAZYLLM_QUERY_PREFIX in value:
+            res = decode_query_with_filepaths(value)
+            if res['files']:
+                replace_path = []
+                for file_path in res['files']:
+                    replace_path.append(process_func(file_path))
+                res['files'] = replace_path
+            return encode_query_with_filepaths(res['query'], res['files'])
+        else:
+            return value
+    elif isinstance(value, (list, tuple)):
+        process_items = []
+        for item in value:
+            process_items.append(proccess_path_recursively(item, process_func))
+        if isinstance(value, tuple):
+            return tuple(process_items)
+        return process_items
+    elif isinstance(value, dict):
+        process_dict = {}
+        for key, val in value.items():
+            process_dict[key] = proccess_path_recursively(val, process_func)
+        return process_dict
+    elif isinstance(value, set):
+        process_set = set()
+        for item in value:
+            process_set.add(proccess_path_recursively(item, process_func))
+        return process_set
+    else:
+        return value
+
+def path_relative_to_absolute(path):
+    if os.path.isabs(path):
+        return path
+    absolute_path = os.path.join(lazyllm.config['temp_dir'], path)
+    if os.path.exists(absolute_path):
+        return os.path.abspath(absolute_path)
+    else:
+        return path
+
+def path_absolute_to_relative(path):
+    if not os.path.isabs(path):
+        return path
+    temp_dir_abs = os.path.abspath(lazyllm.config['temp_dir'])
+    if path.startswith(temp_dir_abs):
+        relative_path = path[len(temp_dir_abs):]
+        if relative_path.startswith(os.sep):
+            relative_path = relative_path[1:]
+        return relative_path
+    else:
+        return path
+
+def transform_path(value, mode='r2a'):
+    assert mode in ('r2a', 'a2r')
+    if mode == 'r2a':
+        return proccess_path_recursively(value, path_relative_to_absolute)
+    else:
+        return proccess_path_recursively(value, path_absolute_to_relative)