From 95159ad4bc5cdb923e0d63eca97eb5ace2c323b6 Mon Sep 17 00:00:00 2001 From: hzjane Date: Tue, 26 Nov 2024 14:36:37 +0800 Subject: [PATCH 1/7] build image to test prefix caching --- docker/llm/serving/xpu/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/llm/serving/xpu/docker/Dockerfile b/docker/llm/serving/xpu/docker/Dockerfile index 0b1e7267047..267430a99a6 100644 --- a/docker/llm/serving/xpu/docker/Dockerfile +++ b/docker/llm/serving/xpu/docker/Dockerfile @@ -85,7 +85,7 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO rm -rf /tmp/neo && \ mkdir -p /llm && \ cd /llm && \ - git clone -b 0.6.2 https://github.com/analytics-zoo/vllm.git /llm/vllm && \ + git clone -b add_prefix_caching https://github.com/hzjane/vllm.git /llm/vllm && \ cd /llm/vllm && \ pip install setuptools-scm && \ pip install --upgrade cmake && \ From ac2335472b0752d409ec90dc45c4a8525a406659 Mon Sep 17 00:00:00 2001 From: hzjane Date: Wed, 18 Dec 2024 16:08:55 +0800 Subject: [PATCH 2/7] test new --- docker/llm/serving/xpu/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/llm/serving/xpu/docker/Dockerfile b/docker/llm/serving/xpu/docker/Dockerfile index 267430a99a6..aa234505376 100644 --- a/docker/llm/serving/xpu/docker/Dockerfile +++ b/docker/llm/serving/xpu/docker/Dockerfile @@ -85,7 +85,7 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO rm -rf /tmp/neo && \ mkdir -p /llm && \ cd /llm && \ - git clone -b add_prefix_caching https://github.com/hzjane/vllm.git /llm/vllm && \ + git clone -b add_logits_attn_softcap https://github.com/hzjane/vllm.git /llm/vllm && \ cd /llm/vllm && \ pip install setuptools-scm && \ pip install --upgrade cmake && \ From ba52ae2a92edd122934a7b5760e52a8fee804def Mon Sep 17 00:00:00 2001 From: hzjane Date: Tue, 24 Dec 2024 14:21:33 +0800 Subject: [PATCH 3/7] test torch 2.5 --- docker/llm/serving/xpu/docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/llm/serving/xpu/docker/Dockerfile b/docker/llm/serving/xpu/docker/Dockerfile index aa234505376..4d3482f668a 100644 --- a/docker/llm/serving/xpu/docker/Dockerfile +++ b/docker/llm/serving/xpu/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM intel/oneapi-basekit:2024.1.1-devel-ubuntu22.04 +FROM intel/oneapi-basekit:2025.0.1-0-devel-ubuntu22.04 ARG http_proxy ARG https_proxy @@ -62,7 +62,7 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO rm -rf ./ipex-llm && \ # Install torch-ccl cd /tmp/ && \ - pip install torch==2.1.0.post2 torchvision==0.16.0.post2 torchaudio==2.1.0.post2 intel-extension-for-pytorch==2.1.30.post0 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/ && \ + pip install torch==2.5.1+cxx11.abi torchvision==0.20.1+cxx11.abi torchaudio==2.5.1+cxx11.abi intel-extension-for-pytorch==2.5.10+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/ && \ # Internal oneccl wget https://sourceforge.net/projects/oneccl-wks/files/2024.0.0.6.2-release/oneccl_wks_installer_2024.0.0.6.2.sh && \ bash oneccl_wks_installer_2024.0.0.6.2.sh && \ From 0ae5dca79865c285e7c7c671850541f7f1693e34 Mon Sep 17 00:00:00 2001 From: hzjane Date: Tue, 24 Dec 2024 14:34:58 +0800 Subject: [PATCH 4/7] test inference --- docker/llm/inference/xpu/docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/llm/inference/xpu/docker/Dockerfile b/docker/llm/inference/xpu/docker/Dockerfile index b96f9203f90..1215c4153e3 100644 --- a/docker/llm/inference/xpu/docker/Dockerfile +++ b/docker/llm/inference/xpu/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM intel/oneapi:2024.2.1-0-devel-ubuntu22.04 +FROM intel/oneapi-basekit:2025.0.1-0-devel-ubuntu22.04 ARG http_proxy ARG https_proxy @@ -56,6 +56,7 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO pip install --upgrade requests argparse urllib3 && \ pip install --pre --upgrade ipex-llm[xpu_arc] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/ && \ pip install --pre pytorch-triton-xpu==3.0.0+1b2f15840e --index-url https://download.pytorch.org/whl/nightly/xpu && \ + pip install torch==2.5.1+cxx11.abi torchvision==0.20.1+cxx11.abi torchaudio==2.5.1+cxx11.abi intel-extension-for-pytorch==2.5.10+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/ && \ # Fix Trivy CVE Issues pip install transformers_stream_generator einops tiktoken && \ # Install opencl-related repos From 7c737ca24e14a0535bad655894b4362c6a0737ca Mon Sep 17 00:00:00 2001 From: hzjane Date: Wed, 25 Dec 2024 11:05:02 +0800 Subject: [PATCH 5/7] update --- docker/llm/serving/xpu/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/llm/serving/xpu/docker/Dockerfile b/docker/llm/serving/xpu/docker/Dockerfile index 4d3482f668a..e6a176159a3 100644 --- a/docker/llm/serving/xpu/docker/Dockerfile +++ b/docker/llm/serving/xpu/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM intel/oneapi-basekit:2025.0.1-0-devel-ubuntu22.04 +FROM intel/oneapi-basekit:2024.2.1-0-devel-ubuntu22.04 ARG http_proxy ARG https_proxy From 4bc9563b49fed330fc2c69803ff72394483f843c Mon Sep 17 00:00:00 2001 From: hzjane Date: Wed, 25 Dec 2024 13:39:25 +0800 Subject: [PATCH 6/7] to debug on docker --- docker/llm/serving/xpu/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/llm/serving/xpu/docker/Dockerfile b/docker/llm/serving/xpu/docker/Dockerfile index e6a176159a3..bb3d5d18e0d 100644 --- a/docker/llm/serving/xpu/docker/Dockerfile +++ b/docker/llm/serving/xpu/docker/Dockerfile @@ -62,7 +62,7 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO rm -rf ./ipex-llm && \ # Install torch-ccl cd /tmp/ && \ - pip install torch==2.5.1+cxx11.abi torchvision==0.20.1+cxx11.abi torchaudio==2.5.1+cxx11.abi intel-extension-for-pytorch==2.5.10+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/ && \ + pip install torch==2.1.0.post2 torchvision==0.16.0.post2 torchaudio==2.1.0.post2 intel-extension-for-pytorch==2.1.30.post0 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/ && \ # Internal oneccl wget https://sourceforge.net/projects/oneccl-wks/files/2024.0.0.6.2-release/oneccl_wks_installer_2024.0.0.6.2.sh && \ bash oneccl_wks_installer_2024.0.0.6.2.sh && \ From cb5dc598f12a7f56df62a6154f8814ba04a8b929 Mon Sep 17 00:00:00 2001 From: hzjane Date: Mon, 30 Dec 2024 11:18:32 +0800 Subject: [PATCH 7/7] update --- docker/llm/serving/xpu/docker/Dockerfile | 36 ++++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docker/llm/serving/xpu/docker/Dockerfile b/docker/llm/serving/xpu/docker/Dockerfile index bb3d5d18e0d..912cea006bb 100644 --- a/docker/llm/serving/xpu/docker/Dockerfile +++ b/docker/llm/serving/xpu/docker/Dockerfile @@ -73,23 +73,23 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO apt-get update && \ apt-get install -y --no-install-recommends libfabric-dev wrk libaio-dev numactl && \ # apt-get install -y intel-opencl-icd intel-level-zero-gpu=1.3.26241.33-647~22.04 level-zero level-zero-dev --allow-downgrades && \ - mkdir -p /tmp/neo && \ - cd /tmp/neo && \ - wget https://github.com/oneapi-src/level-zero/releases/download/v1.18.5/level-zero_1.18.5+u22.04_amd64.deb && \ - wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17791.9/intel-igc-core_1.0.17791.9_amd64.deb && \ - wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17791.9/intel-igc-opencl_1.0.17791.9_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-level-zero-gpu_1.6.31294.12_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-opencl-icd_24.39.31294.12_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/libigdgmm12_22.5.2_amd64.deb && \ - dpkg -i *.deb && \ - rm -rf /tmp/neo && \ - mkdir -p /llm && \ - cd /llm && \ - git clone -b add_logits_attn_softcap https://github.com/hzjane/vllm.git /llm/vllm && \ - cd /llm/vllm && \ - pip install setuptools-scm && \ - pip install --upgrade cmake && \ - VLLM_TARGET_DEVICE=xpu pip install --no-build-isolation -v /llm/vllm && \ + # mkdir -p /tmp/neo && \ + # cd /tmp/neo && \ + # wget https://github.com/oneapi-src/level-zero/releases/download/v1.18.5/level-zero_1.18.5+u22.04_amd64.deb && \ + # wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17791.9/intel-igc-core_1.0.17791.9_amd64.deb && \ + # wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17791.9/intel-igc-opencl_1.0.17791.9_amd64.deb && \ + # wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-level-zero-gpu_1.6.31294.12_amd64.deb && \ + # wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-opencl-icd_24.39.31294.12_amd64.deb && \ + # wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/libigdgmm12_22.5.2_amd64.deb && \ + # dpkg -i *.deb && \ + # rm -rf /tmp/neo && \ + # mkdir -p /llm && \ + # cd /llm && \ + # git clone -b add_logits_attn_softcap https://github.com/hzjane/vllm.git /llm/vllm && \ + # cd /llm/vllm && \ + # pip install setuptools-scm && \ + # pip install --upgrade cmake && \ + # VLLM_TARGET_DEVICE=xpu pip install --no-build-isolation -v /llm/vllm && \ # pip install -r /llm/vllm/requirements-xpu.txt && \ # VLLM_TARGET_DEVICE=xpu python setup.py install && \ pip install mpi4py fastapi uvicorn openai && \ @@ -109,6 +109,6 @@ COPY ./start-fastchat-service.sh /llm/ COPY ./start-pp_serving-service.sh /llm/ COPY ./start-lightweight_serving-service.sh /llm/ -ENV LD_LIBRARY_PATH /usr/local/lib/python3.11/dist-packages/intel_extension_for_pytorch/lib/:/opt/intel/oneapi/tbb/2021.12/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/2021.12/opt/mpi/libfabric/lib:/opt/intel/oneapi/mpi/2021.12/lib:/opt/intel/oneapi/mkl/2024.1/lib:/opt/intel/oneapi/ippcp/2021.11/lib/:/opt/intel/oneapi/ipp/2021.11/lib:/opt/intel/oneapi/dpl/2022.5/lib:/opt/intel/oneapi/dnnl/2024.1/lib:/opt/intel/oneapi/debugger/2024.1/opt/debugger/lib:/opt/intel/oneapi/dal/2024.2/lib:/opt/intel/oneapi/compiler/2024.1/opt/oclfpga/host/linux64/lib:/opt/intel/oneapi/compiler/2024.1/opt/compiler/lib:/opt/intel/oneapi/compiler/2024.1/lib:/opt/intel/oneapi/ccl/2021.12/lib/ +ENV LD_LIBRARY_PATH /usr/local/lib/python3.11/dist-packages/intel_extension_for_pytorch/lib/:/opt/intel/oneapi/tbb/2021.12/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/2021.12/opt/mpi/libfabric/lib:/opt/intel/oneapi/mpi/2021.12/lib:/opt/intel/oneapi/mkl/2024.2/lib:/opt/intel/oneapi/ippcp/2021.11/lib/:/opt/intel/oneapi/ipp/2021.11/lib:/opt/intel/oneapi/dpl/2022.5/lib:/opt/intel/oneapi/dnnl/2024.2/lib:/opt/intel/oneapi/debugger/2024.2/opt/debugger/lib:/opt/intel/oneapi/dal/2024.2/lib:/opt/intel/oneapi/compiler/2024.2/opt/oclfpga/host/linux64/lib:/opt/intel/oneapi/compiler/2024.2/opt/compiler/lib:/opt/intel/oneapi/compiler/2024.2/lib:/opt/intel/oneapi/ccl/2021.12/lib/ WORKDIR /llm/