Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add VCPKG's prerequisites to AMD GPU EPs docker files #23636

Merged
merged 10 commits into from
Feb 11, 2025
Merged

Add VCPKG's prerequisites to AMD GPU EPs docker files #23636

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ea1ec49
Add prerequisites to AMD GPU EPs docker files
snnn Feb 11, 2025
76cb640
Update rocm-ci-pipeline-env.Dockerfile
snnn Feb 11, 2025
f46ac10
Update migraphx-ci-pipeline-env.Dockerfile
snnn Feb 11, 2025
749bb3d
Merge remote-tracking branch 'upstream/main' into snnn/update_docker_…
snnn Feb 11, 2025
19a6c59
update
snnn Feb 11, 2025
cb6a02a
Merge remote-tracking branch 'origin/snnn/update_docker_files' into s…
snnn Feb 11, 2025
905ed93
update
snnn Feb 11, 2025
c5e9801
update
snnn Feb 11, 2025
dbec134
update
snnn Feb 11, 2025
754f562
update
snnn Feb 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions dockerfiles/scripts/install_common_deps.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,6 @@ pip install "wheel>=0.35.1"
rm -rf /opt/miniconda/pkgs

# Dependencies: cmake
wget --quiet https://github.com/Kitware/CMake/releases/download/v3.30.1/cmake-3.30.1-linux-x86_64.tar.gz
tar zxf cmake-3.30.1-linux-x86_64.tar.gz
rm -rf cmake-3.30.1-linux-x86_64.tar.gz
wget --quiet https://github.com/Kitware/CMake/releases/download/v3.31.5/cmake-3.31.5-linux-x86_64.tar.gz
tar zxf cmake-3.31.5-linux-x86_64.tar.gz
rm -rf cmake-3.31.5-linux-x86_64.tar.gz
6 changes: 3 additions & 3 deletions tools/android_custom_build/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ RUN apt-get update && apt-get install --yes --no-install-recommends \
unzip lsb-release

# cmake
RUN CMAKE_VERSION=3.30.1 && \
aria2c -q -d /tmp -o cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz \
--checksum=sha-256=ac31f077ef3378641fa25a3cb980d21b2f083982d3149a8f2eb9154f2b53696b \
RUN CMAKE_VERSION=3.31.5 && \
aria2c -d /tmp -o cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz \
--checksum=sha-256=2984e70515ff60c5e4a41922b5d715a8168a696a89721e3b114e36f453244f72 \
https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && \
tar -zxf /tmp/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz --strip=1 -C /usr

Expand Down
2 changes: 1 addition & 1 deletion tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_tensorrt10
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ FROM nvidia/cuda:11.8.0-cudnn8-devel-ubuntu22.04 AS base
# The local directory into which to build and install CMAKE
ARG ONNXRUNTIME_LOCAL_CODE_DIR=/code

ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/src/tensorrt/bin:${ONNXRUNTIME_LOCAL_CODE_DIR}/cmake-3.30.1-linux-x86_64/bin:/opt/miniconda/bin:${PATH}
ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/src/tensorrt/bin:${ONNXRUNTIME_LOCAL_CODE_DIR}/cmake-3.31.5-linux-x86_64/bin:/opt/miniconda/bin:${PATH}
ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update &&\
Expand Down
2 changes: 1 addition & 1 deletion tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ FROM nvidia/cuda:12.6.3-cudnn-devel-ubuntu22.04 AS base
# The local directory into which to build and install CMAKE
ARG ONNXRUNTIME_LOCAL_CODE_DIR=/code

ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/src/tensorrt/bin:${ONNXRUNTIME_LOCAL_CODE_DIR}/cmake-3.30.1-linux-x86_64/bin:/opt/miniconda/bin:${PATH}
ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/src/tensorrt/bin:${ONNXRUNTIME_LOCAL_CODE_DIR}/cmake-3.31.5-linux-x86_64/bin:/opt/miniconda/bin:${PATH}
ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update &&\
Expand Down
6 changes: 3 additions & 3 deletions tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,13 @@ RUN wget "https://github.com/intel/compute-runtime/releases/download/21.48.21782
sudo dpkg -i *.deb && rm -rf *.deb

RUN mkdir -p /opt/cmake/bin && \
wget https://github.com/Kitware/CMake/releases/download/v3.30.1/cmake-3.30.1-linux-x86_64.tar.gz && \
tar -xf cmake-3.30.1-linux-x86_64.tar.gz --strip 1 -C /opt/cmake && rm -rf /cmake-3.30.1-linux-x86_64.tar.gz && \
wget https://github.com/Kitware/CMake/releases/download/v3.31.5/cmake-3.31.5-linux-x86_64.tar.gz && \
tar -xf cmake-3.31.5-linux-x86_64.tar.gz --strip 1 -C /opt/cmake && rm -rf /cmake-3.31.5-linux-x86_64.tar.gz && \
ln -sf /opt/cmake/bin/* /usr/bin

ARG BUILD_UID=1000
ARG BUILD_USER=onnxruntimedev
WORKDIR /home/$BUILD_USER
RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID
RUN adduser $BUILD_USER video
USER $BUILD_USER
WORKDIR /home/$BUILD_USER
2 changes: 1 addition & 1 deletion tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt_bin
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ FROM nvidia/cuda:12.6.3-cudnn-devel-ubuntu22.04 AS base
# The local directory into which to build and install CMAKE
ARG ONNXRUNTIME_LOCAL_CODE_DIR=/code

ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${ONNXRUNTIME_LOCAL_CODE_DIR}/cmake-3.30.1-linux-x86_64/bin:/opt/miniconda/bin:${PATH}
ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${ONNXRUNTIME_LOCAL_CODE_DIR}/cmake-3.31.5-linux-x86_64/bin:/opt/miniconda/bin:${PATH}
ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update &&\
Expand Down
33 changes: 25 additions & 8 deletions tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ RUN echo "$APT_PREF" > /etc/apt/preferences.d/rocm-pin-600
ENV DEBIAN_FRONTEND noninteractive

RUN apt-get update && \
apt-get install -y --no-install-recommends ca-certificates ninja-build git zip curl libnuma-dev gnupg && \
apt-get install -y --no-install-recommends ca-certificates git unzip zip curl libnuma-dev gnupg && \
curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - &&\
printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ jammy main" | tee /etc/apt/sources.list.d/rocm.list && \
printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu jammy main" | tee /etc/apt/sources.list.d/amdgpu.list && \
Expand All @@ -29,13 +29,30 @@ RUN apt-get update && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

RUN groupadd -g 109 render

# Upgrade to meet security requirements
RUN apt-get update -y && apt-get upgrade -y && apt-get autoremove -y && \
apt-get install -y locales cifs-utils wget half libnuma-dev lsb-release && \
apt-get clean -y

# Cmake
ENV CMAKE_VERSION=3.31.5
RUN cd /usr/local && \
wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz && \
tar -zxf /usr/local/cmake-3.31.5-Linux-x86_64.tar.gz --strip=1 -C /usr

# Install Ninja
COPY scripts/install-ninja.sh /build_scripts/
RUN /bin/bash /build_scripts/install-ninja.sh

# Install VCPKG
ENV VCPKG_INSTALLATION_ROOT=/usr/local/share/vcpkg
ENV VCPKG_FORCE_SYSTEM_BINARIES=ON
COPY scripts/install-vcpkg.sh /build_scripts/
RUN /bin/bash /build_scripts/install-vcpkg.sh

RUN groupadd -g 109 render


ENV MIGRAPHX_DISABLE_FAST_GELU=1
RUN locale-gen en_US.UTF-8
RUN update-locale LANG=en_US.UTF-8
Expand All @@ -44,11 +61,6 @@ ENV LANG C.UTF-8

WORKDIR /stage

# Cmake
ENV CMAKE_VERSION=3.31.5
RUN cd /usr/local && \
wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz && \
tar -zxf /usr/local/cmake-3.31.5-Linux-x86_64.tar.gz --strip=1 -C /usr

# ccache
RUN mkdir -p /tmp/ccache && \
Expand Down Expand Up @@ -81,3 +93,8 @@ RUN ln -sf /usr/lib/x86_64-linux-gnu/libstdc++.so.6 ${CONDA_ENVIRONMENT_PATH}/bi
RUN apt update && apt install -y migraphx

RUN pip install numpy packaging ml_dtypes==0.5.0
ARG BUILD_UID=1000
ARG BUILD_USER=onnxruntimedev
RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID
USER $BUILD_USER
WORKDIR /home/$BUILD_USER
21 changes: 18 additions & 3 deletions tools/ci_build/github/linux/docker/rocm-ci-pipeline-env.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ RUN apt-get update && \
sudo \
libelf1 \
kmod \
file \
file zip unzip \
python3 \
python3-pip \
rocm-dev \
Expand All @@ -44,10 +44,20 @@ ENV LANG C.UTF-8
WORKDIR /stage

# Cmake
ENV CMAKE_VERSION=3.30.1
ENV CMAKE_VERSION=3.31.5
RUN cd /usr/local && \
wget -q https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz && \
tar -zxf /usr/local/cmake-3.30.1-Linux-x86_64.tar.gz --strip=1 -C /usr
tar -zxf /usr/local/cmake-3.31.5-Linux-x86_64.tar.gz --strip=1 -C /usr

# Install Ninja
COPY scripts/install-ninja.sh /build_scripts/
RUN /bin/bash /build_scripts/install-ninja.sh

# Install VCPKG
ENV VCPKG_INSTALLATION_ROOT=/usr/local/share/vcpkg
ENV VCPKG_FORCE_SYSTEM_BINARIES=ON
COPY scripts/install-vcpkg.sh /build_scripts/
RUN /bin/bash /build_scripts/install-vcpkg.sh

# ccache
RUN mkdir -p /tmp/ccache && \
Expand Down Expand Up @@ -88,3 +98,8 @@ RUN pip install packaging \
numpy==1.26.4

RUN apt install -y git
ARG BUILD_UID=1000
ARG BUILD_USER=onnxruntimedev
RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID
USER $BUILD_USER
WORKDIR /home/$BUILD_USER
11 changes: 11 additions & 0 deletions tools/ci_build/github/linux/docker/scripts/install-ninja.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#!/bin/bash
set -e -x
mkdir -p /tmp/ninja
cd /tmp/ninja
curl -O -sSL https://github.com/ninja-build/ninja/archive/v1.12.1.tar.gz
tar -zxvf v1.12.1.tar.gz --strip=1
cmake -Bbuild-cmake -H.
cmake --build build-cmake
mv ./build-cmake/ninja /usr/bin
cd /
rm -rf /tmp/ninja
8 changes: 8 additions & 0 deletions tools/ci_build/github/linux/docker/scripts/install-vcpkg.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/bin/bash
set -e -x
mkdir -p $VCPKG_INSTALLATION_ROOT
cd $VCPKG_INSTALLATION_ROOT
curl -O -sSL https://github.com/microsoft/vcpkg/archive/refs/heads/master.tar.gz
tar --strip=1 -zxf master.tar.gz
./bootstrap-vcpkg.sh
chmod -R 0777 $VCPKG_INSTALLATION_ROOT
10 changes: 5 additions & 5 deletions tools/ci_build/github/linux/docker/scripts/install_os_deps.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
INSTALL_DEPS_DISTRIBUTED_SETUP=false
INSTALL_PREFIX='/usr'
while getopts p:d:v:tmur parameter_Option
do case "${parameter_Option}"

Check warning on line 8 in tools/ci_build/github/linux/docker/scripts/install_os_deps.sh

View workflow job for this annotation

GitHub Actions / Optional Lint 

[shellcheck] reported by reviewdog 🐶
getopts specified -u, but it's not handled by this 'case'.

Raw Output:
./tools/ci_build/github/linux/docker/scripts/install_os_deps.sh:8:4: warning: getopts specified -u, but it's not handled by this 'case'. (ShellCheck.SC2213)

Check warning on line 8 in tools/ci_build/github/linux/docker/scripts/install_os_deps.sh

View workflow job for this annotation

GitHub Actions / Optional Lint 

[shellcheck] reported by reviewdog 🐶
Invalid flags are not handled. Add a *) case.

Raw Output:
./tools/ci_build/github/linux/docker/scripts/install_os_deps.sh:8:4: warning: Invalid flags are not handled. Add a *) case. (ShellCheck.SC2220)
in
p) INSTALL_PREFIX=${OPTARG};;
d) DEVICE_TYPE=${OPTARG};;
Expand All @@ -31,22 +31,22 @@
echo "File '$path' already exists. Skipping download"
return 0
else
rm -rf $path

Check warning on line 34 in tools/ci_build/github/linux/docker/scripts/install_os_deps.sh

View workflow job for this annotation

GitHub Actions / Optional Lint 

[shellcheck] reported by reviewdog 🐶
Double quote to prevent globbing and word splitting.

Raw Output:
./tools/ci_build/github/linux/docker/scripts/install_os_deps.sh:34:14: info: Double quote to prevent globbing and word splitting. (ShellCheck.SC2086)
fi
fi

if [[ -f $uri ]]; then
echo "'$uri' is a file path, copying file to '$path'"
cp $uri $path

Check warning on line 40 in tools/ci_build/github/linux/docker/scripts/install_os_deps.sh

View workflow job for this annotation

GitHub Actions / Optional Lint 

[shellcheck] reported by reviewdog 🐶
Double quote to prevent globbing and word splitting.

Raw Output:
./tools/ci_build/github/linux/docker/scripts/install_os_deps.sh:40:8: info: Double quote to prevent globbing and word splitting. (ShellCheck.SC2086)

Check warning on line 40 in tools/ci_build/github/linux/docker/scripts/install_os_deps.sh

View workflow job for this annotation

GitHub Actions / Optional Lint 

[shellcheck] reported by reviewdog 🐶
Double quote to prevent globbing and word splitting.

Raw Output:
./tools/ci_build/github/linux/docker/scripts/install_os_deps.sh:40:13: info: Double quote to prevent globbing and word splitting. (ShellCheck.SC2086)
return $?
fi

echo "Downloading $uri"
# Use aria2c if available, otherwise use curl
if command -v aria2c > /dev/null; then
aria2c -q -d $(dirname $path) -o $(basename $path) "$uri"

Check warning on line 47 in tools/ci_build/github/linux/docker/scripts/install_os_deps.sh

View workflow job for this annotation

GitHub Actions / Optional Lint 

[shellcheck] reported by reviewdog 🐶
Quote this to prevent word splitting.

Raw Output:
./tools/ci_build/github/linux/docker/scripts/install_os_deps.sh:47:18: warning: Quote this to prevent word splitting. (ShellCheck.SC2046)

Check warning on line 47 in tools/ci_build/github/linux/docker/scripts/install_os_deps.sh

View workflow job for this annotation

GitHub Actions / Optional Lint 

[shellcheck] reported by reviewdog 🐶
Double quote to prevent globbing and word splitting.

Raw Output:
./tools/ci_build/github/linux/docker/scripts/install_os_deps.sh:47:28: info: Double quote to prevent globbing and word splitting. (ShellCheck.SC2086)

Check warning on line 47 in tools/ci_build/github/linux/docker/scripts/install_os_deps.sh

View workflow job for this annotation

GitHub Actions / Optional Lint 

[shellcheck] reported by reviewdog 🐶
Quote this to prevent word splitting.

Raw Output:
./tools/ci_build/github/linux/docker/scripts/install_os_deps.sh:47:38: warning: Quote this to prevent word splitting. (ShellCheck.SC2046)

Check warning on line 47 in tools/ci_build/github/linux/docker/scripts/install_os_deps.sh

View workflow job for this annotation

GitHub Actions / Optional Lint 

[shellcheck] reported by reviewdog 🐶
Double quote to prevent globbing and word splitting.

Raw Output:
./tools/ci_build/github/linux/docker/scripts/install_os_deps.sh:47:49: info: Double quote to prevent globbing and word splitting. (ShellCheck.SC2086)
else
curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail

Check warning on line 49 in tools/ci_build/github/linux/docker/scripts/install_os_deps.sh

View workflow job for this annotation

GitHub Actions / Optional Lint 

[shellcheck] reported by reviewdog 🐶
Double quote to prevent globbing and word splitting.

Raw Output:
./tools/ci_build/github/linux/docker/scripts/install_os_deps.sh:49:30: info: Double quote to prevent globbing and word splitting. (ShellCheck.SC2086)
fi

return $?
Expand All @@ -70,18 +70,18 @@
tar --strip 1 -xf /tmp/azcopy/azcopy.tar.gz -C /tmp/azcopy
cp /tmp/azcopy/azcopy /usr/bin
echo "Installing cmake"
GetFile https://github.com/Kitware/CMake/releases/download/v3.30.1/cmake-3.30.1-Linux-x86_64.tar.gz /tmp/src/cmake-3.30.1-Linux-x86_64.tar.gz
tar -zxf /tmp/src/cmake-3.30.1-Linux-x86_64.tar.gz --strip=1 -C /usr
GetFile https://github.com/Kitware/CMake/releases/download/v3.31.5/cmake-3.31.5-Linux-x86_64.tar.gz /tmp/src/cmake-3.31.5-Linux-x86_64.tar.gz
tar -zxf /tmp/src/cmake-3.31.5-Linux-x86_64.tar.gz --strip=1 -C /usr
echo "Installing Node.js"
# The EOL for nodejs v18.17.1 LTS is April 2025
GetFile https://nodejs.org/dist/v18.17.1/node-v18.17.1-linux-x64.tar.xz /tmp/src/node-v18.17.1-linux-x64.tar.xz
tar -xf /tmp/src/node-v18.17.1-linux-x64.tar.xz --strip=1 -C /usr
else
echo "Installing cmake"
GetFile https://github.com/Kitware/CMake/releases/download/v3.30.1/cmake-3.30.1.tar.gz /tmp/src/cmake-3.30.1.tar.gz
tar -xf /tmp/src/cmake-3.30.1.tar.gz -C /tmp/src
GetFile https://github.com/Kitware/CMake/releases/download/v3.31.5/cmake-3.31.5.tar.gz /tmp/src/cmake-3.31.5.tar.gz
tar -xf /tmp/src/cmake-3.31.5.tar.gz -C /tmp/src
pushd .
cd /tmp/src/cmake-3.30.1
cd /tmp/src/cmake-3.31.5
./bootstrap --prefix=/usr --parallel=$(getconf _NPROCESSORS_ONLN) --system-bzip2 --system-curl --system-zlib --system-expat
make -j$(getconf _NPROCESSORS_ONLN)
make install
Expand Down
Loading