Skip to content

Commit 29c2046

Browse files
QuantuMopeQuantuMope
committed
Add new CICD + cluster docker
Move old docker files to archive directory
1 parent c53d611 commit 29c2046

10 files changed

Lines changed: 297 additions & 108 deletions

File tree

‎.github/workflows/build_hobot_image.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ jobs:
5050
- name: Build and push Hobot CICD image
5151
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
5252
with:
53-
context: ./hobot_cicd
53+
context: ./hobot
5454
push: true
5555
tags: ${{ steps.prep_tag.outputs.tag }}-hobot-cicd
5656
labels: ${{ steps.meta.outputs.labels }}

‎README.md‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,21 @@
11
# gail-container
22

3-
This reporsitory maintains a docker image for running CI/CD of Hobot and Alf (and hopefully for other purposes in the future).
3+
This repository maintains a docker image for running CI/CD of Hobot and Alf (and hopefully for other purposes in the future).
44

55
The main implementation can be found at the top-level [Dockerfile](./Dockerfile).
66

7-
## How to update the docker
7+
## How to update the docker on cluster
8+
9+
Build the docker with the appropriate tag and save it.
10+
```bash
11+
cd hobot
12+
docker build -t horizonrobotics:cuda11.8-torch2.6-py3.12-ubuntu24.04 -f hobot_cluster.dockerfile .
13+
# When saving make sure to convert from ${REPO}:${TAG} to ${TAG}.tar format.
14+
docker save horizonrobotics:cuda11.8-torch2.6-py3.12-ubuntu24.04 -o cuda11.8-torch2.6-py3.12-ubuntu24.04.tar
15+
```
16+
Then simply upload the tar file to `/data/docker_images` on the cluster and `chmod 666` the file.
17+
18+
## How to update the docker on CICD
819

920
1. Update the [requirements.txt](./hobot_cicd/requirements.txt) and commit
1021
2. Create a [new release](https://github.com/HorizonRoboticsInternal/gail-container/releases/new) on GitHub.

‎archive/hobot/10_nvidia.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"file_format_version" : "1.0.0",
3+
"ICD" : {
4+
"library_path" : "libEGL_nvidia.so.0"
5+
}
6+
}

‎archive/hobot/Dockerfile‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
# Copyright (c) 2023 Horizon Robotics. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# For lower nvidia driver <470
16+
#ARG CUDA_BASE=nvidia/cuda:11.1.1-cudnn8-devel-ubuntu20.04
17+
# For higher nvidia driver >=470
18+
ARG CUDA_BASE=nvidia/cuda:11.4.0-cudnn8-devel-ubuntu20.04
19+
20+
######################## Build Vulkan #################################
21+
FROM ubuntu:20.04 as vulkan-khronos
22+
23+
ENV TZ=US/Pacific
24+
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
25+
26+
RUN apt-get update && apt-get install -y --no-install-recommends \
27+
build-essential \
28+
ca-certificates \
29+
cmake \
30+
git \
31+
libegl1-mesa-dev \
32+
libwayland-dev \
33+
libx11-xcb-dev \
34+
libxkbcommon-dev \
35+
libxrandr-dev \
36+
python3 \
37+
python3-distutils \
38+
curl wget && \
39+
rm -rf /var/lib/apt/lists/*
40+
41+
ARG VULKAN_VERSION=sdk-1.1.121.0
42+
43+
# Download and compile vulkan components
44+
RUN ln -s /usr/bin/python3 /usr/bin/python && \
45+
git clone https://github.com/KhronosGroup/Vulkan-ValidationLayers.git /opt/vulkan && \
46+
cd /opt/vulkan && git checkout "${VULKAN_VERSION}" && \
47+
mkdir build && cd build && ../scripts/update_deps.py && \
48+
cmake -C helper.cmake -DCMAKE_BUILD_TYPE=Release .. && \
49+
cmake --build . && make install && ldconfig && \
50+
mkdir -p /usr/local/lib && cp -a Vulkan-Loader/build/install/lib/* /usr/local/lib && \
51+
mkdir -p /usr/local/include/vulkan && cp -r Vulkan-Headers/build/install/include/vulkan/* /usr/local/include/vulkan && \
52+
mkdir -p /usr/local/share/vulkan/registry && \
53+
cp -r Vulkan-Headers/build/install/share/vulkan/registry/* /usr/local/share/vulkan/registry && \
54+
rm -rf /opt/vulkan
55+
56+
FROM ${CUDA_BASE}
57+
58+
ENV TZ=US/Pacific
59+
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
60+
61+
ENV NVIDIA_DRIVER_CAPABILITIES compute,graphics,utility
62+
RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf \
63+
&& echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf
64+
ENV LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/usr/lib/i386-linux-gnu:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
65+
66+
RUN dpkg --add-architecture i386 && apt-get update && apt-get install -y --no-install-recommends \
67+
libxau6 libxau6:i386 \
68+
libxdmcp6 libxdmcp6:i386 \
69+
libxcb1 libxcb1:i386 \
70+
libxext6 libxext6:i386 \
71+
libx11-6 libx11-6:i386 \
72+
libglvnd0 libglvnd0:i386 \
73+
libgl1 libgl1:i386 \
74+
libglx0 libglx0:i386 \
75+
libegl1 libegl1:i386 \
76+
libgles2 libgles2:i386 \
77+
pkg-config \
78+
libglvnd-dev libglvnd-dev:i386 \
79+
libgl1-mesa-dev libgl1-mesa-dev:i386 \
80+
libegl1-mesa-dev libegl1-mesa-dev:i386 \
81+
libgles2-mesa-dev libgles2-mesa-dev:i386 \
82+
libx11-xcb-dev \
83+
libxkbcommon-dev \
84+
libwayland-dev \
85+
libxrandr-dev \
86+
libegl1-mesa-dev
87+
88+
COPY --from=vulkan-khronos /usr/local/bin /usr/local/bin
89+
COPY --from=vulkan-khronos /usr/local/lib /usr/local/lib
90+
COPY --from=vulkan-khronos /usr/local/include/vulkan /usr/local/include/vulkan
91+
COPY --from=vulkan-khronos /usr/local/share/vulkan /usr/local/share/vulkan
92+
COPY ./nvidia_icd.json /etc/vulkan/icd.d/nvidia_icd.json
93+
COPY ./10_nvidia.json /usr/share/glvnd/egl_vendor.d/10_nvidia.json
94+
95+
######################## Hobot dependencies #####################
96+
RUN apt update
97+
98+
RUN apt update && apt install -y --no-install-recommends \
99+
build-essential \
100+
vim tree curl wget unzip git cmake \
101+
libxml2 libxml2-dev libxslt1-dev libfreetype6-dev \
102+
dirmngr gnupg2 lsb-release \
103+
xvfb kmod swig patchelf ffmpeg rsync \
104+
libopenmpi-dev libcups2-dev libssl-dev \
105+
python3.8 python3-pip python3.8-dev python3-setuptools
106+
107+
RUN python3.8 -m pip install --upgrade pip
108+
RUN ln -sf /usr/bin/python3.8 /usr/bin/python \
109+
&& ln -sf /usr/bin/python3.8 /usr/bin/python3
110+
111+
# for opencv
112+
RUN apt-get install -y libglib2.0-0 libsm6 libxrender1 libxext6 libxext-dev
113+
114+
# Install the latest version of alf
115+
# RUN pip install alf@git+https://github.com/HorizonRobotics/alf@pytorch#egg=ALF --extra-index-url https://download.pytorch.org/whl/cu113
116+
117+
# Install pacakges from requirements.txt
118+
COPY ./requirements.txt /tmp/requirements.txt
119+
RUN pip install -r /tmp/requirements.txt
120+
121+
# Test cnest
122+
RUN pip --version
123+
RUN python -c "import cnest"
124+
125+
RUN pip install --upgrade protobuf==3.20.1
126+
127+
# Enable EGL
128+
ENV PYOPENGL_PLATFORM=egl
129+
RUN python -c "from OpenGL.raw.EGL._types import *"
130+
# Enable determinism for pytorch
131+
ENV CUBLAS_WORKSPACE_CONFIG=:4096:8
132+
133+
# gtest/gmock for running c++ unit tests
134+
RUN apt-get install -y libgmock-dev
135+
136+
# Clean up to make the resulting image smaller
137+
RUN rm -rf /var/lib/apt/lists/*

hobot/nvidia_icd.json renamed to archive/hobot/nvidia_icd.json

File renamed without changes.

‎archive/hobot/requirements.txt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Hobot Specific Dependencies
2+
mujoco>=2.1.2
3+
dm_control
4+
numpy-quaternion
5+
pyfcl@git+https://github.com/HorizonRoboticsInternal/python-fcl.git
6+
scikit-image
7+
8+
# Tools
9+
pydocstyle==4.0.0
10+
pylint==2.3.1
11+
yapf==0.28.0
12+
pre-commit
13+
selenium
14+
15+
## upgrade Alf Dependencies
16+
# numpy 1.24 will report 'TypeError: No loop matching the specified signature and casting was found for ufunc greater'
17+
numpy==1.21
18+
pybind11==2.10.4

hobot_cicd/Dockerfile renamed to archive/hobot_cicd/Dockerfile

File renamed without changes.

hobot_cicd/requirements.txt renamed to archive/hobot_cicd/requirements.txt

File renamed without changes.

‎hobot/Dockerfile‎

Lines changed: 54 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,6 @@
1-
# Copyright (c) 2023 Horizon Robotics. All Rights Reserved.
2-
#
3-
# Licensed under the Apache License, Version 2.0 (the "License");
4-
# you may not use this file except in compliance with the License.
5-
# You may obtain a copy of the License at
6-
#
7-
# http://www.apache.org/licenses/LICENSE-2.0
8-
#
9-
# Unless required by applicable law or agreed to in writing, software
10-
# distributed under the License is distributed on an "AS IS" BASIS,
11-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12-
# See the License for the specific language governing permissions and
13-
# limitations under the License.
14-
15-
# For lower nvidia driver <470
16-
#ARG CUDA_BASE=nvidia/cuda:11.1.1-cudnn8-devel-ubuntu20.04
17-
# For higher nvidia driver >=470
18-
ARG CUDA_BASE=nvidia/cuda:11.4.0-cudnn8-devel-ubuntu20.04
19-
20-
######################## Build Vulkan #################################
21-
FROM ubuntu:20.04 as vulkan-khronos
22-
23-
ENV TZ=US/Pacific
24-
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
25-
26-
RUN apt-get update && apt-get install -y --no-install-recommends \
27-
build-essential \
28-
ca-certificates \
29-
cmake \
30-
git \
31-
libegl1-mesa-dev \
32-
libwayland-dev \
33-
libx11-xcb-dev \
34-
libxkbcommon-dev \
35-
libxrandr-dev \
36-
python3 \
37-
python3-distutils \
38-
curl wget && \
39-
rm -rf /var/lib/apt/lists/*
40-
41-
ARG VULKAN_VERSION=sdk-1.1.121.0
42-
43-
# Download and compile vulkan components
44-
RUN ln -s /usr/bin/python3 /usr/bin/python && \
45-
git clone https://github.com/KhronosGroup/Vulkan-ValidationLayers.git /opt/vulkan && \
46-
cd /opt/vulkan && git checkout "${VULKAN_VERSION}" && \
47-
mkdir build && cd build && ../scripts/update_deps.py && \
48-
cmake -C helper.cmake -DCMAKE_BUILD_TYPE=Release .. && \
49-
cmake --build . && make install && ldconfig && \
50-
mkdir -p /usr/local/lib && cp -a Vulkan-Loader/build/install/lib/* /usr/local/lib && \
51-
mkdir -p /usr/local/include/vulkan && cp -r Vulkan-Headers/build/install/include/vulkan/* /usr/local/include/vulkan && \
52-
mkdir -p /usr/local/share/vulkan/registry && \
53-
cp -r Vulkan-Headers/build/install/share/vulkan/registry/* /usr/local/share/vulkan/registry && \
54-
rm -rf /opt/vulkan
55-
56-
FROM ${CUDA_BASE}
1+
FROM ubuntu:24.04
2+
# Avoid prompts from apt
3+
ENV DEBIAN_FRONTEND=noninteractive
574

585
ENV TZ=US/Pacific
596
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
@@ -63,6 +10,7 @@ RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf \
6310
&& echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf
6411
ENV LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:/usr/lib/i386-linux-gnu:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
6512

13+
# Install GPU libraries and dependencies
6614
RUN dpkg --add-architecture i386 && apt-get update && apt-get install -y --no-install-recommends \
6715
libxau6 libxau6:i386 \
6816
libxdmcp6 libxdmcp6:i386 \
@@ -82,56 +30,73 @@ RUN dpkg --add-architecture i386 && apt-get update && apt-get install -y --no-in
8230
libx11-xcb-dev \
8331
libxkbcommon-dev \
8432
libwayland-dev \
85-
libxrandr-dev \
86-
libegl1-mesa-dev
87-
88-
COPY --from=vulkan-khronos /usr/local/bin /usr/local/bin
89-
COPY --from=vulkan-khronos /usr/local/lib /usr/local/lib
90-
COPY --from=vulkan-khronos /usr/local/include/vulkan /usr/local/include/vulkan
91-
COPY --from=vulkan-khronos /usr/local/share/vulkan /usr/local/share/vulkan
92-
COPY ./nvidia_icd.json /etc/vulkan/icd.d/nvidia_icd.json
93-
COPY ./10_nvidia.json /usr/share/glvnd/egl_vendor.d/10_nvidia.json
33+
libxrandr-dev
9434

95-
######################## Hobot dependencies #####################
96-
RUN apt update
35+
COPY ./10_nvidia.json /usr/share/glvnd/egl_vendor.d/10_nvidia.json
9736

37+
# Install build tools and Python dependencies
9838
RUN apt update && apt install -y --no-install-recommends \
9939
build-essential \
10040
vim tree curl wget unzip git cmake \
10141
libxml2 libxml2-dev libxslt1-dev libfreetype6-dev \
10242
dirmngr gnupg2 lsb-release \
10343
xvfb kmod swig patchelf ffmpeg rsync \
10444
libopenmpi-dev libcups2-dev libssl-dev \
105-
python3.8 python3-pip python3.8-dev python3-setuptools
45+
python3 python3-pip python3-dev python3-setuptools python3-venv \
46+
libboost-all-dev \
47+
libosmesa6-dev \
48+
libglib2.0-0 \
49+
libxrender1 \
50+
libglew-dev \
51+
xpra \
52+
libglfw3-dev \
53+
pybind11-dev \
54+
libeigen3-dev \
55+
liboctomap-dev \
56+
libfcl-dev \
57+
libjpeg-dev \
58+
gnupg \
59+
ca-certificates \
60+
libsm6 libxext6 libxext-dev ninja-build
61+
62+
# Install Google Chrome
63+
RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor > /usr/share/keyrings/google-chrome.gpg \
64+
&& echo "deb [arch=amd64 signed-by=/usr/share/keyrings/google-chrome.gpg] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list \
65+
&& apt-get update \
66+
&& apt-get install -y google-chrome-stable
10667

107-
RUN python3.8 -m pip install --upgrade pip
108-
RUN ln -sf /usr/bin/python3.8 /usr/bin/python \
109-
&& ln -sf /usr/bin/python3.8 /usr/bin/python3
68+
# Make Python 3 the default
69+
RUN ln -sf /usr/bin/python3 /usr/bin/python && \
70+
ln -sf /usr/bin/pip3 /usr/bin/pip
11071

111-
# for opencv
112-
RUN apt-get install -y libglib2.0-0 libsm6 libxrender1 libxext6 libxext-dev
72+
# Create a working directory for the mounted volume
73+
WORKDIR /app
74+
RUN python3.12 -m venv /app/venv
11375

114-
# Install the latest version of alf
115-
# RUN pip install alf@git+https://github.com/HorizonRobotics/alf@pytorch#egg=ALF --extra-index-url https://download.pytorch.org/whl/cu113
76+
# Need to run umask 0000 so relax permissions for venv so that Hobot can pip install kincpp and ManiSkill
11677

117-
# Install pacakges from requirements.txt
118-
COPY ./requirements.txt /tmp/requirements.txt
119-
RUN pip install -r /tmp/requirements.txt
78+
RUN umask 0000 && /app/venv/bin/pip install --upgrade pip
12079

121-
# Test cnest
122-
RUN pip --version
123-
RUN python -c "import cnest"
80+
# Install torch 2.6 + cuda 11.8
81+
RUN umask 0000 && /app/venv/bin/pip install torch==2.6.0 torchvision --index-url https://download.pytorch.org/whl/cu118
12482

125-
RUN pip install --upgrade protobuf==3.20.1
83+
# Install all pip dependencies
84+
COPY requirements.txt /app/
85+
RUN umask 0000 && /app/venv/bin/pip install wheel
86+
RUN umask 0000 && /app/venv/bin/pip install -r /app/requirements.txt
87+
88+
# Check for PyTorch with CUDA 11.8
89+
RUN /app/venv/bin/python -c "import torch; import sys; version = torch.__version__; assert '+cu118' in version, f'Expected PyTorch with CUDA 11.8 (+cu118 suffix), but got version {version}'; sys.exit(0)"
12690

127-
# Enable EGL
12891
ENV PYOPENGL_PLATFORM=egl
129-
RUN python -c "from OpenGL.raw.EGL._types import *"
130-
# Enable determinism for pytorch
13192
ENV CUBLAS_WORKSPACE_CONFIG=:4096:8
13293

133-
# gtest/gmock for running c++ unit tests
134-
RUN apt-get install -y libgmock-dev
135-
13694
# Clean up to make the resulting image smaller
137-
RUN rm -rf /var/lib/apt/lists/*
95+
RUN rm -rf /var/lib/apt/lists/*
96+
97+
# Setup the venv
98+
ENV PATH="/app/venv/bin:$PATH"
99+
ENV VIRTUAL_ENV="/app/venv"
100+
101+
# Point to ManiSkill asset directory
102+
ENV MS_ASSET_DIR="/data"

0 commit comments

Comments
 (0)