Skip to content

Commit b2805f5

Browse files
author
Azure Pipelines
committed
Merge remote-tracking branch 'origin/main' into publication
2 parents a82c833 + 8460b48 commit b2805f5

File tree

4 files changed

+196
-12
lines changed

4 files changed

+196
-12
lines changed

‎.actions/assistant.py

+4-12
Original file line numberDiff line numberDiff line change
@@ -143,16 +143,6 @@ class AssistantCLI:
143143
DATASETS_FOLDER = os.environ.get("PATH_DATASETS", "_datasets")
144144
DRY_RUN = bool(int(os.environ.get("DRY_RUN", 0)))
145145
_META_REQUIRED_FIELDS = ("title", "author", "license", "description")
146-
_SKIP_DIRS = (
147-
".actions",
148-
".azure",
149-
".datasets",
150-
".github",
151-
"_docs",
152-
"_TEMP",
153-
"_requirements",
154-
DIR_NOTEBOOKS,
155-
)
156146
_META_FILE_REGEX = ".meta.{yaml,yml}"
157147
_META_PIP_KEY = "pip__"
158148
_META_ACCEL_DEFAULT = _LOCAL_ACCELERATOR.split(",")
@@ -463,6 +453,8 @@ def _replace_images(lines: list, local_dir: str) -> list:
463453
@staticmethod
464454
def _is_ipynb_parent_dir(dir_path: str) -> bool:
465455
"""Determine in recursive fashion of a folder is valid notebook file or any of sub-folders is."""
456+
if dir_path.startswith("_"):
457+
return False
466458
if AssistantCLI._find_meta(dir_path):
467459
return True
468460
sub_dirs = [d for d in glob.glob(os.path.join(dir_path, "*")) if os.path.isdir(d)]
@@ -513,8 +505,8 @@ def group_folders(
513505
dirs = [os.path.join(root_path, d) for d in dirs]
514506
# unique folders
515507
dirs = set(dirs)
516-
# drop folder with skip folder
517-
dirs = [pdir for pdir in dirs if not any(ndir in AssistantCLI._SKIP_DIRS for ndir in pdir.split(os.path.sep))]
508+
# drop folder that start with . or _ as they are meant to be internal use only
509+
dirs = [pdir for pdir in dirs if not any(ndir[0] in (".", "_") for ndir in pdir.split(os.path.sep))]
518510
# valid folder has meta
519511
dirs_exist = [d for d in dirs if os.path.isdir(d)]
520512
dirs_invalid = [d for d in dirs_exist if not AssistantCLI._find_meta(d)]

‎.github/workflows/docker-build.yml

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
name: "Build (& Push) Dockers"
2+
3+
on: # Trigger the workflow on push or pull request, but only for the main branch
4+
push:
5+
branches: [main]
6+
pull_request:
7+
branches: [main]
8+
paths:
9+
- ".github/workflows/docker-build.yml"
10+
- "_requirements/*"
11+
- "requirements.txt"
12+
- "_dockers/**"
13+
workflow_dispatch: {}
14+
15+
concurrency:
16+
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
17+
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
18+
19+
env:
20+
PUSH_DOCKERHUB: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }}
21+
22+
jobs:
23+
build-cuda:
24+
if: github.event.pull_request.draft == false
25+
runs-on: ubuntu-latest
26+
steps:
27+
- uses: actions/checkout@v4
28+
29+
- name: Login to DockerHub
30+
uses: docker/login-action@v3
31+
if: env.PUSH_DOCKERHUB == 'true' && github.repository_owner == 'Lightning-AI'
32+
with:
33+
username: ${{ secrets.DOCKER_USERNAME }}
34+
password: ${{ secrets.DOCKER_PASSWORD }}
35+
36+
- name: Build (and Push) image
37+
uses: docker/build-push-action@v5
38+
with:
39+
#build-args: |
40+
# UBUNTU_VERSION=${{ matrix.ubuntu }}
41+
# PYTHON_VERSION=${{ matrix.python }}
42+
# PYTORCH_VERSION=${{ matrix.pytorch }}
43+
# CUDA_VERSION=${{ matrix.cuda }}
44+
file: _dockers/ubuntu-cuda/Dockerfile
45+
push: ${{ env.PUSH_DOCKERHUB }}
46+
# todo: publish also tag YYYY.MM
47+
tags: "pytorchlightning/tutorials"
48+
timeout-minutes: 55

‎_dockers/README.md

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# Docker images
2+
3+
## Build images from Dockerfiles
4+
5+
You can build it on your own, note it takes lots of time, be prepared.
6+
7+
```bash
8+
git clone https://github.com/Lightning-AI/torchmetrics.git
9+
10+
# build with the default arguments
11+
docker image build -t tutorials:latest -f dockers/ubuntu-cuda/Dockerfile .
12+
13+
# build with specific arguments
14+
docker image build -t tutorials:ubuntu-cuda11.7.1-py3.9-torch1.13 \
15+
-f dockers/base-cuda/Dockerfile \
16+
--build-arg PYTHON_VERSION=3.9 \
17+
--build-arg PYTORCH_VERSION=1.13 \
18+
--build-arg CUDA_VERSION=11.7.1 \
19+
.
20+
```
21+
22+
To run your docker use
23+
24+
```bash
25+
docker image list
26+
docker run --rm -it tutorials:latest bash
27+
```
28+
29+
and if you do not need it anymore, just clean it:
30+
31+
```bash
32+
docker image list
33+
docker image rm tutorials:latest
34+
```
35+
36+
## Run docker image with GPUs
37+
38+
To run docker image with access to your GPUs, you need to install
39+
40+
```bash
41+
# Add the package repositories
42+
distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
43+
curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
44+
curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
45+
46+
sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit
47+
sudo systemctl restart docker
48+
```
49+
50+
and later run the docker image with `--gpus all`. For example,
51+
52+
```bash
53+
docker run --rm -it --gpus all tutorials:ubuntu-cuda11.7.1-py3.9-torch1.12
54+
```

‎_dockers/ubuntu-cuda/Dockerfile

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# Copyright The Lightning AI team.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
ARG UBUNTU_VERSION=22.04
16+
ARG CUDA_VERSION=11.8.0
17+
18+
19+
FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
20+
21+
ARG PYTHON_VERSION=3.10
22+
ARG PYTORCH_VERSION=2.0
23+
24+
SHELL ["/bin/bash", "-c"]
25+
# https://techoverflow.net/2019/05/18/how-to-fix-configuring-tzdata-interactive-input-when-building-docker-images/
26+
ENV \
27+
DEBIAN_FRONTEND="noninteractive" \
28+
TZ="Etc/UTC" \
29+
PATH="$PATH:/root/.local/bin" \
30+
CUDA_TOOLKIT_ROOT_DIR="/usr/local/cuda" \
31+
MKL_THREADING_LAYER="GNU" \
32+
# MAKEFLAGS="-j$(nproc)"
33+
MAKEFLAGS="-j2"
34+
35+
RUN \
36+
apt-get -y update --fix-missing && \
37+
apt-get install -y --no-install-recommends --allow-downgrades --allow-change-held-packages \
38+
build-essential \
39+
ca-certificates \
40+
software-properties-common \
41+
pkg-config \
42+
libopenmpi-dev \
43+
openmpi-bin \
44+
cmake \
45+
git \
46+
wget \
47+
curl \
48+
unzip \
49+
g++ \
50+
cmake \
51+
ffmpeg \
52+
git \
53+
ssh \
54+
tree \
55+
&& \
56+
# Install python
57+
add-apt-repository ppa:deadsnakes/ppa && \
58+
apt-get install -y \
59+
python${PYTHON_VERSION} \
60+
python${PYTHON_VERSION}-distutils \
61+
python${PYTHON_VERSION}-dev \
62+
&& \
63+
update-alternatives --install /usr/bin/python${PYTHON_VERSION%%.*} python${PYTHON_VERSION%%.*} /usr/bin/python${PYTHON_VERSION} 1 && \
64+
update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \
65+
curl https://bootstrap.pypa.io/get-pip.py | python && \
66+
# Cleaning
67+
apt-get autoremove -y && \
68+
apt-get clean && \
69+
rm -rf /root/.cache && \
70+
rm -rf /var/lib/apt/lists/*
71+
72+
ENV PYTHONPATH="/usr/lib/python${PYTHON_VERSION}/site-packages"
73+
74+
COPY .actions/ .actions/
75+
COPY requirements.txt .
76+
COPY _requirements/ _requirements/
77+
78+
RUN \
79+
CUDA_VERSION_MM=${CUDA_VERSION%.*} && \
80+
CU_VERSION_MM=${CUDA_VERSION_MM//'.'/''} && \
81+
pip install --no-cache-dir -r requirements.txt \
82+
--find-links "https://download.pytorch.org/whl/cu${CU_VERSION_MM}/torch_stable.html" && \
83+
rm -rf requirements.txt .actions/ _requirements/
84+
85+
RUN \
86+
# Show what we have
87+
pip --version && \
88+
pip list && \
89+
python -c "import sys; ver = sys.version_info ; assert f'{ver.major}.{ver.minor}' == '$PYTHON_VERSION', ver" && \
90+
python -c "import torch; assert torch.__version__.startswith('$PYTORCH_VERSION'), torch.__version__"

0 commit comments

Comments
 (0)