Skip to content

Refactor/revert builder #104

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rpignolet merged 3 commits into from
Sep 5, 2025
Merged

Refactor/revert builder #104

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b6f16a7
"feat(tdp-builder): new tdp-builder based on apache hadoop 3.4.1 buil…
Aug 20, 2025
38b1655
fix(tdp_builder): maven 3.9.9 replaced with 3.9.11 since not in offic…
Aug 20, 2025
7b60309
refactor(tdp-builder): revert to previous docker-entrypoint.sh
SteBaum Sep 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 26 additions & 38 deletions build-env/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
# Dockerfile for installing the necessary dependencies for building Hadoop.
# See BUILDING.txt.

FROM ubuntu:focal
ARG OS_IDENTIFIER=ubuntu-2004
FROM ubuntu:bionic
ARG OS_IDENTIFIER=ubuntu-1804

WORKDIR /root

Expand All @@ -39,8 +39,6 @@ RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends \
ant \
apt-utils \
autoconf \
automake \
bats \
build-essential \
bzip2 \
Expand All @@ -65,7 +63,7 @@ RUN apt-get -q update \
libsnappy-dev \
libssl-dev \
libtool \
libzstd-dev \
libzstd1-dev \
locales \
make \
openjdk-8-jdk \
Expand All @@ -83,7 +81,6 @@ RUN apt-get -q update \
sudo \
unzip \
valgrind \
vim \
wget \
zlib1g-dev \
&& apt-get clean \
Expand All @@ -94,13 +91,13 @@ ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
ENV PYTHONIOENCODING=utf-8

# Install Maven 3.9.9
RUN mkdir /opt/maven \
&& curl -L https://dlcdn.apache.org/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.zip -o apache-maven-3.9.9-bin.zip \
&& unzip apache-maven-3.9.9-bin.zip \
&& mv apache-maven-3.9.9 /opt/maven \
&& rm apache-maven-3.9.9-bin.zip
RUN mkdir /opt/maven \
&& curl -L https://dlcdn.apache.org/maven/maven-3/3.9.11/binaries/apache-maven-3.9.11-bin.zip -o apache-maven-3.9.11-bin.zip \
&& unzip apache-maven-3.9.11-bin.zip \
&& mv apache-maven-3.9.11 /opt/maven \
&& rm apache-maven-3.9.11-bin.zip

ENV MAVEN_HOME=/opt/maven/apache-maven-3.9.9
ENV MAVEN_HOME=/opt/maven/apache-maven-3.9.11
ENV PATH=$PATH:$MAVEN_HOME/bin

######
Expand All @@ -124,18 +121,16 @@ RUN wget https://cdn.posit.co/r/${OS_IDENTIFIER}/pkgs/r-${R_VERSION}_1_amd64.deb
# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64


#######
# Install Boost 1.86
# Install Boost 1.72 (1.65 ships with Bionic)
#######

# hadolint ignore=DL3003
RUN mkdir -p /opt/boost-library \
&& curl -L https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download >boost_1_86_0.tar.bz2 \
&& mv boost_1_86_0.tar.bz2 /opt/boost-library \
RUN mkdir -p /opt/boost-library \
&& curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
&& mv boost_1_72_0.tar.bz2 /opt/boost-library \
&& cd /opt/boost-library \
&& tar --bzip2 -xf boost_1_86_0.tar.bz2 \
&& cd /opt/boost-library/boost_1_86_0 \
&& tar --bzip2 -xf boost_1_72_0.tar.bz2 \
&& cd /opt/boost-library/boost_1_72_0 \
&& ./bootstrap.sh --prefix=/usr/ \
&& ./b2 --without-python install \
&& cd /root \
Expand All @@ -146,22 +141,21 @@ RUN mkdir -p /opt/boost-library \
#######
RUN mkdir -p /opt/spotbugs \
&& curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
-o /opt/spotbugs.tgz \
-o /opt/spotbugs.tgz \
&& tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
&& chmod +x /opt/spotbugs/bin/*
ENV SPOTBUGS_HOME /opt/spotbugs

######
# Install Google Protobuf 3.21.12
# Install Google Protobuf 3.7.1 (3.0.0 ships with Bionic)
######
# hadolint ignore=DL3003
RUN mkdir -p /opt/protobuf-src \
&& curl -L -s -S \
https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz \
-o /opt/protobuf.tar.gz \
https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-o /opt/protobuf.tar.gz \
&& tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
&& cd /opt/protobuf-src/ \
&& ./autogen.sh \
&& cd /opt/protobuf-src \
&& ./configure --prefix=/opt/protobuf \
&& make "-j$(nproc)" \
&& make install \
Expand All @@ -170,28 +164,22 @@ RUN mkdir -p /opt/protobuf-src \
ENV PROTOBUF_HOME /opt/protobuf
ENV PATH "${PATH}:/opt/protobuf/bin"


# Use venv because of new python system protection
ENV VENV_PATH=/opt/venv
RUN python3 -m venv $VENV_PATH
ENV PATH "$VENV_PATH/bin:$PATH"

####
# Upgrade pip3
####
RUN python3 -m pip install --upgrade pip setuptools wheel
RUN python3 -m pip install --upgrade pip

####
# Install pandas and pyarrow for Spark 3
# venv-pack for jupyterhub venv
####
RUN $VENV_PATH/bin/pip3 install numpy==1.24.4 \
pandas==2.0.3 \
pyarrow==14.0.2 \
RUN pip3 install numpy==1.19.5 \
pandas==1.0.5 \
pyarrow==4.0.1 \
venv-pack==0.2.0

# Install pylint and python-dateutil
RUN $VENV_PATH/bin/pip3 install pylint==2.6.0 python-dateutil==2.8.2
RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1

###
## Install Yarn 1.12.1 for web UI framework
Expand Down Expand Up @@ -257,7 +245,7 @@ RUN mkdir -p /opt/gradle \
&& curl -L https://services.gradle.org/distributions/gradle-8.1.1-bin.zip -o gradle-8.1.1-bin.zip \
&& unzip gradle-8.1.1-bin.zip \
&& mv gradle-8.1.1 /opt/gradle \
&& rm gradle-8.1.1-bin.zip
&& rm gradle-8.1.1-bin.zip

ENV GRADLE_HOME=/opt/gradle/gradle-8.1.1
ENV PATH=$PATH:$GRADLE_HOME/bin
Expand Down
10 changes: 5 additions & 5 deletions build-env/docker-entrypoint.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ set -eo pipefail

if [[ -n "$BUILDER_UID" ]] && [[ -n "$BUILDER_GID" ]]; then
# Create group and user only if they don't exist
[[ ! $(getent group "$BUILDER_GID") ]] && groupadd --gid "$BUILDER_GID" --system builder
if [[ ! $(getent passwd "$BUILDER_UID") ]]; then
useradd --uid "$BUILDER_UID" --system --gid "$BUILDER_GID" --home-dir /home/builder builder
[[ ! $(getent group builder) ]] && groupadd --gid "$BUILDER_GID" --system builder
if [[ ! $(getent passwd builder) ]]; then
useradd --uid "$BUILDER_UID" --system --gid builder --home-dir /home/builder builder
# Avoid useradd warning if home dir already exists by making home dir ourselves.
# Home dir can exists if mounted via "docker run -v ...:/home/builder/...".
mkdir -p /home/builder
Expand All @@ -15,9 +15,9 @@ if [[ -n "$BUILDER_UID" ]] && [[ -n "$BUILDER_GID" ]]; then
# Avoid changing dir if a work dir is specified
[[ "$PWD" == "/root" ]] && cd /home/builder
if [[ -z "$@" ]]; then
exec gosu $BUILDER_UID /bin/bash
exec gosu builder /bin/bash
else
exec gosu $BUILDER_UID "$@"
exec gosu builder "$@"
fi
fi

Expand Down