Skip to content

Commit 399eac1

Browse files
waterfoulwaterfoul
committed
Finished moving pip venvs into cross-compiled layer
1 parent 71bc510 commit 399eac1

File tree

128 files changed

+4778
-511
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

128 files changed

+4778
-511
lines changed

‎.automation/build.py‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -519,35 +519,53 @@ def build_dockerfile(
519519
replace_in_file(dockerfile, "#PIP__START", "#PIP__END", pip_install_command)
520520
# Python packages in venv
521521
if len(pipvenv_packages.items()) > 0:
522-
pipenv_install_command = (
522+
pipenv_download_command = (
523523
"RUN PYTHONDONTWRITEBYTECODE=1 pip3 install"
524-
" --no-cache-dir --upgrade pip virtualenv \\\n"
524+
" --no-cache-dir --upgrade pip crossenv \\\n"
525+
)
526+
pipenv_install_command = (
527+
"RUN echo \\\n"
525528
)
526-
env_path_command = 'ENV PATH="${PATH}"'
529+
pipenv_path_command = 'ENV PATH="${PATH}"'
527530
for pip_linter, pip_linter_packages in pipvenv_packages.items():
531+
pipenv_download_command += (
532+
f' && mkdir -p "/download/{pip_linter}" '
533+
+ f'&& pip download -d "/download/{pip_linter}" '
534+
+ (" ".join(pip_linter_packages))
535+
+ " \\\n"
536+
)
528537
pipenv_install_command += (
529538
f' && mkdir -p "/venvs/{pip_linter}" '
530539
+ f'&& cd "/venvs/{pip_linter}" '
531-
+ "&& virtualenv . "
540+
+ "&& python3 -m crossenv /usr/local/bin/target-python3 . "
532541
+ "&& source bin/activate "
533-
+ "&& PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir "
542+
+ f"&& PYTHONDONTWRITEBYTECODE=1 pip3 install --find-links /download/{pip_linter} --no-cache-dir "
534543
+ (" ".join(pip_linter_packages))
535544
+ " "
536545
+ "&& deactivate "
537546
+ "&& cd ./../.. \\\n"
538547
)
539-
env_path_command += f":/venvs/{pip_linter}/bin"
548+
pipenv_path_command += f":/venvs/{pip_linter}/bin"
540549
pipenv_install_command = pipenv_install_command[:-2] # remove last \
550+
pipenv_download_command = pipenv_download_command[:-2] # remove last \
541551
pipenv_install_command += (
542552
' \\\n && find . | grep -E "(/__pycache__$|\\.pyc$|\\.pyo$)" | xargs rm -rf '
543553
+ "&& rm -rf /root/.cache\n"
544-
+ env_path_command
545554
)
555+
pipenv_download_command += "\n"
546556
else:
547557
pipenv_install_command = ""
558+
pipenv_download_command = ""
559+
pipenv_path_command = ""
548560
replace_in_file(
549561
dockerfile, "#PIPVENV__START", "#PIPVENV__END", pipenv_install_command
550562
)
563+
replace_in_file(
564+
dockerfile, "#PIPVENV_DOWNLOAD__START", "#PIPVENV_DOWNLOAD__END", pipenv_download_command
565+
)
566+
replace_in_file(
567+
dockerfile, "#PIPVENV_PATH__START", "#PIPVENV_PATH__END", pipenv_path_command
568+
)
551569

552570
# Ruby gem packages
553571
gem_install_command = ""

‎flavors/ci_light/Dockerfile‎

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,12 @@ RUN rustup-init -y --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-u
4343

4444
RUN --mount=type=cache,id=cargo-${TARGETARCH},sharing=locked,target=/cargo/.cargo/registry/,uid=63425 \
4545
. /cargo/.cargo/env \
46-
&& cargo install sarif-fmt shellcheck-sarif --root /tmp --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-unknown-linux-musl" || echo "aarch64-unknown-linux-musl")
46+
&& cargo install shellcheck-sarif sarif-fmt --root /tmp --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-unknown-linux-musl" || echo "aarch64-unknown-linux-musl")
4747

4848
FROM scratch AS cargo
4949
COPY --link --from=cargo-build /tmp/bin/* /bin/
50-
RUN ["/bin/sarif-fmt", "--help"]
5150
RUN ["/bin/shellcheck-sarif", "--help"]
51+
RUN ["/bin/sarif-fmt", "--help"]
5252

5353
#FROM__END
5454

@@ -69,6 +69,41 @@ COPY --link --from=gitleaks /usr/bin/gitleaks /usr/bin/
6969
COPY --link --from=cargo /bin/* /usr/bin/
7070
#COPY__END
7171

72+
FROM --platform=$TARGETPLATFORM python:3.11.2-alpine3.17 AS target-python-arm64
73+
RUN mkdir /export-libs && cp /lib/ld-musl-aarch64.so.1 /export-libs
74+
FROM --platform=$TARGETPLATFORM python:3.11.2-alpine3.17 AS target-python-amd64
75+
RUN mkdir /export-libs && cp /lib/ld-musl-x86_64.so.1 /export-libs
76+
FROM target-python-${TARGETARCH} AS target-python
77+
FROM --platform=$BUILDPLATFORM python:3.11.2-alpine3.17 AS python-venv
78+
79+
80+
#############################################################################################
81+
## @generated by .automation/build.py using descriptor files, please do not update manually ##
82+
#############################################################################################
83+
84+
RUN apk add --update --no-cache gcc musl-dev libffi-dev rust cargo cmake make g++ openssl-dev
85+
86+
#PIPVENV_DOWNLOAD__START
87+
RUN PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir --upgrade pip crossenv \
88+
&& mkdir -p "/download/yamllint" && pip download -d "/download/yamllint" yamllint
89+
90+
#PIPVENV_DOWNLOAD__END
91+
92+
RUN mkdir /venvs
93+
94+
COPY --link --from=target-python /usr/local/bin/python3 /usr/local/bin/target-python3
95+
#COPY --link --from=target-python /export-libs/* /lib
96+
97+
#############################################################################################
98+
## @generated by .automation/build.py using descriptor files, please do not update manually ##
99+
#############################################################################################
100+
101+
#PIPVENV__START
102+
RUN echo \
103+
&& mkdir -p "/venvs/yamllint" && cd "/venvs/yamllint" && python3 -m crossenv /usr/local/bin/target-python3 . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --find-links /download/yamllint --no-cache-dir yamllint && deactivate && cd ./../.. \
104+
&& find . | grep -E "(/__pycache__$|\.pyc$|\.pyo$)" | xargs rm -rf && rm -rf /root/.cache
105+
106+
#PIPVENV__END
72107

73108
##################
74109
# Get base image #
@@ -142,19 +177,18 @@ RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin || true && \
142177
# Ignore npm package issues
143178
yarn config set ignore-engines true || true
144179

180+
COPY --link --from=python-venv /venv /venv
181+
145182
#############################################################################################
146183
## @generated by .automation/build.py using descriptor files, please do not update manually ##
147184
#############################################################################################
148185
#PIP__START
149186

150187
#PIP__END
151188

152-
#PIPVENV__START
153-
RUN PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir --upgrade pip virtualenv \
154-
&& mkdir -p "/venvs/yamllint" && cd "/venvs/yamllint" && virtualenv . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir yamllint && deactivate && cd ./../.. \
155-
&& find . | grep -E "(/__pycache__$|\.pyc$|\.pyo$)" | xargs rm -rf && rm -rf /root/.cache
189+
#PIPVENV_PATH__START
156190
ENV PATH="${PATH}":/venvs/yamllint/bin
157-
#PIPVENV__END
191+
#PIPVENV_PATH__END
158192

159193
############################
160194
# Install NPM dependencies #

‎flavors/documentation/Dockerfile‎

Lines changed: 57 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,12 @@ RUN rustup-init -y --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-u
5050

5151
RUN --mount=type=cache,id=cargo-${TARGETARCH},sharing=locked,target=/cargo/.cargo/registry/,uid=63425 \
5252
. /cargo/.cargo/env \
53-
&& cargo install sarif-fmt shellcheck-sarif --root /tmp --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-unknown-linux-musl" || echo "aarch64-unknown-linux-musl")
53+
&& cargo install shellcheck-sarif sarif-fmt --root /tmp --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-unknown-linux-musl" || echo "aarch64-unknown-linux-musl")
5454

5555
FROM scratch AS cargo
5656
COPY --link --from=cargo-build /tmp/bin/* /bin/
57-
RUN ["/bin/sarif-fmt", "--help"]
5857
RUN ["/bin/shellcheck-sarif", "--help"]
58+
RUN ["/bin/sarif-fmt", "--help"]
5959

6060
#FROM__END
6161

@@ -83,6 +83,57 @@ COPY --link --from=gitleaks /usr/bin/gitleaks /usr/bin/
8383
COPY --link --from=cargo /bin/* /usr/bin/
8484
#COPY__END
8585

86+
FROM --platform=$TARGETPLATFORM python:3.11.2-alpine3.17 AS target-python-arm64
87+
RUN mkdir /export-libs && cp /lib/ld-musl-aarch64.so.1 /export-libs
88+
FROM --platform=$TARGETPLATFORM python:3.11.2-alpine3.17 AS target-python-amd64
89+
RUN mkdir /export-libs && cp /lib/ld-musl-x86_64.so.1 /export-libs
90+
FROM target-python-${TARGETARCH} AS target-python
91+
FROM --platform=$BUILDPLATFORM python:3.11.2-alpine3.17 AS python-venv
92+
93+
94+
#############################################################################################
95+
## @generated by .automation/build.py using descriptor files, please do not update manually ##
96+
#############################################################################################
97+
98+
RUN apk add --update --no-cache gcc musl-dev libffi-dev rust cargo cmake make g++ openssl-dev
99+
100+
#PIPVENV_DOWNLOAD__START
101+
RUN PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir --upgrade pip crossenv \
102+
&& mkdir -p "/download/ansible-lint" && pip download -d "/download/ansible-lint" ansible-lint \
103+
&& mkdir -p "/download/djlint" && pip download -d "/download/djlint" djlint \
104+
&& mkdir -p "/download/checkov" && pip download -d "/download/checkov" packaging checkov \
105+
&& mkdir -p "/download/semgrep" && pip download -d "/download/semgrep" semgrep \
106+
&& mkdir -p "/download/snakemake" && pip download -d "/download/snakemake" snakemake \
107+
&& mkdir -p "/download/snakefmt" && pip download -d "/download/snakefmt" snakefmt \
108+
&& mkdir -p "/download/proselint" && pip download -d "/download/proselint" proselint \
109+
&& mkdir -p "/download/sqlfluff" && pip download -d "/download/sqlfluff" sqlfluff \
110+
&& mkdir -p "/download/yamllint" && pip download -d "/download/yamllint" yamllint
111+
112+
#PIPVENV_DOWNLOAD__END
113+
114+
RUN mkdir /venvs
115+
116+
COPY --link --from=target-python /usr/local/bin/python3 /usr/local/bin/target-python3
117+
#COPY --link --from=target-python /export-libs/* /lib
118+
119+
#############################################################################################
120+
## @generated by .automation/build.py using descriptor files, please do not update manually ##
121+
#############################################################################################
122+
123+
#PIPVENV__START
124+
RUN echo \
125+
&& mkdir -p "/venvs/ansible-lint" && cd "/venvs/ansible-lint" && python3 -m crossenv /usr/local/bin/target-python3 . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --find-links /download/ansible-lint --no-cache-dir ansible-lint && deactivate && cd ./../.. \
126+
&& mkdir -p "/venvs/djlint" && cd "/venvs/djlint" && python3 -m crossenv /usr/local/bin/target-python3 . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --find-links /download/djlint --no-cache-dir djlint && deactivate && cd ./../.. \
127+
&& mkdir -p "/venvs/checkov" && cd "/venvs/checkov" && python3 -m crossenv /usr/local/bin/target-python3 . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --find-links /download/checkov --no-cache-dir packaging checkov && deactivate && cd ./../.. \
128+
&& mkdir -p "/venvs/semgrep" && cd "/venvs/semgrep" && python3 -m crossenv /usr/local/bin/target-python3 . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --find-links /download/semgrep --no-cache-dir semgrep && deactivate && cd ./../.. \
129+
&& mkdir -p "/venvs/snakemake" && cd "/venvs/snakemake" && python3 -m crossenv /usr/local/bin/target-python3 . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --find-links /download/snakemake --no-cache-dir snakemake && deactivate && cd ./../.. \
130+
&& mkdir -p "/venvs/snakefmt" && cd "/venvs/snakefmt" && python3 -m crossenv /usr/local/bin/target-python3 . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --find-links /download/snakefmt --no-cache-dir snakefmt && deactivate && cd ./../.. \
131+
&& mkdir -p "/venvs/proselint" && cd "/venvs/proselint" && python3 -m crossenv /usr/local/bin/target-python3 . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --find-links /download/proselint --no-cache-dir proselint && deactivate && cd ./../.. \
132+
&& mkdir -p "/venvs/sqlfluff" && cd "/venvs/sqlfluff" && python3 -m crossenv /usr/local/bin/target-python3 . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --find-links /download/sqlfluff --no-cache-dir sqlfluff && deactivate && cd ./../.. \
133+
&& mkdir -p "/venvs/yamllint" && cd "/venvs/yamllint" && python3 -m crossenv /usr/local/bin/target-python3 . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --find-links /download/yamllint --no-cache-dir yamllint && deactivate && cd ./../.. \
134+
&& find . | grep -E "(/__pycache__$|\.pyc$|\.pyo$)" | xargs rm -rf && rm -rf /root/.cache
135+
136+
#PIPVENV__END
86137

87138
##################
88139
# Get base image #
@@ -158,27 +209,18 @@ RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin || true && \
158209
# Ignore npm package issues
159210
yarn config set ignore-engines true || true
160211

212+
COPY --link --from=python-venv /venv /venv
213+
161214
#############################################################################################
162215
## @generated by .automation/build.py using descriptor files, please do not update manually ##
163216
#############################################################################################
164217
#PIP__START
165218

166219
#PIP__END
167220

168-
#PIPVENV__START
169-
RUN PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir --upgrade pip virtualenv \
170-
&& mkdir -p "/venvs/ansible-lint" && cd "/venvs/ansible-lint" && virtualenv . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir ansible-lint && deactivate && cd ./../.. \
171-
&& mkdir -p "/venvs/djlint" && cd "/venvs/djlint" && virtualenv . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir djlint && deactivate && cd ./../.. \
172-
&& mkdir -p "/venvs/checkov" && cd "/venvs/checkov" && virtualenv . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir packaging checkov && deactivate && cd ./../.. \
173-
&& mkdir -p "/venvs/semgrep" && cd "/venvs/semgrep" && virtualenv . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir semgrep && deactivate && cd ./../.. \
174-
&& mkdir -p "/venvs/snakemake" && cd "/venvs/snakemake" && virtualenv . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir snakemake && deactivate && cd ./../.. \
175-
&& mkdir -p "/venvs/snakefmt" && cd "/venvs/snakefmt" && virtualenv . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir snakefmt && deactivate && cd ./../.. \
176-
&& mkdir -p "/venvs/proselint" && cd "/venvs/proselint" && virtualenv . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir proselint && deactivate && cd ./../.. \
177-
&& mkdir -p "/venvs/sqlfluff" && cd "/venvs/sqlfluff" && virtualenv . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir sqlfluff && deactivate && cd ./../.. \
178-
&& mkdir -p "/venvs/yamllint" && cd "/venvs/yamllint" && virtualenv . && source bin/activate && PYTHONDONTWRITEBYTECODE=1 pip3 install --no-cache-dir yamllint && deactivate && cd ./../.. \
179-
&& find . | grep -E "(/__pycache__$|\.pyc$|\.pyo$)" | xargs rm -rf && rm -rf /root/.cache
221+
#PIPVENV_PATH__START
180222
ENV PATH="${PATH}":/venvs/ansible-lint/bin:/venvs/djlint/bin:/venvs/checkov/bin:/venvs/semgrep/bin:/venvs/snakemake/bin:/venvs/snakefmt/bin:/venvs/proselint/bin:/venvs/sqlfluff/bin:/venvs/yamllint/bin
181-
#PIPVENV__END
223+
#PIPVENV_PATH__END
182224

183225
############################
184226
# Install NPM dependencies #

0 commit comments

Comments
 (0)