Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

stage 4 & stage 5 #37

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
+503 −5
Draft

stage 4 & stage 5 #37

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
adac35b
llvm-project: implement stage4
nickdesaulniers Jun 13, 2022
0027cb7
clean up llvm-project container after make clang
nickdesaulniers Jun 13, 2022
0f42830
build all targets, otherwise we can't profile those builds
nickdesaulniers Jun 13, 2022
2c94a93
llvm-project: stage4 and stage5
nickdesaulniers Jun 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
208 changes: 208 additions & 0 deletions llvm-project/Dockerfile.epoch4
View file
Original file line number Diff line number Diff line change
@@ -84,9 +84,217 @@ RUN ninja -C ${LLVM_BUILD_DIR} \
install-llvm-readelf \
install-llvm-strip
RUN ninja -C ${LLVM_BUILD_DIR} install-compiler-rt
RUN ninja -C ${LLVM_BUILD_DIR} install-llvm-profdata

RUN apk del cmake ninja python3

# Final test
RUN llvm-readelf -p .comment $(which clang) | grep -e clang -e LLD
RUN llvm-readelf -p .comment $(which clang) | grep -v GCC

# Throw away profiles from building LLVM itself.
RUN find ${LLVM_BUILD_DIR} -name \*.profraw | xargs rm

### Start doing kernel builds
# TODO: objtool needs libelf.h and gelf.h? elfutils-dev is the alpine package.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/tools/objtool/Makefile#n22

I think this comment can be deleted.

RUN apk add make flex bison
ARG arch=arm
RUN ARCH=${arch} HOSTCFLAGS=${SYSROOT} HOSTLDFLAGS=${SYSROOT} \
make -C linux-5.18-rc6 LLVM=1 -s -j$(nproc) allnoconfig all && \
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should benchmark the performance gains between allnoconfig and defconfig. I worry that allnoconfig might not provide as much coverage.

llvm-profdata merge --output=${arch}.profdata.prof \
$(find ${LLVM_BUILD_DIR} -name \*.profraw) && \
find ${LLVM_BUILD_DIR} -name \*.profraw | xargs rm
Comment on lines +104 to +106
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the reasoning behind merging the profiles after each kernel build then merging them all at the end? Wouldn't be be simpler to merge them all at the end?

Additionally, if these remain around, I think you can avoid the find call, as the .profraw files should be in ${LLVM_BUILD_DIR}/profiles.

rm ${LLVM_BUILD_DIR}/profiles/*.profraw


ARG arch=arm64
RUN ARCH=${arch} HOSTCFLAGS=${SYSROOT} HOSTLDFLAGS=${SYSROOT} \
make -C linux-5.18-rc6 LLVM=1 -s -j$(nproc) allnoconfig all && \
llvm-profdata merge --output=${arch}.profdata.prof \
$(find ${LLVM_BUILD_DIR} -name \*.profraw) && \
find ${LLVM_BUILD_DIR} -name \*.profraw | xargs rm

ARG arch=hexagon
RUN ARCH=${arch} HOSTCFLAGS=${SYSROOT} HOSTLDFLAGS=${SYSROOT} \
make -C linux-5.18-rc6 LLVM=1 -s -j$(nproc) allnoconfig all && \
llvm-profdata merge --output=${arch}.profdata.prof \
$(find ${LLVM_BUILD_DIR} -name \*.profraw) && \
find ${LLVM_BUILD_DIR} -name \*.profraw | xargs rm

# Mips needs bash if building all targets.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Documentation/process/changes.rst was recently updated to include bash as one of the required tools for building the kernel so this comment seems superfluous.

ARG arch=mips
RUN ARCH=${arch} HOSTCFLAGS=${SYSROOT} HOSTLDFLAGS=${SYSROOT} \
make -C linux-5.18-rc6 LLVM=1 -s -j$(nproc) allnoconfig vmlinux && \
llvm-profdata merge --output=${arch}.profdata.prof \
$(find ${LLVM_BUILD_DIR} -name \*.profraw) && \
find ${LLVM_BUILD_DIR} -name \*.profraw | xargs rm

# powernv_defconfig needs find -printf to work. find in busybox in alpine
# doesn't support that flag.
ARG arch=powerpc
RUN ARCH=${arch} HOSTCFLAGS=${SYSROOT} HOSTLDFLAGS=${SYSROOT} \
make -C linux-5.18-rc6 LLVM=1 -s -j$(nproc) allnoconfig vmlinux && \
llvm-profdata merge --output=${arch}.profdata.prof \
$(find ${LLVM_BUILD_DIR} -name \*.profraw) && \
find ${LLVM_BUILD_DIR} -name \*.profraw | xargs rm

# riscv requires perl to build
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are several perl scripts in the kernel and it is listed as required in Documentation/process/changes.rst so I think this can be dropped like the bash comment above.

RUN apk add perl findutils
ARG arch=riscv
RUN ARCH=${arch} HOSTCFLAGS=${SYSROOT} HOSTLDFLAGS=${SYSROOT} \
make -C linux-5.18-rc6 LLVM=1 -s -j$(nproc) defconfig vmlinux && \
llvm-profdata merge --output=${arch}.profdata.prof \
$(find ${LLVM_BUILD_DIR} -name \*.profraw) && \
find ${LLVM_BUILD_DIR} -name \*.profraw | xargs rm
RUN apk del perl findutils

# s390 needs CC=clang, but Alpine does not package GNU binutils for s390 in
# main, so we need to fetch it from testing.
RUN apk add binutils-s390x --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
# We did not build openssl-dev or elfutils-dev, so hack them into the sysroot.
# Don't reuse the sysroot from stage4!
# s390 needs bash and GNU find.
RUN apk add openssl openssl-dev python3 elfutils-dev bash findutils
RUN \
ln -s /usr/include/gelf.h /sysroot/usr/include/. && \
ln -s /usr/include/libelf.h /sysroot/usr/include/. && \
ln -s /usr/lib/libelf.so /sysroot/usr/lib/. && \
ln -s /usr/include/openssl/ /sysroot/usr/include/. && \
ln -s /usr/lib/libcrypto.so /sysroot/usr/lib/.
# bpf/resolve_btfids doesn't respect HOSTCFLAGS
RUN cp -r /sysroot/usr/include/ /usr/local/
# TODO: something in the s390 build isn't respecting HOSTCC.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You don't set HOSTCC in the make command below, is that the problem?

RUN \
ln -s $(which clang) /usr/local/bin/gcc && \
ln -s $(which clang) /usr/local/bin/s390x-alpine-linux-musl-gcc

ARG arch=s390
#ARCH=s390 CROSS_COMPILE=s390x-linux-gnu- make CC=clang -j72 defconfig
RUN ARCH=${arch} HOSTCFLAGS=${SYSROOT} HOSTLDFLAGS=${SYSROOT} \
CROSS_COMPILE=s390x-alpine-linux-musl- \
CC=clang LLVM_IAS=0 \
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As we mentioned offline, if we upgrade to a 5.19 prerelease, we can drop the LLVM_IAS=0 here.

make -C linux-5.18-rc6 -s -j$(nproc) defconfig
RUN cd linux-5.18-rc6 && \
./scripts/config -d DEBUG_INFO_BTF
RUN ARCH=${arch} HOSTCFLAGS=${SYSROOT} HOSTLDFLAGS=${SYSROOT} \
CROSS_COMPILE=s390x-alpine-linux-musl- \
CC=clang LLVM_IAS=0 \
make -C linux-5.18-rc6 -s -j$(nproc) vmlinux
RUN llvm-profdata merge --output=${arch}.profdata.prof \
$(find ${LLVM_BUILD_DIR} -name \*.profraw) && \
find ${LLVM_BUILD_DIR} -name \*.profraw | xargs rm
# Leave openssl-dev and elfutils-dev since we have symlinks to them that the
# x86 build will use.
RUN apk del binutils-s390x openssl bash findutils

# Objtool on x86 diff from diffutils (BusyBox diff doesn't support -I used in
# tools/objtool/sync-check.sh).
RUN apk add diffutils findutils
# Objtool on x86 requires libelf (but we already did this above for s390).
#RUN apk add elfutils-dev openssl-dev
#### SYSROOT POISON
#RUN \
#ln -s /usr/include/libelf.h /sysroot/usr/include/. && \
#ln -s /usr/include/gelf.h /sysroot/usr/include/. && \
#ln -s /usr/lib/libelf.so /sysroot/usr/lib/. && \
#ln -s /usr/include/openssl /sysroot/usr/include/. && \
#ln -s /usr/lib/libcrypto.so /sysroot/usr/lib/.
#### SYSROOT POISON
#### Objtool doesn't respect HOSTCFLAGS
#RUN cp -r /sysroot/usr/include/ /usr/local/
ARG arch=x86_64
RUN ARCH=${arch} HOSTCFLAGS=${SYSROOT} HOSTLDFLAGS=${SYSROOT} \
KBUILDCFLAGS="-I /sysroot/usr/include" \
make -C linux-5.18-rc6 LLVM=1 -s -j$(nproc) V=1 defconfig vmlinux && \
llvm-profdata merge --output=${arch}.profdata.prof \
$(find ${LLVM_BUILD_DIR} -name \*.profraw) && \
find ${LLVM_BUILD_DIR} -name \*.profraw | xargs rm
#RUN apk del elfutils-dev openssl-dev diffutils
RUN apk del diffutils findutils

RUN llvm-profdata merge --output=profdata.prof \
$(find . -maxdepth 1 -name \*.profdata.prof)

### END STAGE4
### BEGIN STAGE5
FROM alpine:edge AS stage5

COPY --from=stage3 /usr/local/bin /usr/local/bin
COPY --from=stage3 /usr/local/lib /usr/local/lib
COPY --from=stage3 /usr/local/include /usr/local/include
RUN cd /usr/lib/ && \
for library in libc++abi.so.1 libc++.a libc++abi.a libc++.so.1 libunwind.so.1 libunwind.a; \
do ln -s "/usr/local/lib/x86_64-alpine-linux-musl/${library}" . ; \
done

### Linux
COPY --from=source linux-5.18-rc6.tar.gz .
RUN tar xf linux-5.18-rc6.tar.gz
RUN apk add make musl-dev rsync
RUN make -C linux-5.18-rc6 INSTALL_HDR_PATH=/sysroot/usr LLVM=1 -j$(nproc) headers_install
RUN apk del rsync musl-dev make

### Musl
COPY --from=source musl-1.2.3.tar.gz .
RUN tar xf musl-1.2.3.tar.gz
ARG MUSL_DIR=musl-1.2.3/build
RUN mkdir -p ${MUSL_DIR}
RUN cd ${MUSL_DIR} && \
CC=clang AR=llvm-ar RANLIB=llvm-ranlib \
../configure --prefix=/usr --syslibdir=/usr/lib
RUN apk add make
RUN make -C ${MUSL_DIR} -j$(nproc)
RUN make -C ${MUSL_DIR} -j$(nproc) DESTDIR=/sysroot install-headers
RUN make -C ${MUSL_DIR} -j$(nproc) DESTDIR=/sysroot install-libs
RUN apk del make

# Pause for a quick sanity check
COPY hello.c hello.cpp /
ARG SYSROOT=--sysroot=/sysroot
RUN clang ${SYSROOT} hello.c && ./a.out && \
clang ${SYSROOT} hello.c -static && ./a.out && \
clang++ ${SYSROOT} hello.cpp && ./a.out && \
clang++ ${SYSROOT} hello.cpp -static -lc++abi && ./a.out

### Zlib
COPY --from=source zlib-1.2.12.tar.gz .
RUN tar xf zlib-1.2.12.tar.gz
ARG ZLIB_DIR=zlib-1.2.12/build
RUN mkdir -p ${ZLIB_DIR}
RUN cd ${ZLIB_DIR} && \
CC="clang ${SYSROOT}" AR=llvm-ar ../configure --prefix=/sysroot/usr
RUN apk add make
RUN make -C ${ZLIB_DIR} -j$(nproc)
RUN make -C ${ZLIB_DIR} -j$(nproc) install
RUN apk del make
Comment on lines +228 to +267
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we build the sysroot twice? Wouldn't it be better to build it once in its own image then copy it as we need it?


### LLVM
COPY --from=source llvm-project-14.0.1.src.tar.xz .
RUN tar xf llvm-project-14.0.1.src.tar.xz && \
mv llvm-project-14.0.1.src llvm-project
RUN apk add cmake ninja python3
COPY stage5.cmake llvm-project/.
ARG LLVM_BUILD_DIR=llvm-project/llvm/build
RUN cmake \
-B ${LLVM_BUILD_DIR} \
-C llvm-project/stage5.cmake \
-D LLVM_DEFAULT_TARGET_TRIPLE=$(clang -print-target-triple) \
-S llvm-project/llvm \
-G Ninja
RUN ninja -C ${LLVM_BUILD_DIR} clang-tblgen
RUN ninja -C ${LLVM_BUILD_DIR} llvm-tblgen
RUN ninja -C ${LLVM_BUILD_DIR} clang
#RUN ninja -C ${LLVM_BUILD_DIR} install-clang install-lld
#RUN ninja -C ${LLVM_BUILD_DIR} install-clang-resource-headers
#RUN ninja -C ${LLVM_BUILD_DIR} \
#install-llvm-ar \
#install-llvm-nm \
#install-llvm-objcopy \
#install-llvm-objdump \
#install-llvm-ranlib \
#install-llvm-readelf \
#install-llvm-strip

#RUN apk del cmake ninja python3

# Final test
#RUN llvm-readelf -p .comment $(which clang) | grep -e clang -e LLD
#RUN llvm-readelf -p .comment $(which clang) | grep -v GCC
Comment on lines +285 to +300
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably want to keep these, probably also want to think about exporting the whole suit of tools...

3 changes: 1 addition & 2 deletions llvm-project/stage4.cmake
View file
Original file line number Diff line number Diff line change
@@ -59,8 +59,7 @@ set(LLVM_ENABLE_ZLIB "FORCE_ON" CACHE STRING "")
# This is necessary to statically link libc++ into clang.
set(LLVM_STATIC_LINK_CXX_STDLIB "1" CACHE STRING "")

# Just build stage3 to target the host. It's not the end product, so it won't
# be able to target all of the kernel targets we can build.
# Build all relevant targets.
set(LLVM_TARGETS_TO_BUILD "AArch64;ARM;Hexagon;Mips;PowerPC;RISCV;SystemZ;X86" CACHE STRING "")

# Necessary to avoid warnings about counter overflow.
97 changes: 97 additions & 0 deletions llvm-project/stage5.cmake
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
# Please try to keep these cmake variables alphabetically sorted.

# Enable optimizations, as opposed to a debug build.
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "")

# Explicitly use stage3's clang. Not necessary, just being explicit.
set(CMAKE_CXX_COMPILER "/usr/local/bin/clang++" CACHE FILEPATH "")
set(CMAKE_C_COMPILER "/usr/local/bin/clang" CACHE FILEPATH "")

# See above comment for CMAKE_CXX_COMPILER.
# Use the sysroot we've been building up.
set(CMAKE_CXX_FLAGS "--sysroot=/sysroot" CACHE STRING "")
set(CMAKE_C_FLAGS "--sysroot=/sysroot" CACHE STRING "")

# Statically link resulting executable.
set(CMAKE_EXE_LINKER_FLAGS "-static -lc++abi" CACHE STRING "")

# The compiler builtins are necessary.
set(COMPILER_RT_BUILD_BUILTINS ON CACHE BOOL "")

# GWP ASAN fails to build without libexecinfo-dev. Don't need it for stage5.
set(COMPILER_RT_BUILD_GWP_ASAN OFF CACHE BOOL "")

# Don't need libfuzzer, ever.
set(COMPILER_RT_BUILD_LIBFUZZER OFF CACHE BOOL "")

# Don't need memprof, ever.
set(COMPILER_RT_BUILD_MEMPROF OFF CACHE BOOL "")

# Don't need ORC, ever.
set(COMPILER_RT_BUILD_ORC OFF CACHE BOOL "")

# Explicitly enable profiling support. The implicit default is ON.
set(COMPILER_RT_BUILD_PROFILE ON CACHE BOOL "")

# Disable sanitizer support. Not necessary for stage5.
set(COMPILER_RT_BUILD_SANITIZERS OFF CACHE BOOL "")

# Don't need xray.
set(COMPILER_RT_BUILD_XRAY OFF CACHE BOOL "")

# Use libc++ from stage3.
# TODO: is CMAKE_CXX_FLAGS still necessary if this is set?
set(LLVM_ENABLE_LIBCXX ON CACHE BOOL "")

# Use lld from stage3.
set(LLVM_ENABLE_LLD ON CACHE BOOL "")

# TODO: clang segfaults when building llvm-tblgen. Do we need to use
# `--ulimit nofile=65536` for `docker build`?
# Build LLVM with thinLTO. This requires we bump the default memory limit up.
#set(LLVM_ENABLE_LTO "Thin" CACHE STRING "")

# Build clang, lld, and compiler-rt.
set(LLVM_ENABLE_PROJECTS "clang;lld;compiler-rt" CACHE STRING "")

# FORCE_ON causes the build to fail if zlib is not found in the environment
# during configuration, rather than much later during link.
set(LLVM_ENABLE_ZLIB "FORCE_ON" CACHE STRING "")

# Consume PGO training data from stage4.
set(LLVM_PROFDATA_FILE "profdata.prof" CACHE FILEPATH "")

# This is necessary to statically link libc++ into clang.
set(LLVM_STATIC_LINK_CXX_STDLIB "1" CACHE STRING "")

# Build all relevant targets.
set(LLVM_TARGETS_TO_BUILD "AArch64;ARM;Hexagon;Mips;PowerPC;RISCV;SystemZ;X86" CACHE STRING "")

# Set clang's default --stdlib= to libc++.
set(CLANG_DEFAULT_CXX_STDLIB "libc++" CACHE STRING "")

# Set clang's default -fuse-ld= to lld.
set(CLANG_DEFAULT_LINKER "lld" CACHE STRING "")

# Have clang default to llvm-objcopy.
set(CLANG_DEFAULT_OBJCOPY "llvm-objcopy" CACHE STRING "")

# Set clang's default --rtlib= to compiler-rt.
set(CLANG_DEFAULT_RTLIB "compiler-rt" CACHE STRING "")

# Set clang's default --unwindlib= to libunwind.
set(CLANG_DEFAULT_UNWINDLIB "libunwind" CACHE STRING "")

# Disable arc migrate. We don't use that, ever.
set(CLANG_ENABLE_ARCMT OFF CACHE BOOL "")

# Disable static analyzer. Don't need it for stage5.
set(CLANG_ENABLE_STATIC_ANALYZER OFF CACHE BOOL "")

# Disable plugin support. Don't need it, ever.
set(CLANG_PLUGIN_SUPPORT OFF CACHE BOOL "")

# Because we're using --prefix=/sysroot/usr, zlib gets installed to a
# non-standard path.
set(ZLIB_INCLUDE_DIR "/sysroot/usr/include/zlib.h" CACHE FILEPATH "")
set(ZLIB_LIBRARY "/sysroot/usr/lib/libz.a" CACHE FILEPATH "")