diff --git a/.dockerignore b/.dockerignore index a0cabb7fb05..49f54684751 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,4 +4,5 @@ **/Dockerfile **/*.md +build docs diff --git a/.github/workflows/ci-ptf.yml b/.github/workflows/ci-ptf.yml index dd03a3c3315..b73e343e0aa 100644 --- a/.github/workflows/ci-ptf.yml +++ b/.github/workflows/ci-ptf.yml @@ -37,10 +37,16 @@ jobs: with: submodules: recursive + - name: ccache + uses: hendrikmuhs/ccache-action@v1 + with: + key: test-${{ env.UNIFIED }}-${{ runner.os }} + max-size: 1000M + - name: Build (Linux) run: | - tools/start_ccache docker build --network host -t p4c --build-arg MAKEFLAGS=-j8 --build-arg IMAGE_TYPE=test --build-arg ENABLE_UNIFIED_COMPILATION=$UNIFIED --build-arg ENABLE_GMP=$ENABLE_GMP --build-arg INSTALL_PTF_EBPF_DEPENDENCIES=ON . + ./tools/export_ccache.sh - name: Run PTF tests for eBPF backend (Linux) run: | diff --git a/.github/workflows/ci-static-build-test.yml b/.github/workflows/ci-static-build-test.yml index 3dfd9449e1a..1412f9a732a 100644 --- a/.github/workflows/ci-static-build-test.yml +++ b/.github/workflows/ci-static-build-test.yml @@ -7,6 +7,7 @@ on: branches: [ main ] +# Cancel any preceding run on the pull request. concurrency: group: static-build-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} @@ -23,7 +24,14 @@ jobs: - uses: actions/checkout@v2 with: submodules: recursive + + - name: ccache + uses: hendrikmuhs/ccache-action@v1 + with: + key: static-${{ runner.os }} + max-size: 1000M + - name: Build (Linux) run: | - tools/start_ccache - docker build --network ccache_network -t p4c --build-arg IMAGE_TYPE=build --build-arg ENABLE_UNIFIED_COMPILATION=ON --build-arg BUILD_STATIC_RELEASE=ON --build-arg ENABLE_GMP=ON . + docker build -t p4c --build-arg IMAGE_TYPE=test --build-arg ENABLE_UNIFIED_COMPILATION=ON --build-arg BUILD_STATIC_RELEASE=ON --build-arg ENABLE_GMP=ON . + ./tools/export_ccache.sh diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 385d23430c9..a489b2967ac 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -30,13 +30,19 @@ jobs: with: submodules: recursive + - name: ccache + uses: hendrikmuhs/ccache-action@v1 + with: + key: test-${{ matrix.unified }}-${{ runner.os }} + max-size: 1000M + - name: Build (Ubuntu Linux) run: | - tools/start_ccache # To flush out issues with unified vs. non-unified builds, # do a non-unified build before continuing with the rest, # which produces a unified build. - docker build --network ccache_network -t p4c --build-arg IMAGE_TYPE=test --build-arg ENABLE_UNIFIED_COMPILATION=${{matrix.unified}} --build-arg ENABLE_GMP=${{ matrix.enable_gmp }} . + docker build -t p4c --build-arg IMAGE_TYPE=test --build-arg ENABLE_UNIFIED_COMPILATION=${{ matrix.unified }} --build-arg ENABLE_GMP=${{ matrix.enable_gmp }} . + ./tools/export_ccache.sh # run with sudo (...) --privileged # this is needed to create network namespaces for the ebpf tests. @@ -73,11 +79,18 @@ jobs: - name: Install dependencies (Fedora Linux) run: tools/install_fedora_deps.sh + - name: ccache + uses: hendrikmuhs/ccache-action@v1 + with: + key: fedora-test-${{ runner.os }} + max-size: 1000M + - name: Build p4c (Fedora Linux) run: | - ./bootstrap.sh -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_UNIFIED_COMPILATION=${{matrix.unified}} -DENABLE_GMP=${{ matrix.enable_gmp }} + ./bootstrap.sh -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_UNIFIED_COMPILATION=${{ matrix.unified }} -DENABLE_GMP=${{ matrix.enable_gmp }} make -j2 -C build + - name: Run p4c tests (Fedora Linux) run: ctest --output-on-failure --schedule-random working-directory: ./build @@ -101,6 +114,12 @@ jobs: with: submodules: recursive + - name: ccache + uses: hendrikmuhs/ccache-action@v1 + with: + key: test-${{ runner.os }} + max-size: 1000M + - name: Install dependencies (MacOS) run: | tools/install_mac_deps.sh @@ -113,7 +132,7 @@ jobs: # which produces a unified build. # On MacOS we have noticed problems with the C++ # garbage-collector, so we disable it - ./bootstrap.sh -DENABLE_GC=OFF -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_UNIFIED_COMPILATION=${{matrix.unified}} -DENABLE_GMP=${{ matrix.enable_gmp }} && cd build && make -j2 + ./bootstrap.sh -DENABLE_GC=OFF -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_UNIFIED_COMPILATION=${{ matrix.unified }} -DENABLE_GMP=${{ matrix.enable_gmp }} && cd build && make -j2 - name: Run tests (MacOS) run: | diff --git a/.github/workflows/ci-validation.yml b/.github/workflows/ci-validation.yml index b501597a469..580071d4445 100644 --- a/.github/workflows/ci-validation.yml +++ b/.github/workflows/ci-validation.yml @@ -27,10 +27,16 @@ jobs: with: submodules: recursive + - name: ccache + uses: hendrikmuhs/ccache-action@v1 + with: + key: validation-${{ runner.os }} + max-size: 1000M + - name: Build (Linux) run: | - tools/start_ccache - docker build --network ccache_network -t p4c --build-arg IMAGE_TYPE=test --build-arg ENABLE_UNIFIED_COMPILATION=$UNIFIED --build-arg ENABLE_GMP=$ENABLE_GMP --build-arg VALIDATION=ON . || echo "BUILD_SUCCESS=false" >> $GITHUB_ENV + docker build -t p4c --build-arg IMAGE_TYPE=test --build-arg ENABLE_UNIFIED_COMPILATION=$UNIFIED --build-arg ENABLE_GMP=$ENABLE_GMP --build-arg VALIDATION=ON . || echo "BUILD_SUCCESS=false" >> $GITHUB_ENV + ./tools/export_ccache.sh - name: Validate if: env.BUILD_SUCCESS == 'true' diff --git a/Dockerfile b/Dockerfile index 3420367e3f6..6ed2b799a2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,8 @@ ARG VALIDATION=OFF # This creates a release build that includes link time optimization and links # all libraries statically. ARG BUILD_STATIC_RELEASE=OFF +# Toggle usage of the GNU Multiple Precision Arithmetic Library. +ARG ENABLE_GMP=ON # No questions asked during package installation. ARG DEBIAN_FRONTEND=noninteractive # Whether to install dependencies required to run PTF-ebpf tests @@ -25,5 +27,6 @@ ARG INSTALL_PTF_EBPF_DEPENDENCIES=OFF # Delegate the build to tools/ci-build. COPY . /p4c/ +RUN chmod u+x /p4c/tools/ci-build.sh && /p4c/tools/ci-build.sh +# Set the workdir after building p4c. WORKDIR /p4c/ -RUN chmod u+x tools/ci-build.sh && tools/ci-build.sh diff --git a/tools/ci-build.sh b/tools/ci-build.sh index a176b686920..f4c24587461 100644 --- a/tools/ci-build.sh +++ b/tools/ci-build.sh @@ -46,8 +46,6 @@ export P4C_PIP_PACKAGES="ipaddr \ ply==3.8 \ scapy==2.4.5" - - apt-get update apt-get install -y --no-install-recommends \ ${P4C_DEPS} \ @@ -55,6 +53,11 @@ apt-get install -y --no-install-recommends \ ${P4C_RUNTIME_DEPS} \ git +# TODO: Remove this rm -rf line once the ccache memcache config is removed. +rm -rf /usr/local/etc/ccache.conf +/usr/local/bin/ccache --set-config cache_dir=/p4c/.ccache +/usr/local/bin/ccache --set-config max_size=1G + # we want to use Python as the default so change the symlinks ln -sf /usr/bin/python3 /usr/bin/python ln -sf /usr/bin/pip3 /usr/bin/pip @@ -65,7 +68,7 @@ pip3 install $P4C_PIP_PACKAGES # Build libbpf for eBPF tests. cd /p4c backends/ebpf/build_libbpf -cd - +cd /p4c function install_ptf_ebpf_test_deps() ( export P4C_PTF_PACKAGES="gcc-multilib \ diff --git a/tools/export_ccache.sh b/tools/export_ccache.sh new file mode 100755 index 00000000000..ac870d636b3 --- /dev/null +++ b/tools/export_ccache.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker cp $(docker create --rm p4c):/p4c/.ccache . diff --git a/tools/install_fedora_deps.sh b/tools/install_fedora_deps.sh index d668778d166..f02b548ac2d 100755 --- a/tools/install_fedora_deps.sh +++ b/tools/install_fedora_deps.sh @@ -13,6 +13,7 @@ sudo dnf install -y -q \ boost-system \ boost-test \ boost-thread \ + ccache \ clang \ cmake \ cpp \ @@ -69,7 +70,7 @@ pushd "${tmp_dir}" git clone --depth=1 https://github.com/p4lang/behavioral-model cd behavioral-model ./autogen.sh -./configure --with-pdfixed --with-thrift --with-pi --with-stress-tests --enable-debugger +./configure --with-pdfixed --with-thrift --with-pi --with-stress-tests --enable-debugger CC="ccache gcc" CXX="ccache g++" make make install-strip popd diff --git a/tools/install_mac_deps.sh b/tools/install_mac_deps.sh index f5aaebce3dc..6073ef978cd 100755 --- a/tools/install_mac_deps.sh +++ b/tools/install_mac_deps.sh @@ -7,19 +7,10 @@ if [[ ! -x $BREW ]]; then fi $BREW update -$BREW install autoconf automake bdw-gc bison boost ccache cmake git \ +$BREW install autoconf automake bdw-gc bison boost ccache cmake \ libtool openssl pkg-config python -$BREW install gmp --c++11 - -# Install specific version of Protobuf, since newer versions break compatibility -wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protobuf-all-3.6.1.tar.gz -tar xfz protobuf-all-3.6.1.tar.gz -cd protobuf-3.6.1 -./configure -make -sudo make install -cd .. -rm -rf protobuf-3.6.1 +$BREW install protobuf +$BREW install gmp # Prefer Homebrew's bison over the macOS-provided version $BREW link --force bison diff --git a/tools/start_ccache b/tools/start_ccache deleted file mode 100755 index 4dbed290e11..00000000000 --- a/tools/start_ccache +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env sh - -# Create a network on which the ccache server will be exposed. Docker builds -# will connect to this network using `--network ccache_network` to access the -# server. -docker network create ccache_network - -# Pull down the ccache server. -docker pull webhare/ccache-memcached-server:latest - -# Run the ccache server, which will store its data in `$HOME/.ccache`, and make -# it available on the network. This container will start in the background, so -# it'll be available for any build that runs after this point. -docker run --rm \ - --name ccache \ - --network ccache_network \ - --network-alias ccache \ - -v $HOME/.ccache:/opt/memcache-data \ - --detach \ - webhare/ccache-memcached-server:latest