Merge pull request #11 from swift-nav/silverjam/vanilla-clang-6

Support for building vanilla clang 6 cross compiler

Author: Jason Mobarak

.after_failure.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -x
+
+if [[ -e /tmp/base.bash.log ]]; then
+  cp /tmp/base.bash.log /tmp/$VARIANT-base.bash.log
+  ./publish.sh /tmp/$VARIANT-base.bash.log
+  tail -n 200 /tmp/base.bash.log
+fi
+
+if [[ -e /tmp/build.bash.log ]]; then
+  cp /tmp/build.bash.log /tmp/$VARIANT-build.bash.log
+  ./publish.sh /tmp/$VARIANT-build.bash.log
+  tail -n 200 /tmp/build.bash.log;
+fi
+
+if [[ -e /tmp/build_example.bash.log ]]; then
+  cp /tmp/build_example.bash.log /tmp/$VARIANT-build_example.bash.log
+  ./publish.sh /tmp/$VARIANT-build_example.bash.log
+  tail -n 200 /tmp/build_example.bash.log;
+fi

.travis.sh

@@ -27,7 +27,7 @@ fi
   done
 )&
 TICKER_PID=$!
-trap 'kill ${TICKER_PID:-} ${BUILD_PID:-}' EXIT
+trap 'kill ${TICKER_PID:-} ${BUILD_PID:-} &>/dev/null || :' EXIT
 
 ### base.bash
 
@@ -41,9 +41,9 @@ fi
 
 ### build.bash
 
-echo 'Running build of llvm-obfuscator... '
+echo "Running build of llvm-$VARIANT... "
 
-make "ARCH=$ARCH" NO_TTY=y build &>/tmp/build.bash.log &
+make NO_TTY=y "ARCH=$ARCH" "VARIANT=$VARIANT" build &>/tmp/build.bash.log &
 BUILD_PID=$!
 
 wait $BUILD_PID
@@ -52,7 +52,7 @@ echo 'DONE.'
 
 ### build_example.bash
 
-if [[ $ARCH == *arm* ]]; then
+if [[ $ARCH == *arm* ]] && [[ $VARIANT == obfuscator ]]; then
 
   echo -n 'Building example project... '
   make NO_TTY=y build-example &>/tmp/build_example.bash.log

.travis.yml

@@ -10,25 +10,39 @@ branches:
 services:
  - docker
 
-before_install:
- - pip install --user --upgrade awscli
- - sudo apt-get update
- - sudo apt-get install -y jq
-
-script:
- - ./.travis.sh
-
-after_success:
- - travis_wait tar -C output -cJf "llvm-obfuscator-$ARCH.txz" .
- - ./publish.sh llvm-obfuscator-$ARCH.txz
-
-after_failure:
- - if [[ -e /tmp/base.bash.log ]]; then echo "*** base.bash.log ***"; fi
- - if [[ -e /tmp/base.bash.log ]]; then tail -n 500 /tmp/base.bash.log; fi
- - if [[ -e /tmp/build.bash.log ]]; then echo "*** build.bash.log ***"; fi
- - if [[ -e /tmp/build.bash.log ]]; then tail -n 500 /tmp/build.bash.log; fi
- - if [[ -e /tmp/build_example.bash.log ]]; then echo "*** build_example.bash.log ***"; fi
- - if [[ -e /tmp/build_example.bash.log ]]; then tail -n 500 /tmp/build_example.bash.log; fi
+matrix:
+  include:
+    - env:
+        - VARIANT=obfuscator
+        - ARCH=arm,x86
+      before_install:
+        - pip install --user --upgrade awscli
+        - sudo apt-get update
+        - sudo apt-get install -y jq
+      script:
+        - ./pull_ccache.bash --variant=$VARIANT --arch=$ARCH
+        - VARIANT=$VARIANT ./.travis.sh
+      after_success:
+        - travis_wait tar -C output -cJf "llvm-$VARIANT-$ARCH.txz" .
+        - ./publish.sh llvm-$VARIANT-$ARCH.txz
+      after_failure:
+        - ./.after_failure.sh
+
+    - env:
+        - VARIANT=vanilla
+        - ARCH=arm,x86
+      before_install:
+        - pip install --user --upgrade awscli
+        - sudo apt-get update
+        - sudo apt-get install -y jq
+      script:
+        - ./pull_ccache.bash --variant=$VARIANT --arch=$ARCH
+        - VARIANT=$VARIANT ./.travis.sh
+      after_success:
+        - travis_wait tar -C output -cJf "llvm-$VARIANT-$ARCH.txz" .
+        - ./publish.sh llvm-$VARIANT-$ARCH.txz
+      after_failure:
+        - ./.after_failure.sh
 
 env:
   global:
@@ -44,5 +58,3 @@ env:
     - secure: "T1wlLXyhfGnMNMPtHv2FspxsWG09hVZgaucSJ6IxN+PZBbgjLb/TMSlqqLXfIzx6s0SuQIzX1xZE8lb/uOzfLtjmxBdc4dGYLW+qR4AwkghQl2DGMK+hzGAi9ZbeCGKWtX7cN4WxLN1tpcTyQBCC8LcnP0xVxd+7Znytdezmlr86ce2d8dmUo+kyLdN+vhEUnch9zhMpuZI+TxV2HQqCS/6e31myP+3Xzw3B6o6uF8dYZ0QlWlNU4KjUNNAD7cLqkxNd38E7wFrQJdyrk03/hZanatl2mcL7Te5Uxo5sWN5WteTNwq8dCKXiEXpkEqP1++4wWTQDeQWoPjHlhHzO4MxGAdEUnlFWafsV78OJvr/gwV0UxDFmfadPwGG/xPPJMyNJjXwDyeztswknbVaNZdbRXp3m7Iri+bWCvOZE9N7rGde/5q0JOz9rwJXRDMPnp7+lyupHQQIdswNpkFraHz3eZ/h4n+Dkr3RtkW0M6ZpPR+/r18uD7jwm/Ye73E9wzq1b6sL0knX/OEwBv3FVvaBaA4SWzb1glDlru8BKnZfVIZ/UxYas6FqGN6H9ROyCHtva86ffv7HvvnB/gUqcOfY5si7WvtPVApIc9kfILcSWEchRWTHQIeUOdEUBfoq8/Km0BL6gguOO9Ge9zzOrdo0NThh8x7NoE0DhHdQctJY="
     # DOCKER_PASS
     - secure: "qEmUT+mBsHjLkrBYMZWiLl8+ZXq9UAv+DyMDww+N74hPtGiZxMqLuVIFq94qR/WhiI9zd7k+tjhy6TcgD8rpKoQnQSGXcotFGq6typ3UWEV0ekt40IMV5nBqopqGGIBVIHNfXjq0OTeYTF2eGncpNnznJHAQi8Dvj7vD99y5GA5RoZGSdsiqTyxBGMvZ0cE0QApkb1MH3lBVRzBuh6dotpa1RBlGR+i2xbcH3nAqy5RB38wZgMedAPnlUfg+uB+IQiF8+hbuSmHzADiabNLBkkB/UCOz59k/JkaeN+2gnyP6/so+SpOe1BKfpxKBcN74O4n056ZLDBRjTod2YSc49dKqxhZhQijwNJfFi3Tt5iNrFYdU3KPeqqfNLoQKmbBTKjGXbxHHXm3lnUuCTZH0Yd9QGKSAwn5kxDRy6DuyNMUGwFtu8YxzirwTJiQR6ayodMCMWjZCXuxUc0I5I2Y64CsofRWkuMbznjE4wJwptVPzTwIxkZnMt+aWVoQTXBtZVpiHf4CTJb65dyLDXeMOwhYnKzwLUrvvHHpOm7GygvAbbeDAzHumTDvt7s5nl2VwHNLmTK3TizvvpQCucIjG8M8dEd7iu4VRZh/rxGxV9MJ2qUNETa6hnfX+Z2xaLTcgYAG5z7p60sThPfFfYLopdJL1dO5eA/EncPzuQuh087w="
-  matrix:
-    - ARCH=x86,arm

Dockerfile renamed to Dockerfile.base

@@ -23,19 +23,28 @@ RUN    apt-get update \
     && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
     && echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-4.0 main" \
           >/etc/apt/sources.list.d/llvm40.list \
+    && echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main" \
+          >/etc/apt/sources.list.d/llvm60.list \
     && apt-get update \
-    && apt-get upgrade \
+    && apt-get upgrade -y \
     && apt-get install -y libcurl4-openssl-dev \
                           build-essential \
                           bison \
                           flex \
                           ninja-build \
-                          llvm-4.0 \
-                          llvm-4.0-dev \
-                          clang-4.0 \
                           git \
+                          m4 \
+                          gperf \
+                          gawk \
+                          ncurses-dev \
+                          texinfo \
+                          help2man \
                           binutils-dev \
+                          libpthread-stubs0-dev \
+                          libtinfo-dev \
                           python \
+                          python-dev \
+                          ccache \
                           binutils-multiarch-dev \
                           g++-4.8-arm-linux-gnueabihf \
                           gcc-4.8-arm-linux-gnueabihf \
@@ -57,10 +66,9 @@ RUN    apt-get update \
     && tar -xzf cmake-3.10.1.tar.gz \
     && cd cmake-3.10.1 \
     && ./configure \
-    && make -j4 \
-    && make install \
+    && make -j8 \
+    && make -j8 install \
     && cd .. && rm -rf cmake-* \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
+    && apt-get clean
 
 # EOF

Dockerfile.obfuscator

@@ -0,0 +1,14 @@
+# Copyright (C) 2017-2018 Swift Navigation Inc.
+# Contact: Swift Navigation <[email protected]>
+#
+# This source is subject to the license found in the file 'LICENSE' which must
+# be be distributed together with this source. All other rights reserved.
+#
+# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND,
+# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
+
+ARG DOCKER_NAMETAG
+FROM $DOCKER_NAMETAG
+
+RUN    apt-get install -y llvm-4.0 llvm-4.0-dev clang-4.0 libclang-4.0-dev

Dockerfile.vanilla

@@ -0,0 +1,14 @@
+# Copyright (C) 2017-2018 Swift Navigation Inc.
+# Contact: Swift Navigation <[email protected]>
+#
+# This source is subject to the license found in the file 'LICENSE' which must
+# be be distributed together with this source. All other rights reserved.
+#
+# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND,
+# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
+
+ARG DOCKER_NAMETAG
+FROM $DOCKER_NAMETAG
+
+RUN apt-get install -y llvm-6.0 llvm-6.0-dev clang-6.0 libclang-6.0-dev

Makefile

@@ -5,6 +5,10 @@ SCRIPTS := .travis.sh \
 					 publish.sh \
 					 run_build_shell.bash \
 					 stage_sysroot.bash \
+					 do_clang_build.bash \
+					 push_ccache.bash \
+					 pull_ccache.bash \
+					 s3_download.bash \
 
 all: check base build
 
@@ -27,7 +31,7 @@ base: check-base
 	$(CURDIR)/base.bash $(NO_TTY_ARG)
 
 build: check-build
-	$(CURDIR)/build.bash $(NO_TTY_ARG) --arch=$(ARCH)
+	$(CURDIR)/build.bash $(NO_TTY_ARG) --arch=$(ARCH) --variant=$(VARIANT)
 
 stage: check-stage_sysroot
 	$(CURDIR)/stage_sysroot.bash $(NO_TTY_ARG)
@@ -38,7 +42,25 @@ build-example: check-build_example
 run: check-run_build_shell
 	$(CURDIR)/run_build_shell.bash $(NO_TTY_ARG)
 
-clean:
+clean-vanilla-build:
+	docker volume rm vanilla-llvm-build || :
+	docker volume rm vanilla-llvm-ccache || :
+
+clean-vanilla-src:
+	docker volume rm vanilla-llvm || :
+
+clean-vanilla: clean-vanilla-build clean-vanilla-src
+
+clean-obfuscator-build:
 	docker volume rm obfuscator-llvm-build || :
+	docker volume rm obfuscator-llvm-ccache || :
+
+clean-obfuscator-src:
 	docker volume rm obfuscator-llvm || :
+
+clean-obfuscator: clean-obfuscator-build clean-obfuscator-src
+
+clean-build: clean-vanilla clean-obfuscator
+
+clean: clean-build
 	sudo rm -rf output/*

base.bash

@@ -56,11 +56,16 @@ fi
 
 docker build \
   --force-rm --no-cache \
-  -f Dockerfile -t "$DOCKER_NAMETAG" .
+  -f Dockerfile.base -t "$DOCKER_NAMETAG" .
 
-if [[ -n "${DOCKER_USER:-}" ]] && [[ -n "${DOCKER_PASS:-}" ]]; then
-  echo "$DOCKER_PASS" | docker login --username="$DOCKER_USER" --password-stdin
-  docker push "$DOCKER_NAMETAG"
-else
-  echo "WARNING: not pushing new image to Docker Hub..." >&2
-fi
+docker build \
+  --force-rm --no-cache \
+  --build-arg "DOCKER_NAMETAG=$DOCKER_NAMETAG" \
+  -f Dockerfile.vanilla -t "$DOCKER_NAMETAG-vanilla" .
+
+docker build \
+  --force-rm --no-cache \
+  --build-arg "DOCKER_NAMETAG=$DOCKER_NAMETAG" \
+  -f Dockerfile.obfuscator -t "$DOCKER_NAMETAG-obfuscator" .
+
+./push.bash

build.bash

@@ -23,42 +23,70 @@ NO_TTY=
 while [[ $# -gt 0 ]]; do
   case $1 in
 
-  --arch=x86)       ARCH="X86";       shift ;;
-  --arch=arm)       ARCH="ARM";       shift ;;
-  --arch=arm,x86)   ARCH="ARM\\;X86";   shift ;;
-  --arch=x86,arm)   ARCH="X86\\;ARM";   shift ;;
+  --arch=x86)           ARCH="X86";           shift ;;
+  --arch=arm)           ARCH="ARM";           shift ;;
+  --arch=arm,x86)       ARCH="ARM\\;X86";     shift ;;
+  --arch=x86,arm)       ARCH="ARM\\;X86";     shift ;;
 
-  --verbose)        VERBOSE="-v";     shift ;;
-  --no-tty)         NO_TTY=--no-tty;  shift ;;
+  --variant=vanilla)    VARIANT="vanilla";    shift ;;
+  --variant=obfuscator) VARIANT="obfuscator"; shift ;;
 
-  *)                                  shift ;;
+  --verbose)            VERBOSE="-v";         shift ;;
+  --no-tty)             NO_TTY=--no-tty;      shift ;;
+
+  *)                                          shift ;;
   esac
 done
 
 if [[ -z "${ARCH:-}" ]]; then
-  echo "Error: must specify --arch=<arm|x86>"
+  echo "Error: must specify --arch=<arm|x86|arm,x86|x86,arm>"
   exit 1
 fi
 
-set -x
-#        -DLLVM_BUILD_TOOLS:BOOL=FALSE \
+if [[ -z "${VARIANT:-}" ]]; then
+  echo "Error: must a variant to build --variant=<vanilla|obfuscator>"
+  exit 1
+fi
+
+CXX_FLAGS="-L/toolchain/x86/x86_64-buildroot-linux-gnu/sysroot/lib \
+  -L/toolchain/x86/x86_64-buildroot-linux-gnu/sysroot/usr/lib \
+  -I/toolchain/x86/lib/gcc/x86_64-buildroot-linux-gnu/6.4.0/plugin/include"
+
+if [[ "$VARIANT" == "obfuscator" ]]; then
+  LLVM_REPO="https://github.com/obfuscator-llvm/obfuscator.git"
+  LLVM_BRANCH="llvm-4.0"
+  CLANG_REPO=""
+  CLANG_TOOLS_EXTRA_REPO=""
+  CPP_WRAPPER_DEFINE="-DCMAKE_CXX_COMPILER=/bin/cpp_wrapper"
+  PATCH_COMMAND="{ git apply /patches/*.patch || : ; }"
+  COMPILE_CPP_WRAPPER="cp -v /this_dir/cpp_wrapper.c /work/cpp_wrapper.c \
+                       && gcc -std=c99 -O3 -Wall /work/cpp_wrapper.c -o /bin/cpp_wrapper"
+else
+  LLVM_REPO="https://github.com/llvm-mirror/llvm.git"
+  CLANG_REPO="https://github.com/llvm-mirror/clang.git"
+  CLANG_TOOLS_EXTRA_REPO="https://github.com/llvm-mirror/clang-tools-extra.git"
+  LLVM_BRANCH="release_60"
+  CPP_WRAPPER_DEFINE="-DCMAKE_CXX_COMPILER=/toolchain/x86/bin/x86_64-linux-g++"
+  PATCH_COMMAND="true"
+  COMPILE_CPP_WRAPPER="true"
+  CXX_FLAGS+=" -I/work/$VARIANT-llvm/tools/clang/include"
+  CXX_FLAGS+=" -I/work/build/tools/clang/include"
+fi
 
 CMAKE_COMMAND="\
     cmake -G Ninja \
-        /work/obfuscator-llvm \
-        -DCMAKE_INSTALL_PREFIX=/opt/llvm-obfuscator \
+        /work/$VARIANT-llvm \
+        -DCMAKE_INSTALL_PREFIX=/opt/llvm-$VARIANT \
         -DLLVM_TARGETS_TO_BUILD=$ARCH \
-        -DCMAKE_CXX_FLAGS='-DENDIAN_LITTLE=1 -I/toolchain/x86/lib/gcc/x86_64-buildroot-linux-gnu/6.4.0/plugin/include' \
+        -DCMAKE_CXX_FLAGS='-DENDIAN_LITTLE=1 $CXX_FLAGS' \
+        -DLLVM_CCACHE_BUILD=ON \
+        $CPP_WRAPPER_DEFINE \
         -DCMAKE_C_COMPILER=/toolchain/x86/bin/x86_64-linux-gcc \
-        -DCMAKE_CXX_COMPILER=/bin/cpp_wrapper \
         -DCMAKE_BUILD_TYPE=Release \
         -DLLVM_BINUTILS_INCDIR=/usr/include \
-        -DLLDB_DISABLE_CURSES:BOOL=TRUE \
         -DLLVM_ENABLE_TERMINFO=0 \
         -DLLVM_INCLUDE_TESTS=OFF"
 
-PATCH_COMMAND="{ git apply /patches/*.patch || : ; }"
-
 if [[ -z "$NO_TTY" ]]; then
   INTERACTIVE=("-i" "-t")
 else
@@ -70,23 +98,21 @@ docker run ${INTERACTIVE[@]:-} --rm \
     -v "$PWD/output/opt:/opt" \
     -v "$PWD/patches:/patches" \
     -v "$PWD:/this_dir" \
-    -v obfuscator-llvm:/work/obfuscator-llvm \
-    -v obfuscator-llvm-build:/work/build \
-    "$DOCKER_NAMETAG" \
-    /bin/bash -c "if [ ! -d /work/obfuscator-llvm/.git ]; then \
-                      git clone --depth=1 --single-branch -b llvm-4.0 \
-                        https://github.com/obfuscator-llvm/obfuscator.git \
-                        obfuscator-llvm;
-                  else \
-                    (cd /work/obfuscator-llvm && git pull); \
-                  fi \
-                  && cp -v /this_dir/cpp_wrapper.c /work/cpp_wrapper.c \
-                  && gcc -std=c99 -O3 -Wall /work/cpp_wrapper.c -o /bin/cpp_wrapper \
-                  && cd /work/obfuscator-llvm \
-                  && $PATCH_COMMAND \
-                  && cd /work/build \
-                  && $CMAKE_COMMAND \
-                  && ninja $VERBOSE \
-                  && ninja $VERBOSE install"
-
-./stage_sysroot.bash $NO_TTY
+    -v $VARIANT-llvm-ccache:/work/ccache \
+    -v $VARIANT-llvm:/work/$VARIANT-llvm \
+    -v $VARIANT-llvm-build:/work/build \
+    -e VARIANT=$VARIANT -e ARCH=$ARCH \
+    -e VERBOSE=$VERBOSE -e NO_TTY=$NO_TTY \
+    -e CPP_WRAPPER_DEFINE=$CPP_WRAPPER_DEFINE \
+    -e CMAKE_COMMAND="$CMAKE_COMMAND" \
+    -e LLVM_REPO=$LLVM_REPO \
+    -e LLVM_BRANCH=$LLVM_BRANCH \
+    -e CLANG_REPO=$CLANG_REPO \
+    -e CLANG_TOOLS_EXTRA_REPO=$CLANG_TOOLS_EXTRA_REPO \
+    -e PATCH_COMMAND=$PATCH_COMMAND \
+    -e COMPILE_CPP_WRAPPER=$COMPILE_CPP_WRAPPER \
+    -e CCACHE_DIR=/work/ccache \
+    "$DOCKER_NAMETAG-$VARIANT" \
+    /bin/bash -c "/this_dir/do_clang_build.bash"
+
+./stage_sysroot.bash $NO_TTY "--variant=$VARIANT"

build_tag

@@ -0,0 +1 @@
+v17