Overhaul bootstrap.sh script:

- Initial support for build variants (e.g., LTO, sanitizers).
- Build libgo deps from bootstrap script. We will need to do this for
  build variants.
- Filter everything from libgo's make output except package names. This
  makes the output much easier to read.

Author: pcc

Makefile

@@ -14,13 +14,16 @@ check-libgo: bootstrap
 check-llgo: bootstrap
 	$(llvmdir)/bin/llvm-lit -s test
 
-workdir/.bootstrap-stamp: workdir/.update-libgo-stamp workdir/.update-clang-stamp bootstrap.sh build/*.go cmd/gllgo/*.go cmd/cc-wrapper/*.go debug/*.go irgen/*.go ssaopt/*.go
+workdir/.bootstrap-stamp: workdir/.build-libgodeps-stamp bootstrap.sh build/*.go cmd/gllgo/*.go cmd/cc-wrapper/*.go debug/*.go irgen/*.go ssaopt/*.go
 	./bootstrap.sh $(bootstrap) -j$(j)
 
+workdir/.build-libgodeps-stamp: workdir/.update-clang-stamp workdir/.update-libgo-stamp bootstrap.sh
+	./bootstrap.sh libgodeps -j$(j)
+
 workdir/.update-clang-stamp: update_clang.sh
 	./update_clang.sh
 
-workdir/.update-libgo-stamp: workdir/.update-clang-stamp update_libgo.sh
+workdir/.update-libgo-stamp: update_libgo.sh
 	./update_libgo.sh
 
 .SUFFIXES:

README.md

@@ -22,6 +22,7 @@ If you built a newer GCC following the linked instructions above, you will need
     export LD_LIBRARY_PATH=/path/to/gcc-inst/lib64:$LD_LIBRARY_PATH
     export CC=`which gcc`
     export CXX=`which g++`
+    export LIBGO_CFLAGS=--gcc-toolchain=/path/to/gcc-inst
 
 To build and install llgo:
 

bootstrap.sh

@@ -1,4 +1,6 @@
-#!/bin/sh -e
+#!/usr/bin/env bash
+
+set -e
 
 llgodir=$(dirname "$0")
 llgodir=$(cd "$llgodir" && pwd)
@@ -7,73 +9,136 @@ workdir=$llgodir/workdir
 gofrontenddir=$workdir/gofrontend
 gofrontend_builddir=$workdir/gofrontend_build
 
-bootstrap_type="$1"
-shift
-
-case "$bootstrap_type" in
-quick | full)
+case "$1" in
+libgodeps | quick | full)
+  bootstrap_type="$1"
+  shift
   ;;
 
 *)
-  echo "Bootstrap type must be 'quick' or 'full'"
-  exit 1
   ;;
 esac
 
-configure_flags="--disable-multilib --without-libatomic"
+llgo_cc="${LIBGO_CC:-$workdir/clang_build/bin/clang} $LIBGO_CFLAGS"
+llgo_cxx="${LIBGO_CXX:-$workdir/clang_build/bin/clang++} $LIBGO_CFLAGS"
+
+build_libgodeps() {
+  local cflags="$1"
+  local destdir="$2"
+  local variantname="$3"
+  local makeflags="$4"
+
+  for d in libbacktrace libffi ; do
+    rm -rf $destdir/$d
+    mkdir -p $destdir/$d
+    case $d in
+    libbacktrace)
+      config_flags="--enable-host-shared"
+      ;;
+    *)
+      config_flags=""
+      ;;
+    esac
+
+    echo "# Configuring $variantname $d."
+    (cd $destdir/$d && CC="$llgo_cc $cflags" $gofrontenddir/$d/configure --disable-multilib $config_flags > $workdir/${d}-config.log 2>&1 || (echo "# Configure failed, see $workdir/${d}-config.log" && exit 1))
+    echo "# Building $variantname $d."
+    make -C $destdir/$d $makeflags > $workdir/${d}-make.log 2>&1 || (echo "# Build failed, see $workdir/${d}-make.log" && exit 1)
+  done
+}
+
+build_libgo_stage() {
+  local goc="$1"
+  local cflags="$2"
+  local gocflags="$3"
+  local destdir="$4"
+  local variantname="$5"
+  local makeflags="$6"
+
+  # Wrap Clang in a program that understands -fgo-dump-spec and -fplan9-extensions.
+  libgo_cc="$llgo_cc $cflags"
+  libgo_wrapped_cc="env REAL_CC=$(echo $libgo_cc | sed -e 's/ /@SPACE@/g') $workdir/cc-wrapper"
+
+  mkdir -p $destdir
+  echo "# Configuring $variantname libgo."
+  (cd $destdir && $gofrontenddir/libgo/configure --disable-multilib --without-libatomic CC="$libgo_wrapped_cc" GOC="$goc -no-prefix $GOCFLAGS $gocflags" > $workdir/${variantname}-config.log 2>&1 || (echo "# Configure failed, see $workdir/${variantname}-config.log" && exit 1))
+  echo "# Building $variantname libgo."
+  make -C $destdir $makeflags 2>&1 | tee $workdir/${variantname}-make.log | $workdir/makefilter
+  test "${PIPESTATUS[0]}" = "0" || (echo "# Build failed, see $workdir/${variantname}-make.log" && exit 1)
+  echo
+}
+
+build_libgo_variant() {
+  local goc="$1"
+  local cflags="$2"
+  local gocflags="$3"
+  local variantname="$4"
+  local makeflags="$5"
+
+  local destdir="$workdir/gofrontend_build_$variantname"
+  rm -rf $destdir
+  build_libgodeps "$cflags" "$destdir" "$variantname" "$makeflags"
+  build_libgo_stage "$goc" "$cflags" "$gocflags" "$destdir/libgo" "$variantname" "$makeflags"
+}
+
+if [ "$bootstrap_type" = "libgodeps" ]; then
+  build_libgodeps "$LIBGO_CFLAGS" "$gofrontend_builddir" "normal" "$*"
+  touch $workdir/.build-libgodeps-stamp
+  exit 0
+fi
 
-# Wrap Clang in a program that understands -fgo-dump-spec and -fplan9-extensions.
-(cd $llgodir/cmd/cc-wrapper && go build -o $workdir/cc-wrapper)
-libgo_wrapped_cc="$(echo "${LIBGO_CC:-$workdir/clang_build/bin/clang}" | sed -e 's/ /@SPACE@/g')"
-libgo_cc="env REAL_CC=${libgo_wrapped_cc} $workdir/cc-wrapper"
+if [ "$bootstrap_type" != "" ]; then
+  # Clean up any previous libgo stages.
+  rm -rf $gofrontend_builddir/libgo* $workdir/gofrontend_build_*
 
-# Clean up any previous libgo stages.
-rm -rf $gofrontend_builddir/libgo*
+  echo "# Building helper programs."
+  (cd $llgodir/cmd/cc-wrapper && go build -o $workdir/cc-wrapper)
+  (cd $llgodir/cmd/makefilter && go build -o $workdir/makefilter)
 
-# Build a stage1 compiler with gc.
-(cd $llgodir/cmd/gllgo && go build -o $workdir/gllgo-stage1)
+  # Build a stage1 compiler with gc.
+  echo "# Building stage1 compiler."
+  (cd $llgodir/cmd/gllgo && go build -o $workdir/gllgo-stage1)
 
-# Build libgo with the stage1 compiler.
-mkdir -p $gofrontend_builddir/libgo-stage1
-(cd $gofrontend_builddir/libgo-stage1 && $gofrontenddir/libgo/configure $configure_flags CC="$libgo_cc" GOC="$workdir/gllgo-stage1 -no-prefix")
-make -C $gofrontend_builddir/libgo-stage1 "$@"
+  # Build libgo with the stage1 compiler.
+  build_libgo_stage $workdir/gllgo-stage1 "" "" $gofrontend_builddir/libgo-stage1 stage1 "$*"
 
-# Set up a directory which when added to $PATH causes "gccgo" to resolve
-# to our stage1 compiler. This is necessary because the logic in "go build"
-# for locating the compiler is fixed.
-mkdir -p $gofrontend_builddir/stage1-path
-ln -sf $workdir/gllgo-stage1 $gofrontend_builddir/stage1-path/gccgo
+  # Set up a directory which when added to $PATH causes "gccgo" to resolve
+  # to our stage1 compiler. This is necessary because the logic in "go build"
+  # for locating the compiler is fixed.
+  mkdir -p $gofrontend_builddir/stage1-path
+  ln -sf $workdir/gllgo-stage1 $gofrontend_builddir/stage1-path/gccgo
 
-# Build a stage2 compiler using the stage1 compiler and libgo.
-gllgoflags="-no-prefix -L$gofrontend_builddir/libgo-stage1 -L$gofrontend_builddir/libgo-stage1/.libs -static-libgo"
-(cd $llgodir/cmd/gllgo && PATH=$gofrontend_builddir/stage1-path:$PATH go build -compiler gccgo -gccgoflags "$gllgoflags" -o $workdir/gllgo-stage2)
+  # Build a stage2 compiler using the stage1 compiler and libgo.
+  echo "# Building stage2 compiler."
+  gllgoflags="-no-prefix -L$gofrontend_builddir/libgo-stage1 -L$gofrontend_builddir/libgo-stage1/.libs -static-libgo $GOCFLAGS"
+  (cd $llgodir/cmd/gllgo && PATH=$gofrontend_builddir/stage1-path:$PATH CC="$llgo_cc" CXX="$llgo_cxx" go build -compiler gccgo -gccgoflags "$gllgoflags" -o $workdir/gllgo-stage2)
 
-# If this is a quick bootstrap, do not rebuild libgo with the stage2 compiler.
-# Instead, use the stage1 libgo.
+  # If this is a quick bootstrap, do not rebuild libgo with the stage2 compiler.
+  # Instead, use the stage1 libgo.
 
-if [ "$bootstrap_type" == "full" ] ; then
-  # Build libgo with the stage2 compiler.
-  mkdir -p $gofrontend_builddir/libgo-stage2
-  (cd $gofrontend_builddir/libgo-stage2 && $gofrontenddir/libgo/configure $configure_flags CC="$libgo_cc" GOC="$workdir/gllgo-stage2 -no-prefix")
-  make -C $gofrontend_builddir/libgo-stage2 "$@"
+  if [ "$bootstrap_type" == "full" ] ; then
+    # Build libgo with the stage2 compiler.
+    build_libgo_stage $workdir/gllgo-stage2 "" "" $gofrontent_builddir/libgo-stage2 stage2 "$*"
 
-  # Set up $gllgoflags to use the stage2 libgo.
-  gllgoflags="-no-prefix -L$gofrontend_builddir/libgo-stage2 -L$gofrontend_builddir/libgo-stage2/.libs -static-libgo"
-fi
+    # Set up $gllgoflags to use the stage2 libgo.
+    gllgoflags="-no-prefix -L$gofrontend_builddir/libgo-stage2 -L$gofrontend_builddir/libgo-stage2/.libs -static-libgo $GOCFLAGS"
+  fi
 
-# Set up a directory which when added to $PATH causes "gccgo" to resolve
-# to our stage2 compiler.
-mkdir -p $gofrontend_builddir/stage2-path
-ln -sf $workdir/gllgo-stage2 $gofrontend_builddir/stage2-path/gccgo
+  # Set up a directory which when added to $PATH causes "gccgo" to resolve
+  # to our stage2 compiler.
+  mkdir -p $gofrontend_builddir/stage2-path
+  ln -sf $workdir/gllgo-stage2 $gofrontend_builddir/stage2-path/gccgo
 
-# Build the stage3 compiler.
-(cd $llgodir/cmd/gllgo && PATH=$gofrontend_builddir/stage2-path:$PATH go build -compiler gccgo -gccgoflags "$gllgoflags" -o $workdir/gllgo-stage3)
+  # Build the stage3 compiler.
+  echo "# Building stage3 compiler."
+  (cd $llgodir/cmd/gllgo && PATH=$gofrontend_builddir/stage2-path:$PATH CC="$llgo_cc" CXX="$llgo_cxx" go build -compiler gccgo -gccgoflags "$gllgoflags" -o $workdir/gllgo-stage3)
 
-# Strip the compiler binaries. The binaries are currently only
-# expected to compare equal modulo debug info.
-strip -R .note.gnu.build-id -o $workdir/gllgo-stage2.stripped $workdir/gllgo-stage2
-strip -R .note.gnu.build-id -o $workdir/gllgo-stage3.stripped $workdir/gllgo-stage3
+  # Strip the compiler binaries. The binaries are currently only
+  # expected to compare equal modulo debug info.
+  strip -R .note.gnu.build-id -o $workdir/gllgo-stage2.stripped $workdir/gllgo-stage2
+  strip -R .note.gnu.build-id -o $workdir/gllgo-stage3.stripped $workdir/gllgo-stage3
 
-cmp $workdir/gllgo-stage2.stripped $workdir/gllgo-stage3.stripped && \
-echo "Bootstrap completed successfully" && touch $workdir/.bootstrap-stamp && exit 0 || \
-echo "Bootstrap failed, binaries differ" && exit 1
+  cmp $workdir/gllgo-stage2.stripped $workdir/gllgo-stage3.stripped && \
+  echo "# Bootstrap completed successfully." && touch $workdir/.bootstrap-stamp || \
+  (echo "# Bootstrap failed, binaries differ." && exit 1)
+fi

cmd/gllgo/gllgo.go

@@ -381,6 +381,14 @@ func getDataInlineAsm(data []byte) string {
 	return string(edata[0 : j+2])
 }
 
+// Get the lib-relative path to the standard libraries for the given driver
+// options. This is normally '.' but can vary for cross compilation, LTO,
+// sanitizers etc.
+func getVariantDir(opts *driverOptions) string {
+	path := "."
+	return path
+}
+
 func performAction(opts *driverOptions, kind actionKind, inputs []string, output string) error {
 	switch kind {
 	case actionPrint:
@@ -391,8 +399,7 @@ func performAction(opts *driverOptions, kind actionKind, inputs []string, output
 			os.Stdout.Write(out)
 			return err
 		case "-print-multi-os-directory":
-			// TODO(pcc): Vary this for cross-compilation.
-			fmt.Println(".")
+			fmt.Println(getVariantDir(opts))
 			return nil
 		case "--version":
 			displayVersion()
@@ -541,7 +548,7 @@ func performAction(opts *driverOptions, kind actionKind, inputs []string, output
 			linkerPath = opts.bprefix + "gcc"
 
 			if opts.prefix != "" {
-				libdir := filepath.Join(opts.prefix, "lib")
+				libdir := filepath.Join(opts.prefix, "lib", getVariantDir(opts))
 				args = append(args, "-L", libdir)
 				if !opts.staticLibgo {
 					args = append(args, "-Wl,-rpath,"+libdir)

cmd/makefilter/main.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"strings"
+)
+
+// A simple filter program which improves the readability of libgo's build
+// output by filtering out everything except the package names.
+func main() {
+	stdin := bufio.NewReader(os.Stdin)
+	pkgs := make(map[string]struct{})
+	for {
+		line, err := stdin.ReadString('\n')
+		if err == io.EOF {
+			return
+		} else if err != nil {
+			panic("read error: " + err.Error())
+		}
+		line = line[0 : len(line)-1]
+		if strings.HasPrefix(line, "libtool: compile:") {
+			words := strings.Split(line, " ")
+			for _, word := range words {
+				if strings.HasPrefix(word, "-fgo-pkgpath=") {
+					pkg := word[13:]
+					if _, ok := pkgs[pkg]; !ok {
+						os.Stdout.WriteString(pkg + "\n")
+						pkgs[pkg] = struct{}{}
+					}
+				}
+			}
+		}
+	}
+}

install.sh

@@ -24,6 +24,15 @@ else
   make -C $gofrontend_builddir/libgo-stage1 install "prefix=$prefix"
 fi
 
+# Install the build variant libraries, but not the export data, which is shared
+# between variants.
+for i in $workdir/gofrontend_build_*/libgo ; do
+  if [ -d $i ] ; then
+    make -C $i install-toolexeclibLIBRARIES install-toolexeclibLTLIBRARIES \
+      "prefix=$prefix"
+  fi
+done
+
 # Set up the symlink required by llgo-go.
 mkdir -p "$prefix/lib/llgo/go-path"
 ln -sf ../../../bin/llgo "$prefix/lib/llgo/go-path/gccgo"

update_libgo.sh

@@ -1,7 +1,6 @@
 #!/bin/sh -e
 
-# Fetch libgo and its dependencies, and build the dependencies.
-# We build libgo itself while bootstrapping.
+# Fetch libgo and its dependencies.
 
 llgodir=$(dirname "$0")
 llgodir=$(cd "$llgodir" && pwd)
@@ -14,7 +13,6 @@ gccrev=209880
 
 workdir=$llgodir/workdir
 gofrontenddir=$workdir/gofrontend
-gofrontend_builddir=$workdir/gofrontend_build
 
 mkdir -p $workdir
 if [ -d $gofrontenddir/.hg ] ; then
@@ -54,17 +52,6 @@ echo "#define IS_ABSOLUTE_PATH(path) ((path)[0] == '/')" > $gofrontenddir/includ
 
 for d in libbacktrace libffi ; do
   svn co -r $gccrev $gccrepo/$d $gofrontenddir/$d
-  mkdir -p $gofrontend_builddir/$d
-  case $d in
-  libbacktrace)
-    config_flags="--enable-host-shared"
-    ;;
-  *)
-    config_flags=""
-    ;;
-  esac
-  (cd $gofrontend_builddir/$d && CC="${LIBGO_CC:-$workdir/clang_build/bin/clang}" $gofrontenddir/$d/configure --disable-multilib $config_flags)
-  make -C $gofrontend_builddir/$d -j4
 done
 
 touch $workdir/.update-libgo-stamp