Improve docker images and configuration; create compiler-zoo image

llvm-svn: 351667

6 files changed, 244 insertions, 208 deletions

diff --git a/libcxx/utils/docker/scripts/build_gcc.sh b/libcxx/utils/docker/scripts/build_gcc.sh
deleted file mode 100755
index 3627a9ae9a8..00000000000
--- a/libcxx/utils/docker/scripts/build_gcc.sh
+++ /dev/null
@@ -1,90 +0,0 @@
-#!/usr/bin/env bash
-#===- libcxx/utils/docker/scripts/build-gcc.sh ----------------------------===//
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===-----------------------------------------------------------------------===//
-
-set -e
-
-
-function show_usage() {
-  cat << EOF
-Usage: build-gcc.sh [options]
-
-Run autoconf with the specified arguments. Used inside docker container.
-
-Available options:
-  -h|--help           show this help message
-  --source            the source path from which to run the configuration.
-  --to                destination directory where to install the targets.
-Required options: --to, at least one --install-target.
-
-All options after '--' are passed to CMake invocation.
-EOF
-}
-
-GCC_INSTALL_DIR=""
-GCC_SOURCE_DIR=""
-
-while [[ $# -gt 0 ]]; do
-  case "$1" in
-    --to)
-      shift
-      GCC_INSTALL_DIR="$1"
-      shift
-      ;;
-    --source)
-      shift
-      GCC_SOURCE_DIR="$1"
-      shift
-      ;;
-    -h|--help)
-      show_usage
-      exit 0
-      ;;
-    *)
-      echo "Unknown option: $1"
-      exit 1
-  esac
-done
-
-if [ "$GCC_INSTALL_DIR" == "" ]; then
-  echo "No install directory. Please specify the --to argument."
-  exit 1
-fi
-
-if [ "$GCC_SOURCE_DIR" == "" ]; then
-  echo "No source directory. Please specify the --source argument."
-  exit 1
-fi
-
-GCC_NAME=`basename $GCC_SOURCE_DIR`
-GCC_BUILD_DIR="/tmp/gcc-build-root/build-$GCC_NAME"
-
-mkdir -p "$GCC_INSTALL_DIR"
-mkdir -p "$GCC_BUILD_DIR"
-pushd "$GCC_BUILD_DIR"
-
-# Run the build as specified in the build arguments.
-echo "Running configuration"
-$GCC_SOURCE_DIR/configure --prefix=$GCC_INSTALL_DIR \
-  --disable-bootstrap --disable-libgomp --disable-libitm \
-  --disable-libvtv --disable-libcilkrts --disable-libmpx \
-  --disable-liboffloadmic --disable-libcc1 --enable-languages=c,c++
-
-NPROC=`nproc`
-echo "Running build with $NPROC threads"
-make -j$NPROC
-
-echo "Installing to $GCC_INSTALL_DIR"
-make install -j$NPROC
-
-popd
-
-# Cleanup.
-rm -rf "$GCC_BUILD_DIR"
-
-echo "Done"
\ No newline at end of file
diff --git a/libcxx/utils/docker/scripts/build_gcc_version.sh b/libcxx/utils/docker/scripts/build_gcc_version.sh
new file mode 100755
index 00000000000..f569d7e18d2
--- /dev/null
+++ b/libcxx/utils/docker/scripts/build_gcc_version.sh
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+#===- libcxx/utils/docker/scripts/build-gcc.sh ----------------------------===//
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===-----------------------------------------------------------------------===//
+
+set -e
+
+function show_usage() {
+  cat << EOF
+Usage: build_gcc_version.sh [options]
+
+Run autoconf with the specified arguments. Used inside docker container.
+
+Available options:
+  -h|--help           show this help message
+  --branch            the branch of gcc you want to build.
+  --cherry-pick       a commit hash to apply to the GCC sources.
+  --install           destination directory where to install the targets.
+Required options: --install and --branch
+
+All options after '--' are passed to CMake invocation.
+EOF
+}
+
+GCC_INSTALL_DIR=""
+GCC_BRANCH=""
+CHERRY_PICK=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --install)
+      shift
+      GCC_INSTALL_DIR="$1"
+      shift
+      ;;
+    --branch)
+      shift
+      GCC_BRANCH="$1"
+      shift
+      ;;
+    --cherry-pick)
+      shift
+      CHERRY_PICK="$1"
+      shift
+      ;;
+    -h|--help)
+      show_usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+  esac
+done
+
+if [ "$GCC_INSTALL_DIR" == "" ]; then
+  echo "No install directory. Please specify the --install argument."
+  exit 1
+fi
+
+if [ "$GCC_BRANCH" == "" ]; then
+  echo "No branch specified. Please specify the --branch argument."
+  exit 1
+fi
+
+set -x
+
+NPROC=`nproc`
+TMP_ROOT="$(mktemp -d -p /tmp)"
+GCC_SOURCE_DIR="$TMP_ROOT/gcc"
+GCC_BUILD_DIR="$TMP_ROOT/build"
+
+echo "Cloning source directory for branch $GCC_BRANCH"
+git clone --branch "$GCC_BRANCH" --single-branch --depth=1 git://gcc.gnu.org/git/gcc.git $GCC_SOURCE_DIR
+
+pushd "$GCC_SOURCE_DIR"
+if [ "$CHERRY_PICK" != "" ]; then
+  git fetch origin trunk --unshallow # Urg, we have to get the entire history. This will take a while.
+  git cherry-pick --no-commit -X theirs "$CHERRY_PICK"
+fi
+./contrib/download_prerequisites
+popd
+
+
+mkdir "$GCC_BUILD_DIR"
+pushd "$GCC_BUILD_DIR"
+
+# Run the build as specified in the build arguments.
+echo "Running configuration"
+$GCC_SOURCE_DIR/configure --prefix=$GCC_INSTALL_DIR \
+  --disable-bootstrap --disable-libgomp --disable-libitm \
+  --disable-libvtv --disable-libcilkrts --disable-libmpx \
+  --disable-liboffloadmic --disable-libcc1 --enable-languages=c,c++
+
+echo "Running build with $NPROC threads"
+make -j$NPROC
+echo "Installing to $GCC_INSTALL_DIR"
+make install -j$NPROC
+popd
+
+# Cleanup.
+rm -rf "$TMP_ROOT"
+
+echo "Done"
\ No newline at end of file
diff --git a/libcxx/utils/docker/scripts/build_install_llvm.sh b/libcxx/utils/docker/scripts/build_install_llvm.sh
deleted file mode 100755
index d37e52d326a..00000000000
--- a/libcxx/utils/docker/scripts/build_install_llvm.sh
+++ /dev/null
@@ -1,113 +0,0 @@
-#!/usr/bin/env bash
-#===- llvm/utils/docker/scripts/build_install_llvm.sh ---------------------===//
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===-----------------------------------------------------------------------===//
-
-set -e
-
-function show_usage() {
-  cat << EOF
-Usage: build_install_llvm.sh [options] -- [cmake-args]
-
-Run cmake with the specified arguments. Used inside docker container.
-Passes additional -DCMAKE_INSTALL_PREFIX and puts the build results into
-the directory specified by --to option.
-
-Available options:
-  -h|--help           show this help message
-  -i|--install-target name of a cmake install target to build and include in
-                      the resulting archive. Can be specified multiple times.
-  --install           destination directory where to install the targets.
-  --source            location of the source tree.
-  --build             location to use as the build directory.
-Required options: --to, --source, --build, and at least one --install-target.
-
-All options after '--' are passed to CMake invocation.
-EOF
-}
-
-CMAKE_ARGS=""
-CMAKE_INSTALL_TARGETS=""
-CLANG_INSTALL_DIR=""
-CLANG_SOURCE_DIR=""
-CLANG_BUILD_DIR=""
-
-while [[ $# -gt 0 ]]; do
-  case "$1" in
-    -i|--install-target)
-      shift
-      CMAKE_INSTALL_TARGETS="$CMAKE_INSTALL_TARGETS $1"
-      shift
-      ;;
-    --source)
-      shift
-      CLANG_SOURCE_DIR="$1"
-      shift
-      ;;
-    --build)
-      shift
-      CLANG_BUILD_DIR="$1"
-      shift
-      ;;
-    --install)
-      shift
-      CLANG_INSTALL_DIR="$1"
-      shift
-      ;;
-    --)
-      shift
-      CMAKE_ARGS="$*"
-      shift $#
-      ;;
-    -h|--help)
-      show_usage
-      exit 0
-      ;;
-    *)
-      echo "Unknown option: $1"
-      exit 1
-  esac
-done
-
-if [ "$CLANG_SOURCE_DIR" == "" ]; then
-  echo "No source directory. Please pass --source."
-  exit 1
-fi
-
-if [ "$CLANG_BUILD_DIR" == "" ]; then
-  echo "No build directory. Please pass --build"
-  exit 1
-fi
-
-if [ "$CMAKE_INSTALL_TARGETS" == "" ]; then
-  echo "No install targets. Please pass one or more --install-target."
-  exit 1
-fi
-
-if [ "$CLANG_INSTALL_DIR" == "" ]; then
-  echo "No install directory. Please specify the --to argument."
-  exit 1
-fi
-
-echo "Building in $CLANG_BUILD_DIR"
-mkdir -p "$CLANG_BUILD_DIR"
-pushd "$CLANG_BUILD_DIR"
-
-# Run the build as specified in the build arguments.
-echo "Running build"
-cmake -GNinja \
-  -DCMAKE_INSTALL_PREFIX="$CLANG_INSTALL_DIR" \
-  $CMAKE_ARGS \
-  "$CLANG_SOURCE_DIR"
-ninja $CMAKE_INSTALL_TARGETS
-
-popd
-
-# Cleanup.
-rm -rf "$CLANG_BUILD_DIR"
-
-echo "Done"
diff --git a/libcxx/utils/docker/scripts/build_llvm_version.sh b/libcxx/utils/docker/scripts/build_llvm_version.sh
new file mode 100755
index 00000000000..eb093c426db
--- /dev/null
+++ b/libcxx/utils/docker/scripts/build_llvm_version.sh
@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+#===- libcxx/utils/docker/scripts/build_install_llvm_version_default.sh -----------------------===//
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===-------------------------------------------------------------------------------------------===//
+
+set -e
+
+function show_usage() {
+  cat << EOF
+Usage: build_install_llvm.sh [options] -- [cmake-args]
+
+Run cmake with the specified arguments. Used inside docker container.
+Passes additional -DCMAKE_INSTALL_PREFIX and puts the build results into
+the directory specified by --to option.
+
+Available options:
+  -h|--help           show this help message
+  --install           destination directory where to install the targets.
+  --branch            the branch or tag of LLVM to build
+Required options: --install, and --version.
+
+All options after '--' are passed to CMake invocation.
+EOF
+}
+
+LLVM_BRANCH=""
+CMAKE_ARGS=""
+LLVM_INSTALL_DIR=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --install)
+      shift
+      LLVM_INSTALL_DIR="$1"
+      shift
+      ;;
+    --branch)
+      shift
+      LLVM_BRANCH="$1"
+      shift
+      ;;
+    --)
+      shift
+      CMAKE_ARGS="$*"
+      shift $#
+      ;;
+    -h|--help)
+      show_usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+  esac
+done
+
+
+if [ "$LLVM_INSTALL_DIR" == "" ]; then
+  echo "No install directory. Please specify the --install argument."
+  exit 1
+fi
+
+if [ "$LLVM_BRANCH" == "" ]; then
+  echo "No install directory. Please specify the --branch argument."
+  exit 1
+fi
+
+if [ "$CMAKE_ARGS" == "" ]; then
+  CMAKE_ARGS="-DCMAKE_BUILD_TYPE=RELEASE '-DCMAKE_C_FLAGS=-gline-tables-only' '-DCMAKE_CXX_FLAGS=-gline-tables-only' -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON"
+fi
+
+set -x
+
+TMP_ROOT="$(mktemp -d -p /tmp)"
+LLVM_SOURCE_DIR="$TMP_ROOT/llvm-project"
+LLVM_BUILD_DIR="$TMP_ROOT/build"
+LLVM="$LLVM_SOURCE_DIR/llvm"
+
+git clone --branch $LLVM_BRANCH --single-branch --depth=1 https://github.com/llvm/llvm-project.git $LLVM_SOURCE_DIR
+
+pushd "$LLVM_SOURCE_DIR"
+
+# Setup the source-tree using the old style layout
+ln -s $LLVM_SOURCE_DIR/libcxx $LLVM/projects/libcxx
+ln -s $LLVM_SOURCE_DIR/libcxxabi $LLVM/projects/libcxxabi
+ln -s $LLVM_SOURCE_DIR/compiler-rt $LLVM/projects/compiler-rt
+ln -s $LLVM_SOURCE_DIR/clang $LLVM/tools/clang
+ln -s $LLVM_SOURCE_DIR/clang-tools-extra $LLVM/tools/clang/tools/extra
+
+popd
+
+# Configure and build
+mkdir "$LLVM_BUILD_DIR"
+pushd "$LLVM_BUILD_DIR"
+cmake -GNinja "-DCMAKE_INSTALL_PREFIX=$LLVM_INSTALL_DIR" $CMAKE_ARGS $LLVM
+ninja install
+popd
+
+# Cleanup
+rm -rf "$TMP_ROOT/"
+
+echo "Done"
diff --git a/libcxx/utils/docker/scripts/install_clang_packages.sh b/libcxx/utils/docker/scripts/install_clang_packages.sh
index 4a46c90a4c7..94c98d6ad74 100755
--- a/libcxx/utils/docker/scripts/install_clang_packages.sh
+++ b/libcxx/utils/docker/scripts/install_clang_packages.sh
@@ -20,7 +20,7 @@ Available options:
 EOF
 }
 
-VERSION=""
+VERSION="9"
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -39,12 +39,29 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-
+set -x
 
 curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
 add-apt-repository -s "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs) main"
 apt-get update
-apt-get install -y --no-install-recommends clang
+apt-get upgrade -y
+apt-get install -y --no-install-recommends "clang-$VERSION"
+
+# FIXME(EricWF): Remove this once the clang packages are no longer broken.
+if [ -f "/usr/local/bin/clang" ]; then
+  echo "clang already exists"
+  exit 1
+else
+  CC_BINARY="$(which clang-$VERSION)"
+  ln -s "$CC_BINARY" "/usr/local/bin/clang"
+fi
+if [ -f "/usr/local/bin/clang++" ]; then
+  echo "clang++ already exists"
+  exit 1
+else
+  CXX_BINARY="$(which clang++-$VERSION)"
+  ln -s "$CXX_BINARY" "/usr/local/bin/clang++"
+fi
 
 echo "Testing clang version..."
 clang --version
@@ -58,6 +75,7 @@ if [ "$VERSION" == "" ]; then
   echo "Installing version '$VERSION'"
 fi
 
+apt-get purge -y "libc++-$VERSION-dev" "libc++abi-$VERSION-dev"
 apt-get install -y --no-install-recommends "libc++-$VERSION-dev" "libc++abi-$VERSION-dev"
 
 echo "Done"
diff --git a/libcxx/utils/docker/scripts/run_buildbot.sh b/libcxx/utils/docker/scripts/run_buildbot.sh
index 96e832dd4e4..c135fc4af89 100755
--- a/libcxx/utils/docker/scripts/run_buildbot.sh
+++ b/libcxx/utils/docker/scripts/run_buildbot.sh
@@ -5,9 +5,14 @@ BOT_DIR=/b
 BOT_NAME=$1
 BOT_PASS=$2
 
-mkdir -p $BOT_DIR
+pushd /tmp
+curl -sSO https://dl.google.com/cloudagents/install-monitoring-agent.sh
+bash install-monitoring-agent.sh
+curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh
+bash install-logging-agent.sh --structured
+popd
 
-#curl "https://repo.stackdriver.com/stack-install.sh" | bash -s -- --write-gcm
+mkdir -p $BOT_DIR
 
 apt-get update -y
 apt-get upgrade -y