Reconfigure docker builders to be more modular.

And other various cleanups to the configuration.

llvm-svn: 363722

5 files changed, 120 insertions, 298 deletions

diff --git a/libcxx/utils/docker/scripts/checkout_git.sh b/libcxx/utils/docker/scripts/checkout_git.sh
deleted file mode 100755
index 2c5563b43f7..00000000000
--- a/libcxx/utils/docker/scripts/checkout_git.sh
+++ /dev/null
@@ -1,129 +0,0 @@
-#!/usr/bin/env bash
-#===- llvm/utils/docker/scripts/checkout.sh ---------------------===//
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===-----------------------------------------------------------------------===//
-
-set -e
-
-function show_usage() {
-  cat << EOF
-Usage: checkout.sh [options]
-
-Checkout svn sources into /tmp/clang-build/src. Used inside a docker container.
-
-Available options:
-  -h|--help           show this help message
-  -b|--branch         svn branch to checkout, i.e. 'trunk',
-                      'branches/release_40'
-                      (default: 'trunk')
-  -p|--llvm-project   name of an svn project to checkout.
-                      For clang, please use 'clang', not 'cfe'.
-                      Project 'llvm' is always included and ignored, if
-                      specified.
-                      Can be specified multiple times.
-EOF
-}
-
-LLVM_BRANCH=""
-# We always checkout llvm
-LLVM_PROJECTS="llvm"
-SOURCE_DIR=""
-
-function contains_project() {
-  local TARGET_PROJ="$1"
-  local PROJ
-  for PROJ in $LLVM_PROJECTS; do
-    if [ "$PROJ" == "$TARGET_PROJ" ]; then
-      return 0
-    fi
-  done
-  return 1
-}
-
-while [[ $# -gt 0 ]]; do
-  case "$1" in
-    --to)
-      shift
-      SOURCE_DIR="$1"
-      shift
-      ;;
-    -b|--branch)
-      shift
-      LLVM_BRANCH="$1"
-      shift
-      ;;
-    -p|--llvm-project)
-      shift
-      PROJ="$1"
-      shift
-
-      if [ "$PROJ" == "cfe" ]; then
-        PROJ="clang"
-      fi
-
-      if ! contains_project "$PROJ" ; then
-        if [ "$PROJ" == "clang-tools-extra" ] && [ ! contains_project "clang" ]; then
-          echo "Project 'clang-tools-extra' specified before 'clang'. Adding 'clang' to a list of projects first."
-          LLVM_PROJECTS="$LLVM_PROJECTS clang"
-        fi
-        LLVM_PROJECTS="$LLVM_PROJECTS $PROJ"
-      else
-        echo "Project '$PROJ' is already enabled, ignoring extra occurrences."
-      fi
-      ;;
-    -h|--help)
-      show_usage
-      exit 0
-      ;;
-    *)
-      echo "Unknown option: $1"
-      exit 1
-  esac
-done
-
-if [ "$SOURCE_DIR" == "" ]; then
-  echo "Must specify checkout directory using --to"
-  exit 1
-fi
-
-if [ "$LLVM_BRANCH" == "" ]; then
-  GIT_BRANCH_ARG=""
-else
-  GIT_BRANCH_ARG="--branch $LLVM_BRANCH"
-fi
-
-if [ "$LLVM_SVN_REV" != "" ]; then
-  SVN_REV_ARG="-r$LLVM_SVN_REV"
-  echo "Checking out svn revision r$LLVM_SVN_REV."
-else
-  SVN_REV_ARG=""
-  echo "Checking out latest svn revision."
-fi
-
-# Get the sources from svn.
-echo "Checking out sources from git"
-
-for LLVM_PROJECT in $LLVM_PROJECTS; do
-  if [ "$LLVM_PROJECT" == "llvm" ]; then
-    CHECKOUT_DIR="$SOURCE_DIR"
-  elif [ "$LLVM_PROJECT" == "libcxx" ] || [ "$LLVM_PROJECT" == "libcxxabi" ] || [ "$LLVM_PROJECT" == "compiler-rt" ]; then
-    CHECKOUT_DIR="$SOURCE_DIR/projects/$LLVM_PROJECT"
-  elif [ "$LLVM_PROJECT" == "clang" ]; then
-    CHECKOUT_DIR="$SOURCE_DIR/tools/clang"
-  elif [ "$LLVM_PROJECT" == "clang-tools-extra" ]; then
-    CHECKOUT_DIR="$SOURCE_DIR/tools/clang/tools/extra"
-  else
-    CHECKOUT_DIR="$SOURCE_DIR/$LLVM_PROJECT"
-  fi
-
-  echo "Checking out https://git.llvm.org/git/$LLVM_PROJECT to $CHECKOUT_DIR"
-  git clone --depth=1 $GIT_BRANCH_ARG \
-    "https://git.llvm.org/git/$LLVM_PROJECT.git" \
-    "$CHECKOUT_DIR"
-done
-
-echo "Done"
diff --git a/libcxx/utils/docker/scripts/docker_start_buildbots.sh b/libcxx/utils/docker/scripts/docker_start_buildbots.sh
index 0531d6fa3b6..f47ddcd2481 100755
--- a/libcxx/utils/docker/scripts/docker_start_buildbots.sh
+++ b/libcxx/utils/docker/scripts/docker_start_buildbots.sh
@@ -1,12 +1,8 @@
 #!/usr/bin/env bash
 set -x
 
-if [[ "$1" == "--new" ]]; then
-  shift
-fi
-
 # Update the libc++ sources in the image in order to use the most recent version of
 # run_buildbots.sh
 cd /libcxx
 git pull
-/libcxx/utils/docker/scripts/run_buildbot_new.sh "$@"
+/libcxx/utils/docker/scripts/run_buildbot.sh "$@"
diff --git a/libcxx/utils/docker/scripts/docker_update_bot.sh b/libcxx/utils/docker/scripts/docker_update_bot.sh
new file mode 100755
index 00000000000..5c027a860f9
--- /dev/null
+++ b/libcxx/utils/docker/scripts/docker_update_bot.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -x
+
+cd /libcxx
+git pull
+
+
+#pushd /tmp
+#curl -sSO https://dl.google.com/cloudagents/install-monitoring-agent.sh
+#bash install-monitoring-agent.sh
+#curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh
+#bash install-logging-agent.sh --structured
+#popd
+
+
+apt-get update -y
+apt-get upgrade -y
+
+apt-get install sudo -y
+
+# FIXME(EricWF): Remove this hack. It's only in place to temporarily fix linking libclang_rt from the
+# debian packages.
+# WARNING: If you're not a buildbot, DO NOT RUN!
+apt-get install lld-9
+rm /usr/bin/ld
+ln -s /usr/bin/lld-9 /usr/bin/ld
+
+systemctl set-property buildslave.service TasksMax=100000
+
+systemctl daemon-reload
+service buildslave restart
diff --git a/libcxx/utils/docker/scripts/run_buildbot.sh b/libcxx/utils/docker/scripts/run_buildbot.sh
index ede004913cf..bd1d82afa3a 100755
--- a/libcxx/utils/docker/scripts/run_buildbot.sh
+++ b/libcxx/utils/docker/scripts/run_buildbot.sh
@@ -1,32 +1,22 @@
 #!/usr/bin/env bash
 set -x
 
-script_dir=`dirname "$0"`
+readonly BOT_ROOT=/b
+readonly BOT_ROOT_NAME=$1
+readonly BOT_PASS=$2
 
-# TODO(EricWF): Replace this file with the new script once the existing
-# bots have been migrated.
-if [ "$1" == "--new" ]; then
-  shift
-  $script_dir/run_buildbot_new.sh "$@"
-  shutdown now
-fi
-
-BOT_DIR=/b
-BOT_NAME=$1
-BOT_PASS=$2
-
-pushd /tmp
-curl -sSO https://dl.google.com/cloudagents/install-monitoring-agent.sh
-bash install-monitoring-agent.sh
-curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh
-bash install-logging-agent.sh --structured
-popd
-
-mkdir -p $BOT_DIR
+#pushd /tmp
+#curl -sSO https://dl.google.com/cloudagents/install-monitoring-agent.sh
+#bash install-monitoring-agent.sh
+#curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh
+#bash install-logging-agent.sh --structured
+#popd
 
 apt-get update -y
 apt-get upgrade -y
 
+apt-get install sudo -y
+
 # FIXME(EricWF): Remove this hack. It's only in place to temporarily fix linking libclang_rt from the
 # debian packages.
 # WARNING: If you're not a buildbot, DO NOT RUN!
@@ -36,42 +26,85 @@ ln -s /usr/bin/lld-9 /usr/bin/ld
 
 systemctl set-property buildslave.service TasksMax=100000
 
-buildslave stop $BOT_DIR
-
-chown buildbot:buildbot $BOT_DIR
-
-echo "Connecting as $BOT_NAME"
-buildslave create-slave --allow-shutdown=signal $BOT_DIR lab.llvm.org:9990 $BOT_NAME $BOT_PASS
-
-echo "Eric Fiselier <ericwf@google.com>" > $BOT_DIR/info/admin
-
-{
-  uname -a | head -n1
-  cmake --version | head -n1
-  g++ --version | head -n1
-  ld --version | head -n1
-  date
-  lscpu
-} > $BOT_DIR/info/host
-
-echo "SLAVE_RUNNER=/usr/bin/buildslave
-SLAVE_ENABLED[1]=\"1\"
-SLAVE_NAME[1]=\"buildslave1\"
-SLAVE_USER[1]=\"buildbot\"
-SLAVE_BASEDIR[1]=\"$BOT_DIR\"
-SLAVE_OPTIONS[1]=\"\"
-SLAVE_PREFIXCMD[1]=\"\"" > /etc/default/buildslave
-
-chown -R buildbot:buildbot $BOT_DIR
-systemctl daemon-reload
-service buildslave restart
-
-sleep 30
-cat $BOT_DIR/twistd.log
-grep "slave is ready" $BOT_DIR/twistd.log || shutdown now
+function setup_numbered_bot() {
+  local BOT_NAME=$1
+  local BOT_DIR=$2
+  mkdir -p $BOT_DIR
+
+  buildslave stop $BOT_DIR
+  chown buildbot:buildbot $BOT_DIR
+  rm -rf $BOT_DIR/*
+
+  buildslave create-slave --allow-shutdown=signal "$BOT_DIR" "lab.llvm.org:9990" "$BOT_NAME" "$BOT_PASS"
+
+  echo "Eric Fiselier <ericwf@google.com>" > $BOT_DIR/info/admin
+
+  echo "Connecting as $1"
+  {
+    uname -a | head -n1
+    cmake --version | head -n1
+    g++ --version | head -n1
+    ld --version | head -n1
+    date
+    lscpu
+  } > $BOT_DIR/info/host
+
+
+#echo "SLAVE_RUNNER=/usr/bin/buildslave
+#SLAVE_ENABLED[1]=\"1\"
+#SLAVE_NAME[1]=\"$BOT_NAME\"
+#SLAVE_USER[1]=\"buildbot\"
+#SLAVE_BASEDIR[1]=\"$BOT_DIR\"
+#SLAVE_OPTIONS[1]=\"\"
+#SLAVE_PREFIXCMD[1]=\"\"" > $BOT_DIR/buildslave.cfg
+
+  ls $BOT_DIR/
+  cat $BOT_DIR/buildbot.tac
+}
+
+function try_start_builder {
+  local N=$1
+  local BOT_DIR="$BOT_ROOT/b$N"
+  local BOT_NAME="$BOT_ROOT_NAME$N"
+
+  systemctl daemon-reload
+  service buildslave restart
+  setup_numbered_bot "$BOT_NAME" "$BOT_DIR"
+
+  systemctl daemon-reload
+  service buildslave restart
+
+  chown -R buildbot:buildbot $BOT_DIR/
+  sudo -u buildbot /usr/bin/buildslave start $BOT_DIR/
+
+  sleep 30
+  cat $BOT_DIR/twistd.log
+  if grep --quiet "slave is ready" $BOT_DIR/twistd.log; then
+    return 0
+  fi
+  if grep --quiet "configuration update complete" $BOT_DIR/twistd.log; then
+    return 0
+  fi
+  if grep "rejecting duplicate slave" $BOT_DIR/twistd.log; then
+    return 1
+  fi
+  echo "Unknown error"
+  cat $BOT_DIR/twistd.log
+  exit 1
+}
+
+for N in `shuf -i 1-5`
+do
+  if try_start_builder $N; then
+    break
+  fi
+  echo "failed to start any buildbot"
+  shutdown now
+done
 
 # GCE can restart instance after 24h in the middle of the build.
 # Gracefully restart before that happen.
 sleep 72000
 while pkill -SIGHUP buildslave; do sleep 5; done;
 shutdown now
+
diff --git a/libcxx/utils/docker/scripts/run_buildbot_new.sh b/libcxx/utils/docker/scripts/run_buildbot_new.sh
deleted file mode 100755
index cb747a912ab..00000000000
--- a/libcxx/utils/docker/scripts/run_buildbot_new.sh
+++ /dev/null
@@ -1,110 +0,0 @@
-#!/usr/bin/env bash
-set -x
-
-BOT_ROOT=/b
-BOT_ROOT_NAME=$1
-BOT_PASS=$2
-
-pushd /tmp
-curl -sSO https://dl.google.com/cloudagents/install-monitoring-agent.sh
-bash install-monitoring-agent.sh
-curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh
-bash install-logging-agent.sh --structured
-popd
-
-apt-get update -y
-apt-get upgrade -y
-
-apt-get install sudo -y
-
-# FIXME(EricWF): Remove this hack. It's only in place to temporarily fix linking libclang_rt from the
-# debian packages.
-# WARNING: If you're not a buildbot, DO NOT RUN!
-apt-get install lld-9
-rm /usr/bin/ld
-ln -s /usr/bin/lld-9 /usr/bin/ld
-
-systemctl set-property buildslave.service TasksMax=100000
-
-systemctl daemon-reload
-service buildslave restart
-
-function setup_numbered_bot {
-  local BOT_NAME=$1
-  local BOT_DIR=$2
-  mkdir -p $BOT_DIR
-
-  buildslave stop $BOT_DIR
-  chown buildbot:buildbot $BOT_DIR
-  rm -rf $BOT_DIR/*
-
-  buildslave create-slave --allow-shutdown=signal $BOT_DIR lab.llvm.org:9990 \
-      $BOT_NAME $BOT_PASS
-
-  echo "Eric Fiselier <ericwf@google.com>" > $BOT_DIR/info/admin
-
-  echo "Connecting as $BOT_BASE_NAME$1"
-  {
-    uname -a | head -n1
-    cmake --version | head -n1
-    g++ --version | head -n1
-    ld --version | head -n1
-    date
-    lscpu
-  } > $BOT_DIR/info/host
-
-
- # echo "SLAVE_RUNNER=/usr/bin/buildslave
- # SLAVE_ENABLED[1]=\"1\"
- # SLAVE_NAME[1]=\"buildslave$1\"
- # SLAVE_USER[1]=\"buildbot\"
- # SLAVE_BASEDIR[1]=\"$BOT_DIR\"
- # SLAVE_OPTIONS[1]=\"\"
- # SLAVE_PREFIXCMD[1]=\"\"" > $BOT_DIR/buildslave.cfg
-
-}
-
-function try_start_builder {
-  local $N=$1
-  local BOT_DIR=$BOT_ROOT/b$N
-  local BOT_NAME=$BOT_ROOT_NAME$N
-  setup_numbered_bot $BOT_NAME $BOT_DIR
-  chown -R buildbot:buildbot $BOT_DIR/
-  sudo -u buildbot /usr/bin/buildslave start $BOT_DIR
-
-  sleep 30
-  cat $BOT_DIR/twistd.log
-  if grep --quiet "slave is ready" $BOT_DIR/twistd.log; then
-    return 0
-  fi
-  if grep --quiet "configuration update complete" $BOT_DIR/twistd.log; then
-    return 0
-  fi
-  if grep "rejecting duplicate slave" $BOT_DIR/twistd.log; then
-    return 1
-  fi
-  echo "Unknown error"
-  cat $BOT_DIR/twistd.log
-  exit 1
-}
-
-for N in `shuf -i 1-5`
-do
-  if try_start_builder $N; then
-    break
-  fi
-  echo "failed to start any buildbot"
-  shutdown now
-done
-
-# GCE can restart instance after 24h in the middle of the build.
-# Gracefully restart before that happen.
-sleep 72000
-while pkill -SIGHUP buildslave; do sleep 5; done;
-shutdown now
-
-echo "Failed to start any buildbot"
-shutdown now
-
-
-