Reorganize citest scripts.

    Split pieces of the autocitest script out so that we can control the
    order of the pieces, allow some to run in parallel, etc.  This will
    enable us to build the error log parsers as part of Jenkins testing.

Change-Id: I017c7b0516673dc267eb90adf03b4604602281e6
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/10673
Tested-by: Jenkins Server
Reviewed-by: Donald E. Dahle <dedahle@us.ibm.com>
Reviewed-by: STEPHEN M. CPREK <smcprek@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

8 files changed, 275 insertions, 142 deletions

diff --git a/src/build/citest/autocitest b/src/build/citest/autocitest
index 582db3cd8..48ba60d92 100755
--- a/src/build/citest/autocitest
+++ b/src/build/citest/autocitest
@@ -107,25 +107,15 @@ fi
 ##
 DRIVER="$1"                     ##  backing tree
 export bb=$DRIVER               ##  simulate a workon
-TEST_SANDBOX="$2"               ##  test sandbox
-HBICORE_TEST="$3"               ##  path to hbicore_test.img
+HBICORE_TEST="$2"               ##  path to hbicore_test.img
 
 export SHELL="/bin/bash"        ##  Needed to fool workon to launch a bash 
                                 ##  shell, which autosim expects.
 NOWIN="--nowin"                 ##  remote execution does not have a display
 
-MACHINE=${MACHINE:-$DEFAULT_MACHINE}
-export MACHINE
-HBOT_SIMICS_SETUP="/gsa/ausgsa/home/a/n/andrewg/web/public/hostboot/simics/hbot_simics_setup.pl"
-
 ##  jenkins creates a new workspace for every compile.
 ##     cxxtest-start.sh will verify this and then export WORKSPACE_DIR for us.
-SBXHOME=$WORKSPACE_DIR
-
-##  autosim, mksb, workon, etc should look in $SANDBOXRC for the location of the sandbox.
-##  this is more effective than using -rc option (autosim has no option for this anyway)
-SANDBOXRC="$SBXHOME/hbsandboxrc"
-
+SBXHOME=$HOSTBOOTROOT
 
 if [ "$DRIVER" = "" ] ; then
         echo "ERROR: you must specify a driver (backing tree) to run the tests with"
@@ -133,12 +123,6 @@ if [ "$DRIVER" = "" ] ; then
         exit 1
 fi
 
-if [ "$TEST_SANDBOX" = "" ] ; then
-        echo "ERROR: must specify a TEST_SANDBOX="
-        usage
-        exit 1
-fi
-
 if [ ! -f "${SBXHOME}/${HBICORE_TEST}" ] ; then
         echo "ERROR: you must specify a valid hbicore_test.bin file."
         usage
@@ -148,8 +132,6 @@ fi
 ##  Find the .syms file for the image.  Remove extension, add .syms.
 HBICORE_TEST_SYMS="${HBICORE_TEST%%.*}.syms"
 
-HBICORE_TEST_EXTENDED="${HBICORE_TEST%%.*}_extended.bin"
-
 CPFILES="${SBXHOME}/src/build/tools/hbDistribute"
 
 
@@ -160,9 +142,7 @@ echo    "$0"
 echo    "VERBOSE                    = $VERBOSE"
 echo    "NOWIN                      = $NOWIN"
 echo    "BACKING_TREE               = $DRIVER"
-echo    "TEST_SANDBOX               = $TEST_SANDBOX"
 echo    "HBICORE_TEST               = $HBICORE_TEST"
-echo    "HBICORE_TEST_EXTENDED      = $HBICORE_TEST_EXTENDED"
 echo    "HBICORE_TEST_SYMS          = $HBICORE_TEST_SYMS"
 echo    "SANDBOXRC                  = $SANDBOXRC"
 echo    "SBXHOME                    = $SBXHOME"
@@ -174,57 +154,16 @@ waitkb
 timestamp=`date +'%H:%M:%S'`
 echo "$timestamp Starting autosample test..."
 
-
-# Set up the test sandbox backed to the specified build
-##  Assumption here is that we will always run inside the workspace created by 
-##  jenkins, so we always create a new sandbox
-##
-
-echo "Setting up sandbox environment..."
-
-##  mksb fills the log with "please type backing tree file" if it can't find
-##  the -back file.   We wish to avoid this.
-if [ ! -d "$DRIVER" ] ; then
-        echo "cannot access the backing tree $DRIVER , aborting"
-        exit 1
-fi
-
-##  set $SANDBOXRC to the local one, this should cause this to make and run 
-##  in a new sandbox at the base workspace dir.
-export SANDBOXRC
-export SANDBOXROOT=$SBXHOME
-export SANDBOXNAME=$TEST_SANDBOX
-export SANDBOXBASE=$SANDBOXROOT/$SANDBOXNAME
-export AUTOSIM_WORKDIR=$SANDBOXBASE
-
-if [ ! -d "$SBXHOME/$TEST_SANDBOX" ] ; then
-        echo "run mksb to create the sandbox "
-        mksb  -rc $SANDBOXRC -dir $SBXHOME -back $DRIVER -sb $TEST_SANDBOX -m ppc
-        if [ $? -ne 0 ] ; then
-            echo "ERROR: mksb failed: $?"
-            exit 1
-        fi
-else
-        echo "$SBXHOME/$TEST_SANDBOX already exists, " `ls -la`
-        `usrsbinfo -s`
-        chkdrv=`usrsbinfo -s | grep " $TEST_SANDBOX "| awk '{ print $2 }'`
-        if [ "$chkdrv" != "$DRIVER" ] ; then
-                resb $DRIVER -sb $TEST_SANDBOX
-        fi
-fi
-
-
-waitkb
-
 # Run set up in current shell
 echo    "run autosimsetup.."
+export AUTOSIM_WORKDIR=$SANDBOXBASE
 . autosimsetup
 
 waitkb
 
 # Start the simics server
-echo "Starting simics server in sandbox $TEST_SANDBOX on machine $MACHINE"
-autosim $NOWIN $VERBOSE --startserver --wp --sandbox $TEST_SANDBOX --machine $MACHINE
+echo "Starting simics server in sandbox $SANDBOXNAME on machine $MACHINE"
+autosim $NOWIN $VERBOSE --startserver --wp --sandbox $SANDBOXNAME --machine $MACHINE
 if [ $? -ne 0 ] ; then
         echo "ERROR: Unable to start simics server $?"
         exit 1
@@ -232,62 +171,23 @@ fi
 
 waitkb
 
-##  under normal circumstances $sb would be set by start_simics
-export sb=$SBXHOME/$TEST_SANDBOX/src
-echo "sandbox dir = $sb"
-
-echo "Running workarounds (presimsetup)."
-if [ -f ${SBXHOME}/src/build/citest/etc/workarounds.presimsetup ] ; then
-    ${SBXHOME}/src/build/citest/etc/workarounds.presimsetup 
-fi
-
-echo "** Set up simics directories **"
-echo "start_simics -no_start -machine $MACHINE  -batch_mode " > $sb/my_simics_env_setup
-
-# Do a workon and immediately execute the above commands
-chmod 775 $sb/my_simics_env_setup
-workon -sb $TEST_SANDBOX -mppc -c ./my_simics_env_setup
-
-echo "** Executing workarounds (postsimsetup) **"
-if [ -f ${SBXHOME}/src/build/citest/etc/workarounds.postsimsetup ] ; then
-    ${SBXHOME}/src/build/citest/etc/workarounds.postsimsetup
-fi
-
-echo "** Create img directory for hbicore.bin **"
-mkdir -p $sb/../img
-
-
-waitkb
-
-
-echo "Copy .syms, bin, hbotStringfile, etc to base sandbox dir."
-echo "$sb"
-$CPFILES --test
-if [ $? -ne 0 ] ; then
-    echo "ERROR: $CPFILES failed: $?"
-    exit 1
-fi
-
-waitkb
-
-
 # Start simics and wait for it to reach standby
 echo "Starting simics"
 
 # If we are running under Jenkins, do "no remote execution" because built in
 # /tmp.
-if [ -z $JOB_NAME -o -z $BUILD_NUMBER ];
+if [ -z $RUNNING_UNDER_JENKINS -a -z $SANDBOX_IN_TEMP ];
 then
     NRE=""
 else
     # Trick autosim into thinking we're running on a vexec system.
     echo "echo \"Running on server $HOSTNAME ---\"" >> \
             ${SANDBOXBASE}/obj/ppc/simu/scripts/hbfw/startup.simics
-    NRE="-nre"
+    NRE="--sim_parms -nre"
 fi
 
 #autosim $VERBOSE --startsim --chkpt standby <<< chkpt not working yet
-autosim $NOWIN $VERBOSE --startsim --sim_parms $NRE
+autosim $NOWIN $VERBOSE --startsim $NRE
 if [ $? -ne 0 ] ; then
         echo "ERROR: Unable to start simics $?"
         stopserver      ##  kill the server again.
diff --git a/src/build/citest/build-script b/src/build/citest/build-script
index 6f53a7735..12cdb43bc 100755
--- a/src/build/citest/build-script
+++ b/src/build/citest/build-script
@@ -22,8 +22,48 @@
 #
 # IBM_PROLOG_END_TAG
 
-source "$WORKSPACE/env.bash"
+if [ -z $HOSTBOOTROOT ]; then
+    source "$WORKSPACE/env.bash"
+fi
+
+source "$HOSTBOOTROOT/src/build/citest/setup-env"
+
 #export HOSTBOOT_DEBUG=1    #-- Ensure printkd / tracd is compiled in.
 
+# Check copyright.
+check-copyright > copyright.log 2>&1 &
+COPYRIGHT_PID=$!
+
+# Create simics sandbox.
+create-sandbox > create-sandbox.log 2>&1 &
+CREATESANDBOX_PID=$!
+
+# Build Hostboot.
 make -j32 || exit -1
-$WORKSPACE/src/build/citest/cxxtest-start.sh || exit -1
+
+# Check sandbox create completion.
+wait $CREATESANDBOX_PID
+if [ $? -eq 0 ]; then
+    cat create-sandbox.log
+else
+    echo "----Sandbox creation failed."
+    cat create-sandbox.log
+    exit -1
+fi
+
+# Add Hostboot files to simics sandbox.
+populate-sandbox || exit -1
+
+# Start CxxTest Simics execution.
+cxxtest-start.sh || exit -1
+
+# Check copyright completion.
+wait $COPYRIGHT_PID
+if [ $? -eq 0 ]; then
+    cat copyright.log
+else
+    echo "----Copyright check failed."
+    cat copyright.log
+    exit -1
+fi
+
diff --git a/src/build/citest/check-copyright b/src/build/citest/check-copyright
new file mode 100755
index 000000000..10542d578
--- /dev/null
+++ b/src/build/citest/check-copyright
@@ -0,0 +1,27 @@
+#!/bin/sh
+# IBM_PROLOG_BEGIN_TAG
+# This is an automatically generated prolog.
+#
+# $Source: src/build/citest/check-copyright $
+#
+# IBM CONFIDENTIAL
+#
+# COPYRIGHT International Business Machines Corp. 2014
+#
+# p1
+#
+# Object Code Only (OCO) source materials
+# Licensed Internal Code Source Materials
+# IBM HostBoot Licensed Internal Code
+#
+# The source code for this program is not published or otherwise
+# divested of its trade secrets, irrespective of what has been
+# deposited with the U.S. Copyright Office.
+#
+# Origin: 30
+#
+# IBM_PROLOG_END_TAG
+
+COPYRIGHT_CHECK=${HOSTBOOTROOT}/src/build/tools/copyright-check.sh
+
+$COPYRIGHT_CHECK || exit -1
diff --git a/src/build/citest/create-sandbox b/src/build/citest/create-sandbox
new file mode 100755
index 000000000..10a332078
--- /dev/null
+++ b/src/build/citest/create-sandbox
@@ -0,0 +1,70 @@
+#!/bin/sh
+# IBM_PROLOG_BEGIN_TAG
+# This is an automatically generated prolog.
+#
+# $Source: src/build/citest/create-sandbox $
+#
+# IBM CONFIDENTIAL
+#
+# COPYRIGHT International Business Machines Corp. 2014
+#
+# p1
+#
+# Object Code Only (OCO) source materials
+# Licensed Internal Code Source Materials
+# IBM HostBoot Licensed Internal Code
+#
+# The source code for this program is not published or otherwise
+# divested of its trade secrets, irrespective of what has been
+# deposited with the U.S. Copyright Office.
+#
+# Origin: 30
+#
+# IBM_PROLOG_END_TAG
+
+echo "Creating ODE sandbox..."
+
+# Ensure backing build is accessible.
+if [ ! -d "$BACKING_BUILD" ];
+then
+    echo "Cannot access the backing build: $BACKING_BUILD"
+    exit -1
+fi
+
+# Delete existing sandbox if it exists.
+if [ -d "$SANDBOXBASE" ];
+then
+    echo "----Removing old sandbox."
+    mksb -undo -auto -rc $SANDBOXRC -dir $SANDBOXROOT -sb $SANDBOXNAME
+fi
+if [ -d "$SANDBOXBASE" ];
+then
+    rm -rf $SANDBOXBASE
+fi
+
+# Create sandbox.
+mksb -rc $SANDBOXRC -dir $SANDBOXROOT -back $BACKING_BUILD -sb $SANDBOXNAME \
+      -m ppc  || exit -1
+
+# workaround scripts expect $sb variable to be populated.
+export sb=$SANDBOXBASE/src
+
+# Run presimsetup workarounds.
+echo "----Running presimsetup workarounds."
+if [ -f ${CITESTPATH}/etc/workarounds.presimsetup ];
+then
+    ${CITESTPATH}/etc/workarounds.presimsetup || exit -1
+fi
+
+# Run start_simics to populate simics directories.
+echo "----Setting up simics."
+execute_in_sandbox "start_simics -no_start -machine $MACHINE -batch_mode" \
+                   "ppc" || exit -1
+
+# Run postsimsetup workarounds.
+echo "----Running postsimsetup workarounds."
+if [ -f ${CITESTPATH}/etc/workarounds.postsimsetup ];
+then
+    ${CITESTPATH}/etc/workarounds.postsimsetup || exit -1
+fi
+
diff --git a/src/build/citest/cxxtest-start.sh b/src/build/citest/cxxtest-start.sh
index 91bdd351f..077d04b17 100755
--- a/src/build/citest/cxxtest-start.sh
+++ b/src/build/citest/cxxtest-start.sh
@@ -22,41 +22,17 @@
 #
 # IBM_PROLOG_END_TAG
 #
-export WORKSPACE_DIR=`pwd`
-export COPYRIGHT_CHECK=${WORKSPACE_DIR}/src/build/tools/copyright-check.sh
-export AUTOCITEST=${WORKSPACE_DIR}/src/build/citest/autocitest
-export BACKING_BUILD=`cat ${WORKSPACE_DIR}/src/build/citest/etc/bbuild`
-HOSTBOOT_IMAGE=img/hbicore_test.bin
-
-if [ -z $JOB_NAME -o -z $BUILD_NUMBER ];
+if [ -z $HOSTBOOT_CI_ENV_SETUP ];
 then
-    CXXTEST_REMOTE_SANDBOX="cxxtest_sb"
-else
-    CXXTEST_REMOTE_SANDBOX=`echo $JOB_NAME $BUILD_NUMBER | md5sum | head -c10`
+    source "$HOSTBOOTROOT/src/build/citest/setup-env"
 fi
 
-#
-#   Validate Copyright blocks
-if [ -f $COPYRIGHT_CHECK ]; then
-    $COPYRIGHT_CHECK
-    if [ $? -ne 0 ]; then
-        exit 1
-    fi
-fi
+HOSTBOOT_IMAGE=img/hbicore_test.bin
 
-#
 #   Front end to autocitest - script to execute unit tests under simics.
 #
 ##  when jenkins runs it will create a workspace with the built code tree
 ##  and drop us into it.
-
-
-
-##  klog to get access to simics tools
-AFSPASSWD="/gsa/ausgsa/projects/h/hostboot/gerrit-server/afspasswd.sh"
-$AFSPASSWD | klog -cell austin -pipe
-$AFSPASSWD | klog -cell rchland -pipe
-
-$AUTOCITEST ${BACKING_BUILD} ${CXXTEST_REMOTE_SANDBOX} ${HOSTBOOT_IMAGE}
+autocitest ${BACKING_BUILD} ${HOSTBOOT_IMAGE}
 
 exit $?
diff --git a/src/build/citest/populate-sandbox b/src/build/citest/populate-sandbox
new file mode 100755
index 000000000..d256caeef
--- /dev/null
+++ b/src/build/citest/populate-sandbox
@@ -0,0 +1,42 @@
+#!/bin/sh
+# IBM_PROLOG_BEGIN_TAG
+# This is an automatically generated prolog.
+#
+# $Source: src/build/citest/populate-sandbox $
+#
+# IBM CONFIDENTIAL
+#
+# COPYRIGHT International Business Machines Corp. 2014
+#
+# p1
+#
+# Object Code Only (OCO) source materials
+# Licensed Internal Code Source Materials
+# IBM HostBoot Licensed Internal Code
+#
+# The source code for this program is not published or otherwise
+# divested of its trade secrets, irrespective of what has been
+# deposited with the U.S. Copyright Office.
+#
+# Origin: 30
+#
+# IBM_PROLOG_END_TAG
+
+# Ensure sandbox exists (create-sandbox has been called).
+if [ -z $SANDBOXBASE ];
+then
+    echo "SANDBOXBASE variable uninitialized.."
+    exit -1
+fi
+
+if [ ! -d "$SANDBOXBASE" ];
+then
+    echo "Cannot find sandbox: $SANDBOXBASE"
+    exit -1
+fi
+
+# Create img directory.
+mkdir -p $SANDBOXBASE/img || exit -1
+
+# Copy files.
+hbDistribute --test || exit -1
diff --git a/src/build/citest/setup-env b/src/build/citest/setup-env
new file mode 100755
index 000000000..a351cce98
--- /dev/null
+++ b/src/build/citest/setup-env
@@ -0,0 +1,78 @@
+#!/bin/sh
+# IBM_PROLOG_BEGIN_TAG
+# This is an automatically generated prolog.
+#
+# $Source: src/build/citest/setup-env $
+#
+# IBM CONFIDENTIAL
+#
+# COPYRIGHT International Business Machines Corp. 2014
+#
+# p1
+#
+# Object Code Only (OCO) source materials
+# Licensed Internal Code Source Materials
+# IBM HostBoot Licensed Internal Code
+#
+# The source code for this program is not published or otherwise
+# divested of its trade secrets, irrespective of what has been
+# deposited with the U.S. Copyright Office.
+#
+# Origin: 30
+#
+# IBM_PROLOG_END_TAG
+
+export CITESTPATH=${HOSTBOOTROOT}/src/build/citest
+export PATH=${CITESTPATH}:${PATH}
+
+# Determine backing build.
+export BACKING_BUILD=`cat ${CITESTPATH}/etc/bbuild`
+
+# If we are running under Jenkins we need to pick a random-ish sandbox name
+# so that the autoipl tools do not have a /tmp name collision.
+if [ -z $JOB_NAME -o -z $BUILD_NUMBER ];
+then
+    CXXTEST_SANDBOX="cxxtest_sb"
+else
+    export RUNNING_UNDER_JENKINS=1
+    CXXTEST_SANDBOX=`echo $JOB_NAME $BUILD_NUMBER | md5sum | head -c10`  
+fi
+
+# Setup sandbox location variables.
+if [ -z $SANDBOXROOT ];
+then
+    export SANDBOXROOT=${HOSTBOOTROOT}
+fi
+export SANDBOXNAME=${CXXTEST_SANDBOX}
+export SANDBOXBASE=${SANDBOXROOT}/${SANDBOXNAME}
+export SANDBOXRC=${SANDBOXROOT}/hbsandboxrc
+
+# Useful utility function.
+execute_in_sandbox()
+{
+    WORKON_CMD="workon -rc ${SANDBOXRC} -sb ${SANDBOXNAME} -m $2 "
+    echo $1 > ${SANDBOXBASE}/src/sandbox_execute_cmd
+    chmod 700 ${SANDBOXBASE}/src/sandbox_execute_cmd
+
+    ${WORKON_CMD} -c ./sandbox_execute_cmd
+
+    if [ $? -ne 0 ]; then
+        return -1
+    fi
+
+    rm ${SANDBOXBASE}/src/sandbox_execute_cmd
+}
+export -f execute_in_sandbox
+
+# Determine if sandbox is in /tmp, which requires -nre to simics.
+echo $SANDBOXROOT | grep "/tmp" > /dev/null
+if [ $? -eq 0 ]; 
+then
+    export SANDBOX_IN_TEMP=1
+fi
+
+# Setup machine type.
+export MACHINE=${MACHINE:-$DEFAULT_MACHINE}
+
+# Indicate we setup the CI environment.
+export HOSTBOOT_CI_ENV_SETUP=1
diff --git a/src/build/tools/hb b/src/build/tools/hb
index f5690f990..3405277eb 100755
--- a/src/build/tools/hb
+++ b/src/build/tools/hb
@@ -6,7 +6,7 @@
 #
 # IBM CONFIDENTIAL
 #
-# COPYRIGHT International Business Machines Corp. 2011,2013
+# COPYRIGHT International Business Machines Corp. 2011,2014
 #
 # p1
 #
@@ -444,16 +444,16 @@ hb_errlparser()
     # Trigger a build of the hostboot plugins we shipped with the
     # hb prime cmd.
     execute_in_sandbox "cd ${SANDBOXBASE}/src/hbfw/ \
-                                             && mk -a "  "x86.nfp"
+                                             && mk -a -j32"  "x86.nfp"
 
     # build srci in x86 context
     execute_in_sandbox "cd ${SANDBOXBASE}/src/srci \
-                                             && mk -a "  "x86.nfp"
+                                             && mk -a -j32"  "x86.nfp"
 
     # build errl in x86 context, to pick-up the parser libs from
     # hostboot.
     execute_in_sandbox "cd ${SANDBOXBASE}/src/errl \
-                                             && mk -a "  "x86.nfp"
+                                             && mk -a -j32"  "x86.nfp"
 
     cd ${SANDBOXBASE}/simics
     tar -xvf  $SANDBOXBASE/obj/x86.nfp/errl/nfp/tool/errl.tar  errl