New functions added for automating mustfix releases

Change-Id: Ib098262d51acbda5fe8f67450c86da097648f49c
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/52383
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Prachi Gupta <pragupta@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

2 files changed, 469 insertions, 3 deletions

diff --git a/src/build/citest/auto-release b/src/build/citest/auto-release
index ba5105c31..104b5f5d0 100755
--- a/src/build/citest/auto-release
+++ b/src/build/citest/auto-release
@@ -36,6 +36,259 @@ function setup {
     fi
 }
 
+# Overview of how must-fix release work:
+# These steps are used by releases requiring must-fix approvals
+# First, function "build-create-track" builds a release & creates a CMVC track
+# It sends approver(s) an email with fips-rel, feature/track and commits info.
+# We call another funtion "chk-n-wait-approval".
+# It allows them a time of 1 day (wait-time input taken from Jenkins)
+# It emails approvers and loops continually to check if track was approved.
+# Post approval, it exits the loop, calls next function "checkinBuildToTrack"
+# which locks files, adds them to track, and completes(not integrate) track.
+# Next, Jenkins job "fspCI-n-CMVCtrack-integrate" runs fsp-CI and if successful
+# integrates the track - this is the last step of the must-fix release
+
+function buildrel-create-track {
+
+    echo -e "\nInside \"function buildrel-create-track\"\n"
+    echo -e "RELEASE_NAME=$RELEASE_NAME\n"
+    echo -e "PREVIOUS_RELEASE=$PREVIOUS_RELEASE\n"
+    echo -e "FIPS_RELEASE=$FIPS_RELEASE\n"
+    echo -e "GIT_BRANCH=$GIT_BRANCH\n"
+    echo -e "BASE=$BASE\n"
+    echo -e "EXISTING_TRACK=$EXISTING_TRACK\n"
+    echo -e "RELEASENOTESTXT=$RELEASENOTESTXT\n"
+
+    ### Define and setup release branch
+    export PATH=${PATH}:$HOSTBOOT_WORKSPACE/src/build/tools
+
+    echo -e "[$(date)]\n#####-> hbRelease define --level $RELEASE_NAME\
+        --released $PREVIOUS_RELEASE --base $BASE --branch $GIT_BRANCH\n"
+    hbRelease define --level $RELEASE_NAME --released $PREVIOUS_RELEASE\
+        --base $BASE --branch $GIT_BRANCH || exit -1
+
+    echo -e "[$(date)]\n#####-> hbRelease release --level $RELEASE_NAME\n"
+    hbRelease release --level $RELEASE_NAME || exit -1
+
+    export SANDBOXROOT=$WORKSPACE
+    export SANDBOXNAME=$RELEASE_NAME
+    export SANDBOXBASE=$SANDBOXROOT/$SANDBOXNAME
+    mkdir -p $SANDBOXBASE || exit -1
+    mkdir -p $CMVC_DIR/src/hbfw/nimbus || exit -1
+    mkdir -p $CMVC_DIR/src/hbfw/cumulus || exit -1
+
+    ### Compile and provide binaries
+    setup
+    if echo $FIPS_RELEASE | grep -q fips9; then
+        # First bulid the standalone
+
+        echo -e "[$(date)]\n#####-> BUILD_MINIMAL=1 nice -+20 make -j16\n"
+        BUILD_MINIMAL=1 nice -+20 make -j16 || exit -1
+
+        # With PNOR env var not set, hb prime will make all of the pnor images
+        unset PNOR
+        # prime will put the files into expected locations
+
+        echo -e "[$(date)]\n#####-> hb simsetup || exit -1\n"
+        hb simsetup || exit -1
+        echo -e "[$(date)]\n#####-> hb prime || exit -1\n"
+        hb prime || exit -1
+        echo -e "[$(date)]\n#####-> cp \
+            $SANDBOXBASE/obj/ppc/hbfw/img/nimbus.pnor \
+            $CMVC_DIR/src/hbfw/nimbus || exit -1\n"
+        cp $SANDBOXBASE/obj/ppc/hbfw/img/nimbus.pnor \
+            $CMVC_DIR/src/hbfw/nimbus || exit -1
+        echo -e "[$(date)]\n#####-> cp $SANDBOXBASE/src/hbfw/simics.tar \
+            $CMVC_DIR/src/hbfw/nimbus || exit -1\n"
+        cp $SANDBOXBASE/src/hbfw/simics.tar $CMVC_DIR/src/hbfw/nimbus \
+            || exit -1
+        echo -e "[$(date)]\n#####-> cp \
+            $SANDBOXBASE/obj/ppc/hbfw/img/cumulus.pnor \
+            $CMVC_DIR/src/hbfw/cumulus\n"
+        cp $SANDBOXBASE/obj/ppc/hbfw/img/cumulus.pnor \
+            $CMVC_DIR/src/hbfw/cumulus || exit -1
+        echo -e "[$(date)]\n#####-> cp $SANDBOXBASE/src/hbfw/simics.tar \
+            $CMVC_DIR/src/hbfw/cumulus || exit -1\n"
+        cp $SANDBOXBASE/src/hbfw/simics.tar $CMVC_DIR/src/hbfw/cumulus \
+            || exit -1
+
+        echo -e "[$(date)]\n#####-> make clean\n"
+        #make clean
+        make clobber
+
+        # Now build the FSP-based images
+        export CONFIG_FILE="$HOSTBOOT_WORKSPACE/src/build/configs/fsprelease.config"
+
+        echo -e "[$(date)]\n#####-> BUILD_MINIMAL=1 nice -+20 make -j16 \n"
+        BUILD_MINIMAL=1 nice -+20 make -j16 || exit -1
+        #
+    else
+        #
+        echo -e "[$(date)]\n##### -> BUILD_MINIMAL=1 nice -+20 make -j16 \n"
+        BUILD_MINIMAL=1 nice -+20 make -j16 || exit -1
+    fi
+
+    # Run hb distribute to put binaries in cmvc checkout directory
+
+    echo -e "[$(date)]\n#####->  hb prime --release $SANDBOXBASE || exit -1\n"
+    hb prime --release $SANDBOXBASE || exit -1
+
+    echo -e "[$(date)]\n#####-> cp -rf $SANDBOXBASE/fsp/* $CMVC_DIR/src/hbfw\n"
+    cp -rf $SANDBOXBASE/fsp/* $CMVC_DIR/src/hbfw || exit -1
+
+    echo -e "[$(date)]\n#####-> rm -rf $SANDBOXBASE/fsp || exit -1\n"
+    rm -rf $SANDBOXBASE/fsp || exit -1
+
+    ### Push tag to gerrit, and create track
+    echo -e "[$(date)]\n##### -> \
+        git push ssh://hostboot-us@gerrit-server/hostboot $RELEASE_NAME\n"
+    git push ssh://hostboot-us@gerrit-server/hostboot $RELEASE_NAME || exit -1
+
+    #!--------Remove these lines of codes after merging hostboot commit,
+    #! this temporary adj is to reinstate the files---!#
+    cp src/build/citest/auto-release src/build/citest/auto-release.save
+    cp src/build/tools/hbRelease src/build/tools/hbRelease.save
+    #!--------------------------------------------------!#
+
+    echo -e "[$(date)]\n##### -> git checkout gerrit/master || exit -1\n"
+    git checkout gerrit/master || exit -1
+
+    #!--------Remove these lines of codes after merging hostboot commit,
+    #! this temporary adj is to reinstate the files---!#
+    cp src/build/citest/auto-release.save  src/build/citest/auto-release
+    cp src/build/tools/hbRelease.save  src/build/tools/hbRelease
+    #!--------------------------------------------------!#
+
+    echo -e "[$(date)]\n#####->  hbRelease create-track --level $RELEASE_NAME\
+        --release $FIPS_RELEASE --released $PREVIOUS_RELEASE --track \
+        $EXISTING_TRACK || exit -1\n"
+    hbRelease create-track --level $RELEASE_NAME --release $FIPS_RELEASE \
+        --released $PREVIOUS_RELEASE --track $EXISTING_TRACK || exit -1
+    # Check if feature was successfully created and if so set $FEATURE
+    if [ -f $CMVC_FEATURE_FILE ]; then
+        FEATURE=`cat $CMVC_FEATURE_FILE`;
+    else
+        echo "No feature created"
+        exit -1
+    fi
+
+    ### @todo RTC:171143
+    ### Sleep for 5 minutes to allow time for CMVC req's to take effect
+    echo "Waiting for fspCI requirements to take effect. 5 minutes..."
+    sleep 5m
+    echo -e "\nExiting \"function buildrel-create-track\"\n";
+}
+
+function chk-n-wait-approval {
+    echo -e "\nInside \"function chk-n-wait-approval\"\n"
+    echo -e "RELEASE_NAME=$RELEASE_NAME\n"
+    echo -e "FIPS_RELEASE=$FIPS_RELEASE\n"
+    echo -e "PREVIOUS_RELEASE=$PREVIOUS_RELEASE\n"
+    echo -e "EXISTING_TRACK=$EXISTING_TRACK\n"
+    #
+    local WAITED_TIME_SECS=0
+    local TMOUTSEC=$WAIT_TIME_SECS
+    local GOT_APPROVAL=""
+    local APPROVAL_STATUS=-1
+    local TRACK_APPROVED_YET="no"
+    local TRACK_APPROVAL_FILE=$WORKSPACE/track_approval_file
+    local EXPECTED_STATE="fix"
+    #
+    export NEXTJOB="https://hostboot-jenkins.swg-devops.com/job/Hostboot/job/Release/job/\
+    auto-release-HB/job/rel-fips/job/rel-fips-4-checkinBuildToTrack/\
+    build?delay=0sec"
+    echo -e "[$(date)]\nUse the information below  to manually perform next\
+        steps of the mustfix release:" >  $TRACK_APPROVAL_FILE
+    echo -e "\n 1. Point your browser to \n$NEXTJOB\n" >> $TRACK_APPROVAL_FILE
+    echo -e "\n 2. Copy artifacts by supplying information \
+        below:\n" >> $TRACK_APPROVAL_FILE
+    echo -e "\tFIPS_RELEASE=$FIPS_RELEASE\
+        \n\tFEATURE=$FEATURE" >> $TRACK_APPROVAL_FILE
+    echo -e "\tREL_FIPS_TRIGGER_PROJ=$REL_FIPS_TRIGGER_PROJ\
+        \n\tREL_FIPS_TRIGGER_BUILD_NUM=$REL_FIPS_TRIGGER_BUILD_NUM"\
+        >> $TRACK_APPROVAL_FILE
+    echo -e "\n\n\tOptionally supply the following" >> $TRACK_APPROVAL_FILE
+    echo -e "\tMUSTFIX_APPROVERS=$MUSTFIX_APPROVERS" >> $TRACK_APPROVAL_FILE
+    echo -e "\tRELEASE_ADMINS=$RELEASE_ADMINS" >> $TRACK_APPROVAL_FILE
+    echo -e "\n\nPeriodic polling status below:" >> $TRACK_APPROVAL_FILE
+
+    if [[ "$REL_NEEDS_APPR" == "no" ]]; then
+        echo -e "\n<E-O-F>\n" >> $TRACK_APPROVAL_FILE
+        APPROVAL_STATUS=0
+    else
+        #Get approval for must-fix release
+        while true
+        do
+            #Poll for approval - if it is 'approve', it is not approved yet.
+            #If it is 'fix' or 'build', then it is approved
+            GOT_APPROVAL=$(Report -g TrackView -where\
+                "releaseName='$FIPS_RELEASE' and defectName='$FEATURE'"\
+                -select state)
+            echo "{$(date)]: GOT_APPROVAL=\"$GOT_APPROVAL\" - \
+                [Waiting will quit for state: \"$EXPECTED_STATE\"]"\
+                >> $TRACK_APPROVAL_FILE
+            GOT_APPROVAL=$(grep "GOT_APPROVAL=\"$EXPECTED_STATE\"" \
+                $TRACK_APPROVAL_FILE)
+            if [[ -z "$GOT_APPROVAL" && $WAITED_TIME_SECS -lt \
+                "$WAIT_TIME_SECS" ]];then
+                echo -e "APPROVAL STATE=\"$GOT_APPROVAL\" \
+                [EXPECTED_STATE=$EXPECTED_STATE]\n CMVC track NOT approved yet \
+                - waiting for approval, will timeout in $TMOUTSEC seconds \
+                [WAIT_TIME_SECS=$WAIT_TIME_SECS; \
+                WAITED_TIME_SECS=$WAITED_TIME_SECS] ......"
+                /bin/sleep $POLL_AFTER_SECS
+                WAITED_TIME_SECS=$((WAITED_TIME_SECS+POLL_AFTER_SECS))
+                TMOUTSEC=$((WAIT_TIME_SECS-WAITED_TIME_SECS))
+            else
+                echo -e "\n<E-O-F>\n" >> $TRACK_APPROVAL_FILE
+                MAIL_MSG="Track $FIPS_RELEASE:$FEATURE approval status:\
+                \n\tAPPROVAL STATE:\n\t\"$GOT_APPROVAL\"\
+                \n\n\t[Expected state: \"$EXPECTED_STATE\"]"
+                export MAIL_MSG="$MAIL_MSG\n\n\nContent of approval file \
+                    below:\n$(cat $TRACK_APPROVAL_FILE)\n"
+                if [[ -n "$GOT_APPROVAL" ]];then
+                    export MAIL_SUB="Approval status of Track $FIPS_RELEASE \
+                    : $FEATURE [state = $EXPECTED_STATE, after \
+                    $WAITED_TIME_SECS/$WAIT_TIME_SECS secs]"
+                    APPROVAL_STATUS=0
+                else
+                    APPROVAL_STATUS=-1
+                    export MAIL_SUB="Failed approval of Track $FIPS_RELEASE \
+                    : $FEATURE [after $WAITED_TIME_SECS/$WAIT_TIME_SECS secs]"
+                fi
+                echo -e "$MAIL_SUB\n$MAIL_MSG\n"
+                echo -e "$MAIL_MSG" | \
+                    mail -s "$MAIL_SUB" $RELEASE_ADMINS $MUSTFIX_APPROVERS
+                break
+            fi
+        done
+    fi
+
+    echo -e "\nExiting \"function chk-n-wait-approval\"\n";
+    exit $APPROVAL_STATUS
+}
+
+function checkinBuildToTrack {
+    ### Define and setup release branch
+    export PATH=${PATH}:$HOSTBOOT_WORKSPACE/src/build/tools
+    #
+    echo "#####-> hbRelease define --level $RELEASE_NAME \
+        --released $PREVIOUS_RELEASE --base $BASE --branch $GIT_BRANCH"
+    hbRelease define --level $RELEASE_NAME --released $PREVIOUS_RELEASE \
+    --base $BASE --branch $GIT_BRANCH || exit -1
+    #
+    echo "#####-> hbRelease release --level $RELEASE_NAME || exit -1"
+    hbRelease release --level $RELEASE_NAME || exit -1
+
+    hbRelease checkinBuildToTrack --level $RELEASE_NAME \
+    --release $FIPS_RELEASE --checkInDir $CMVC_DIR \
+    --released $PREVIOUS_RELEASE || exit -1
+    ### @todo RTC:171143
+    ### Sleep for 5 minutes to allow time for CMVC req's to take effect
+    echo "Waiting for fspCI requirements to take effect. 5 minutes..."
+    sleep 5m
+}
+
 function build {
     ### Define and setup release branch
     export PATH=${PATH}:$HOSTBOOT_WORKSPACE/src/build/tools
@@ -67,7 +320,8 @@ function build {
         cp $SANDBOXBASE/obj/ppc/hbfw/img/cumulus_cdimm.pnor $CMVC_DIR/src/hbfw/cumulus_cdimm || exit -1
         cp $SANDBOXBASE/src/hbfw/simics.tar $CMVC_DIR/src/hbfw/cumulus_cdimm || exit -1
 
-        make clean
+        #make clean
+        make clobber
         # Now build the FSP-based images
         export CONFIG_FILE="$HOSTBOOT_WORKSPACE/src/build/configs/fsprelease.config"
         BUILD_MINIMAL=1 nice -+20 make -j16 || exit -1
@@ -94,6 +348,7 @@ function build {
 
     ### @todo RTC:171143
     ### Sleep for 5 minutes to allow time for CMVC req's to take effect
+
     echo "Waiting for fspCI requirements to take effect. 5 minutes..."
     sleep 5m
 }
@@ -175,7 +430,9 @@ function run_fsp_ci {
     while true;
     do
         ((retries--))
-        OUTPUT="$(fsp-CI-jenkins -r $FIPS_RELEASE -t $FEATURE --test_on_hardware=1 --force)";
+
+        # Call fsp-CI without force; omitting force allows fsp-CI to kill duplicate jobs
+        OUTPUT="$(fsp-CI-jenkins -r $FIPS_RELEASE -t $FEATURE --test_on_hardware=1)"
 
         # Check if fsp-CI-jenkins failed to start
         if [ $? -eq 0 ]; then
@@ -219,7 +476,7 @@ then
     exit 1
 fi
 
-while getopts "btch" opt; do
+while getopts "btcmwfh" opt; do
     case $opt in
         b)
             build
@@ -232,6 +489,15 @@ while getopts "btch" opt; do
         c)
             run_fsp_ci
             ;;
+        m)
+            buildrel-create-track
+            ;;
+        w)
+            chk-n-wait-approval
+            ;;
+        f)
+            checkinBuildToTrack
+            ;;
         h)
             usage
             ;;
diff --git a/src/build/tools/hbRelease b/src/build/tools/hbRelease
index dd783e9dc..07ae63d32 100755
--- a/src/build/tools/hbRelease
+++ b/src/build/tools/hbRelease
@@ -95,6 +95,8 @@ my %commands = ( "define" => \&execute_define,
                  "gerrit-commit" => \&execute_gerrit_commit,
                  "pre-release" => \&execute_pre_release,
                  "post-release" => \&execute_post_release,
+                 "create-track" => \&execute_create_track,
+                 "checkinBuildToTrack" => \&execute_checkinBuildToTrack,
                  "help" => \&execute_help,
                );
 
@@ -499,6 +501,198 @@ sub execute_pre_release
     print_debug_info($level, $feature, $cq_id, $released);
 }
 
+sub execute_create_track
+{
+    my $level = "";
+    my $release = "";
+    my $released = "";
+    my $track = "";
+
+    GetOptions("level:s" => \$level);
+    GetOptions("release:s" => \$release);
+    GetOptions("released:s" => \$released);
+    GetOptions("track:s" => \$track);
+
+    die "Missing level" if ($level eq "");
+    die "Missing release" if ($release eq "");
+    die "Missing released" if ($released eq "");
+
+    #
+    my $feature = "";
+    my $rel_needs_appr = "";
+    my $approval_flag = "yes";
+
+    if($track eq "")
+    {
+        # CMVC open feature
+        my $open_feature = "Feature -open -component $globals{component}";
+        $open_feature .= " -remarks \"Hostboot Release - $level\" -verbose";
+        # Get the Feature number from the output, should be the only numbers.
+        print "open_feature=> $open_feature\n";
+        $feature = run_system_command($open_feature);
+        $feature =~ s/[^0-9]//g;
+        #
+        print "\n\n--------------------------------------\n";
+        print "level (passed as \"RELEASE_NAME\") => $level\n";
+        print "release (passed as \"FIPS_RELEASE\") => $release\n";
+        print "released (passed as \"PREVIOUS_RELEASE\") => $released\n";
+        print "owner=> $globals{cmvcUser}\n";
+        print "feature=> $feature\n\n";
+        print "\n--------------------------------------\n\n";
+
+        # CMVC assign feature
+        print "#####-> run_system_command(\"Feature -assign $feature -owner $globals{cmvcUser} -verbose\")\n";
+        run_system_command("Feature -assign $feature -owner $globals{cmvcUser} -verbose");
+
+        # CMVC accept feature
+        print "#####-> run_system_command(\"Feature -accept $feature -verbose\");\n";
+        run_system_command("Feature -accept $feature -verbose");
+
+        # CMVC create track
+        print "#####-> run_system_command(\"Track -create -release $release -feature $feature -verbose\")\n";
+        run_system_command("Track -create -release $release -feature $feature -verbose");
+
+        # CMVC check track
+        print "#####-> $rel_needs_appr = run_system_command(\"Report -g ReleaseView -where \"name='$release'\" -select approve\")\n";
+        chomp($rel_needs_appr = run_system_command("Report -g ReleaseView -where \"name='$release'\" -select approve"));
+        print "\nrel_needs_appr=> \"$rel_needs_appr\"\n";
+        print "\napproval_flag=> \"$approval_flag\"\n";
+    }
+    else
+    {
+        $feature = $track;
+    }
+    #
+    my $featureFile = $ENV{CMVC_FEATURE_FILE};
+    open(CMVC_FEATURE,">",$featureFile)
+        or die("Cannot open: $featureFile: $!");
+    print CMVC_FEATURE "FEATURE=$feature";
+    close(CMVC_FEATURE);
+
+    my $rel_needs_appr_file = $ENV{REL_NEEDS_APPR_FILE};
+    open(REL_NEEDS_APPR,">",$rel_needs_appr_file)
+        or die("Cannot open: $rel_needs_appr_file: $!");
+    print REL_NEEDS_APPR "REL_NEEDS_APPR=$rel_needs_appr";
+    close(REL_NEEDS_APPR);
+
+    my $notify_email_file = $ENV{'NOTIFY_EMAIL_FILE'};
+    open(NOTIFY_EMAIL_TXT,">",$notify_email_file)
+        or die("Cannot open: $notify_email_file: $!");
+
+    chomp(my $datenow = run_system_command("date"));
+    #
+    open(TXTRELNOTE, "< ", $ENV{'RELEASENOTESTXT'});
+    my $txtrelnote = <TXTRELNOTE>;
+    close(TXTRELNOTE);
+    #
+    my $NEXTJOB=\
+    "https://hostboot-jenkins.swg-devops.com/job/Hostboot/job/Release/job/\
+    auto-release-HB/job/rel-fips/job/rel-fips-4-checkinBuildToTrack/\
+    build?delay=0sec";
+    if($rel_needs_appr eq $approval_flag)
+    {
+        print "(in IF loop)\nYes, found \$rel_needs_appr eq \$approval_flag: ($rel_needs_appr eq $approval_flag)\n";
+        print "MUSTFIX_APPROVERS=> $ENV{'MUSTFIX_APPROVERS'}\n";
+        print "RELEASE_ADMINS=> $ENV{'RELEASE_ADMINS'}\n";
+        my $MAIL_SUB = "Action Required: Your approval required for Track \
+        $ENV{'FIPS_RELEASE'}:$feature";
+        print NOTIFY_EMAIL_TXT "[$datenow]\n\n\nDear Approver(s) [$ENV{MUSTFIX_APPROVERS}],\n\tThere is a new track/feature opened pending your approval for a mustfix release.\n\tDetails are below, please approve it ASAP, thanks!\n";
+        print NOTIFY_EMAIL_TXT "\n\t(Note: Jenkins job will time-out if approval not obtained within a day of receiving this notice)\n\n";
+        print NOTIFY_EMAIL_TXT "\tFIPS_RELEASE=$ENV{FIPS_RELEASE}\n\tFEATURE=$feature\n";
+        print NOTIFY_EMAIL_TXT "\n\n$txtrelnote\n\n";
+        print NOTIFY_EMAIL_TXT "#--------------------------#";
+        print NOTIFY_EMAIL_TXT "\n\n\nDear Release_admin(s) [$ENV{'RELEASE_ADMINS'}],\n\tOnce this track is approved, please use following information if/as required to \tretrigger the release job (e.g. if it  had timed out beyond the wait-time of 1 day):\n\n";
+        print NOTIFY_EMAIL_TXT "\n\tNEXTJOB=$NEXTJOB";
+        print NOTIFY_EMAIL_TXT "\n\tREL_FIPS_TRIGGER_PROJ=$ENV{REL_FIPS_TRIGGER_PROJ}\n\tREL_FIPS_TRIGGER_BUILD_NUM=$ENV{REL_FIPS_TRIGGER_BUILD_NUM}\n";
+        print NOTIFY_EMAIL_TXT "\n\n\n(A copy of this email has been sent to:\n\t$ENV{'MUSTFIX_APPROVERS'}\n\t$ENV{'RELEASE_ADMINS'}\n\t)\n\n\n";
+        print NOTIFY_EMAIL_TXT "#-----------<E-O-F>---------------#\n";
+        close(NOTIFY_EMAIL_TXT);
+        run_system_command("cat $notify_email_file | mail -s \"$MAIL_SUB\" $ENV{'MUSTFIX_APPROVERS'} $ENV{'RELEASE_ADMINS'}");
+    }
+    else
+    {
+
+        print "(in ELSE loop)\nSorry, \$rel_needs_appr ne \$approval_flag: ($rel_needs_appr ne $approval_flag)\n";
+        print NOTIFY_EMAIL_TXT "#--------------------------#\n";
+        print NOTIFY_EMAIL_TXT "[$datenow]\n\n\nNo approval required for:\nFIPS_RELEASE=$ENV{FIPS_RELEASE}\nFEATURE=$feature\n";
+        print NOTIFY_EMAIL_TXT "\n\n$txtrelnote\n\n";
+        print NOTIFY_EMAIL_TXT "\nNEXTJOB=$NEXTJOB";
+        print NOTIFY_EMAIL_TXT "\nREL_FIPS_TRIGGER_PROJ=$ENV{REL_FIPS_TRIGGER_PROJ}\nREL_FIPS_TRIGGER_BUILD_NUM=$ENV{REL_FIPS_TRIGGER_BUILD_NUM}\n\n";
+        print NOTIFY_EMAIL_TXT "\n\n(Since, no approval required, no email or other notification sent out)\n\n";
+        print NOTIFY_EMAIL_TXT "#-----------<E-O-F>---------------#\n";
+        close(NOTIFY_EMAIL_TXT);
+    }
+
+    # Add in cmvc coReqs and cmvc preReqs
+    print "#####-> execute_cmvc_reqs($feature,$level,$released,$release)\n";
+    execute_cmvc_reqs($feature,$level,$released,$release);
+}
+
+sub execute_checkinBuildToTrack
+{
+    my $level = "";
+    my $release = "";
+    my $checkInDir = "";
+    my $released = "";
+
+    GetOptions("level:s" => \$level);
+    GetOptions("release:s" => \$release);
+    GetOptions("checkInDir:s" => \$checkInDir);
+    GetOptions("released:s" => \$released);
+
+    die "Missing level" if ($level eq "");
+    die "Missing release" if ($release eq "");
+    die "Missing check in directory" if ($checkInDir eq "");
+    die "Missing released" if ($released eq "");
+
+    my $featureFile = $ENV{CMVC_FEATURE_FILE};
+    open(CMVC_FEATURE,"<",$featureFile)
+        or die("Cannot open: $featureFile: $!");
+    #Read the feature from the file:
+    my $keyval = <CMVC_FEATURE>;
+    my ($key, $feature) = split /=/, $keyval ;
+    close(CMVC_FEATURE);
+
+    if($release =~ m/fips9/)
+    {
+        $CMVC_FILES .= "src/hbfw/img/hostboot_bootloader.bin ";
+        $CMVC_FILES .= "src/hbfw/img/hostboot_securerom.bin ";
+    }
+
+    print "\n\n--------------------------------------\n";
+    print "level (passed as \"RELEASE_NAME\") => $level\n";
+    print "release (passed as \"FIPS_RELEASE\") => $release\n";
+    print "checkInDir (passed as \"CMVC_DIR\") => $checkInDir\n";
+    print "released (passed as \"PREVIOUS_RELEASE\") => $released\n";
+    print "feature (read from the \"$ENV{CMVC_FEATURE_FILE}\" file => $feature\n";
+    print "CMVC_FILES =>  $CMVC_FILES\n";
+    print "\n--------------------------------------\n\n";
+
+    #Lock Files
+    print  "#####-> run_system_command(\"File -lock $CMVC_FILES -release $release -feature $feature -verbose\")\n";
+    run_system_command("File -lock $CMVC_FILES -release $release -feature $feature -verbose");
+    #Check In Files
+    print "Present working directory:\n";
+    run_system_command("pwd");
+    print "Switching directory to \"$checkInDir\" .....\n";
+    chdir ($checkInDir);
+    run_system_command("pwd");
+    print "#####-> run_system_command(\"File -checkin $CMVC_FILES -top $checkInDir -release $release -feature $feature -verbose\")\n";
+    run_system_command("File -checkin $CMVC_FILES -top $checkInDir -release $release -feature $feature -verbose");
+
+    #Complete Fix Records
+    print "#####-> run_system_command(\"Fix -complete -feature $feature -release $release -component esw_hbfw -verbose\")\n";
+    run_system_command("Fix -complete -feature $feature -release $release -component esw_hbfw -verbose");
+
+    #Get ClearQuest ID
+    my $response = run_system_command("Feature -view $feature");
+    my ($cq_id) = $response =~ /CQID\s+(.*)/;
+    print $cq_id if $debug;
+
+    #Print out Debug information
+    print_debug_info($level, $feature, $cq_id, $released);
+}
+
 sub execute_post_release
 {
     my $feature = "";
@@ -757,6 +951,8 @@ sub create_release_notes
     my $commits = git_commit_history("HEAD", $level_info->{released});
 
     open RELNOTE, "> ".git_root()."/releaseNotes.html";
+    open TXTRELNOTE, "> ".$ENV{'RELEASENOTESTXT'};
+    print TXTRELNOTE "The list of commits included for approval are:\n";
     print RELNOTE "<html>\n";
     print RELNOTE "    <head><title>Release notes for $level</title></head>\n";
     print RELNOTE <<STYLESHEET;
@@ -826,6 +1022,7 @@ STYLESHEET
         print RELNOTE "        <td>$subject</td>\n";
         print RELNOTE "        <td>$commit</td>\n";
         print RELNOTE "    </tr>\n";
+        print TXTRELNOTE "$commit\n";
     }
     print RELNOTE "</table>\n";
 
@@ -833,8 +1030,10 @@ STYLESHEET
     print RELNOTE "</html>\n";
 
     close RELNOTE;
+    close TXTRELNOTE;
 
     system "git add ".git_root()."/releaseNotes.html";
+    #system "git add ".git_root()."/releaseNotes.txt";
     system "git commit -m \"Release notes for $level\"";
 
 }
@@ -2701,6 +2900,7 @@ sub run_system_command
 
     print "$cmd\n";
     my $output = `$cmd`;
+    #print "output=$output\n";
     die "failed running system command $cmd - $?" if ($?);
     print $output if $debug;