VPO implementation of debug framework.

Change-Id: Ic4efd38fb189efa7c42a7b3089c82c7f6d408c14
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/596
Tested-by: Jenkins Server
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

15 files changed, 1106 insertions, 661 deletions

diff --git a/src/build/debug/Hostboot/CallFunc.pm b/src/build/debug/Hostboot/CallFunc.pm
index 69cf0130b..a6e9bf417 100644
--- a/src/build/debug/Hostboot/CallFunc.pm
+++ b/src/build/debug/Hostboot/CallFunc.pm
@@ -143,17 +143,18 @@ sub main
     ::userDisplay ::read64($address + CALLFUNC_DEBUG_RETVAL_OFFSET)."\n";
 }
 
-sub help
+sub helpInfo
 {
-    ::userDisplay "Tool: CallFunc\n";
-    ::userDisplay "\tInteractively execute a function.\n";
-    ::userDisplay "\n    Options:\n";
-    ::userDisplay "\tfunction='function name' - Function to execute.\n";
-    ::userDisplay "\targuments=arg0,arg1... - List of arguments to pass.\n";
-    ::userDisplay "\tforce - Run command even if state doesn't not appear ".
-                  "correct.\n";
-    ::userDisplay "\n\tfunc can be used as a short-name for 'function'.\n";
-    ::userDisplay "\targs, parameters, or parms can be used as a short-name ";
-    ::userDisplay "for arguments.\n";
+    my %info = (
+        name => "CallFunc",
+        intro => ["Interactively execute a function."],
+        options => {
+                    "function='function name'" => ["Function to execute."],
+                    "arguments=arg0,arg1..." =>["List of arguments to pass function."],
+                    "force" => ["Run command even if state does not appear correct."],
+                   },
+        notes => ["func can be used as a short-name for 'function'.",
+                  "args, parameters, or parms can be used as a short-name for arguments."]
+    );
 }
 
diff --git a/src/build/debug/Hostboot/CxxTest.pm b/src/build/debug/Hostboot/CxxTest.pm
index e542e0794..6e0bf4e98 100644
--- a/src/build/debug/Hostboot/CxxTest.pm
+++ b/src/build/debug/Hostboot/CxxTest.pm
@@ -1,3 +1,25 @@
+#  IBM_PROLOG_BEGIN_TAG
+#  This is an automatically generated prolog.
+#
+#  $Source: src/build/debug/Hostboot/CxxTest.pm $
+#
+#  IBM CONFIDENTIAL
+#
+#  COPYRIGHT International Business Machines Corp. 2011 - 2012
+#
+#  p1
+#
+#  Object Code Only (OCO) source materials
+#  Licensed Internal Code Source Materials
+#  IBM HostBoot Licensed Internal Code
+#
+#  The source code for this program is not published or other-
+#  wise divested of its trade secrets, irrespective of what has
+#  been deposited with the U.S. Copyright Office.
+#
+#  Origin: 30
+#
+#  IBM_PROLOG_END
 use strict;
 
 package Hostboot::CxxTest;
@@ -32,9 +54,11 @@ sub main
 
 }
 
-sub help
+sub helpInfo
 {
-    ::userDisplay "Tool: CxxTest\n";
-    ::userDisplay "\tDisplays the number of CxxTest cases executed and their\n";
-    ::userDisplay "\tresults, warnings, etc.\n";
+    my %info = (
+        name => "CxxTest",
+        intro => ["Displays the number of CxxTest cases executed and their results,",
+                 "warnings, etc."],
+    );
 }
diff --git a/src/build/debug/Hostboot/Errl.pm b/src/build/debug/Hostboot/Errl.pm
index 9e8a7efd1..654b10e46 100755
--- a/src/build/debug/Hostboot/Errl.pm
+++ b/src/build/debug/Hostboot/Errl.pm
@@ -142,19 +142,17 @@ sub main
     return 0;
 }
 
-sub help
+sub helpInfo
 {
-
-    ::userDisplay "Tool: Errl\n";
-    ::userDisplay "\tList or display the error log entries.\n";
-    ::userDisplay "\n";
-    ::userDisplay "    Options:\n";
-    ::userDisplay "\tdisplay=<id> - Display a specific error log by id.\n";
-    ::userDisplay "\tdisplay=all - Display all error logs in the repository.\n";
-    ::userDisplay "\n";
-    ::userDisplay "    The default behavior is to list all the committed error\n";
-    ::userDisplay "    logs unless requested to display a specific error log\n";
-    ::userDisplay "    or all error logs.\n";
-
+    my %info = (
+        name => "Errl",
+        intro => ["List or display the error log entries."],
+        options => {
+                    "display=<id>|all" => ["<id> - Display a specific error log by id.",
+                                           "all - Display all error logs in the repository."],
+                   },
+        notes => ["The default behavior is to list all the committed error logs unless",
+                  "requested to display a specific error log or all error logs."]
+    );
 }
 
diff --git a/src/build/debug/Hostboot/Example.pm b/src/build/debug/Hostboot/Example.pm
index 25ca61365..35025f502 100755
--- a/src/build/debug/Hostboot/Example.pm
+++ b/src/build/debug/Hostboot/Example.pm
@@ -38,8 +38,10 @@ sub main
     return 0;
 }
 
-sub help
+sub helpInfo
 {
-    ::userDisplay "Tool: Example\n";
-    ::userDisplay "\tDoesn't really do anything special.\n";
+    my %info = (
+        name => "Example",
+        intro => ["Doesn't really do anything special."],
+    );
 }
diff --git a/src/build/debug/Hostboot/MemStats.pm b/src/build/debug/Hostboot/MemStats.pm
index 415cceb01..7ec650fe1 100644
--- a/src/build/debug/Hostboot/MemStats.pm
+++ b/src/build/debug/Hostboot/MemStats.pm
@@ -1,3 +1,25 @@
+#  IBM_PROLOG_BEGIN_TAG
+#  This is an automatically generated prolog.
+#
+#  $Source: src/build/debug/Hostboot/MemStats.pm $
+#
+#  IBM CONFIDENTIAL
+#
+#  COPYRIGHT International Business Machines Corp. 2011 - 2012
+#
+#  p1
+#
+#  Object Code Only (OCO) source materials
+#  Licensed Internal Code Source Materials
+#  IBM HostBoot Licensed Internal Code
+#
+#  The source code for this program is not published or other-
+#  wise divested of its trade secrets, irrespective of what has
+#  been deposited with the U.S. Copyright Office.
+#
+#  Origin: 30
+#
+#  IBM_PROLOG_END
 use strict;
 
 package Hostboot::MemStats;
@@ -68,8 +90,10 @@ sub main
 
 }
 
-sub help
+sub helpInfo
 {
-    ::userDisplay "Tool: MemStats\n";
-    ::userDisplay "\tDisplays Hostboot memory usage information\n";
+    my %info = (
+        name => "MemStats",
+        intro => ["Displays Hostboot memory usage information."],
+    );
 }
diff --git a/src/build/debug/Hostboot/Printk.pm b/src/build/debug/Hostboot/Printk.pm
index deaf6c72d..35ef5dbef 100755
--- a/src/build/debug/Hostboot/Printk.pm
+++ b/src/build/debug/Hostboot/Printk.pm
@@ -30,15 +30,19 @@ our @EXPORT_OK = ('main');
 
 sub main
 {
-    ::userDisplay "------------Kernel Printk Parser------------\n";
     my ($symAddr, $symSize) = ::findSymbolAddress("kernel_printk_buffer");
     if (not defined $symAddr) { ::userDisplay "Cannot find symbol.\n"; die; }
-    ::userDisplay ::readData($symAddr,$symSize);
-    ::userDisplay "--------------------------------------------\n";
+    my $data = ::readData($symAddr,$symSize);
+    $data =~ s/\0+//g; #strip off nulls
+    ::userDisplay "------------Kernel Printk Parser------------\n";
+    ::userDisplay $data;
+    ::userDisplay "\n--------------------------------------------\n";
 }
 
-sub help
+sub helpInfo
 {
-    ::userDisplay "Tool: Printk\n";
-    ::userDisplay "\tDisplays the printk buffer.\n";
+    my %info = (
+        name => "Printk",
+        intro => ["Displays the printk buffer."],
+    );
 }
diff --git a/src/build/debug/Hostboot/Ps.pm b/src/build/debug/Hostboot/Ps.pm
index 0048ccc87..a5a56ba02 100755
--- a/src/build/debug/Hostboot/Ps.pm
+++ b/src/build/debug/Hostboot/Ps.pm
@@ -167,8 +167,10 @@ sub makeTabs
     return $result;
 }
 
-sub help
+sub helpInfo
 {
-    ::userDisplay "Tool: Ps\n";
-    ::userDisplay "\tDisplays a tree of all tasks and their current state.\n";
+    my %info = (
+        name => "Ps",
+        intro => ["Displays a tree of all tasks and their current state."],
+    );
 }
diff --git a/src/build/debug/Hostboot/Trace.pm b/src/build/debug/Hostboot/Trace.pm
index c80c1985c..d28990494 100755
--- a/src/build/debug/Hostboot/Trace.pm
+++ b/src/build/debug/Hostboot/Trace.pm
@@ -32,6 +32,7 @@ use constant MAX_NUM_TRACE_BUFFERS => 48;
 use constant DESC_ARRAY_ENTRY_SIZE => 24;
 use constant OFFSET_TRAC_BUFFER_SIZE => 20;
 use constant OFFSET_BUFFER_ADDRESS => 16;
+use constant BUFFER_ADDRESS_SIZE => 8;
 
 use File::Temp ('tempfile');
 
@@ -58,22 +59,29 @@ sub main
     my ($fh,$fname) = tempfile();
     binmode($fh);
 
+    # read the entire g_desc_array instead of reading each entry which is much slower in VBU
+    my $result = ::readData($symAddr, $symSize);
+
+    $symAddr = 0;
     my $foundBuffer = 0;
 
     for (my $i = 0; $i < MAX_NUM_TRACE_BUFFERS; $i++)
     {
         # component name is first in g_desc_array[$i]
-        my $compName = ::readStr($symAddr);
+        my $compName = substr $result, $symAddr, OFFSET_BUFFER_ADDRESS;
+        # strip off null paddings
+        $compName = unpack('A*', $compName);
         last if ($compName eq "");
 
-        # get the pointer to its trace buffer
-        my $buffAddr = ::read64($symAddr  + OFFSET_BUFFER_ADDRESS);
-
-        # get the size of this trace buffer 
-        my $buffSize = ::read32($buffAddr + OFFSET_TRAC_BUFFER_SIZE);
-
         if ((not defined $traceBuffers) or (uc($traceBuffers) =~ m/$compName/))
         {
+            # get the pointer to its trace buffer
+            my $buffAddr = substr $result, $symAddr + OFFSET_BUFFER_ADDRESS, BUFFER_ADDRESS_SIZE;
+            $buffAddr= hex (unpack('H*',$buffAddr));
+
+            # get the size of this trace buffer
+            my $buffSize = ::read32($buffAddr + OFFSET_TRAC_BUFFER_SIZE);
+
             $foundBuffer = 1;
             print $fh (::readData($buffAddr, $buffSize ));
         }
@@ -99,15 +107,17 @@ sub main
     unlink($fname);
 }
 
-sub help
+sub helpInfo
 {
-    ::userDisplay "Tool: Trace\n";
-    ::userDisplay "\tDisplays the trace buffer.\n";
-    ::userDisplay "\n    Options:\n";
-    ::userDisplay "\tfsp-trace=<path> - Path to non-default ".
-                      "fsp-trace utility.\n";
-    ::userDisplay "\tcomponents=<list> - Comma separated list of components\n".
-                  "\t                    to display trace buffers for.\n";
-    ::userDisplay "\twith-file-names - Trace statements will include file\n".
-                  "\t                  name of place the trace was defined.\n";
+    my %info = (
+        name => "Trace",
+        intro => ["Displays the trace buffer."],
+        options => {
+                    "fsp-trace=<path>" => ["Path to non-default fsp-trace utility."],
+                    "components=<list>" =>["Comma separated list of components to display trace",
+                                           "buffers for."],
+                    "with-file-names" => ["Trace statements will include file name of place the",
+                                          "trace was defined."],
+                   },
+    );
 }
diff --git a/src/build/debug/Hostboot/_DebugFramework.pm b/src/build/debug/Hostboot/_DebugFramework.pm
index fd62127bf..071174021 100755
--- a/src/build/debug/Hostboot/_DebugFramework.pm
+++ b/src/build/debug/Hostboot/_DebugFramework.pm
@@ -42,7 +42,7 @@ use strict;
 package Hostboot::_DebugFramework;
 use Exporter 'import';
 
-our @EXPORT = ( 'callToolModule', 'callToolModuleHelp',
+our @EXPORT = ( 'callToolModule', 'callToolModuleHelp', 'callToolModuleHelpInfo',
                 'parseToolOpts', 'determineImagePath',
                 'findSymbolAddress', 'findSymbolTOCAddress',
                 'findSymbolByAddress',
@@ -93,20 +93,64 @@ sub callToolModule
 
 # @sub callToolModuleHelp
 #
-# Executes the 'help' function of the requested tool module to display
-# tool usage.
+# Display the tool usage.
 #
 # @param string - Tool to call.
 #
 sub callToolModuleHelp
 {
     my $tool = shift;
+    my %info = callToolModuleHelpInfo($tool);
+
+    ::userDisplay("\nTool: $tool\n");
+
+    for my $i ( 0 .. $#{ $info{intro} } )
+    {
+        ::userDisplay("\t$info{intro}[$i]\n");
+    }
+
+    if (defined $info{options})
+    {
+        ::userDisplay("\nOptions:\n");
+
+        for my $key ( keys %{$info{options}} )
+        {
+            ::userDisplay("\t$key\n");
+
+            for my $i (0 .. $#{ $info{options}{$key} } )
+            {
+                ::userDisplay("\t\t$info{options}{$key}[$i]\n");
+            }
+        }
+    }
+
+    if (defined $info{notes})
+    {
+        ::userDisplay("\n");
+        for my $i (0 .. $#{ $info{notes} } )
+        {
+            ::userDisplay("$info{notes}[$i]\n");
+        }
+    }
+}
+
+# @sub callToolModuleHelpInfo
+#
+# Executes the 'helpInfo' function of the requested tool module to get
+# the tool usage info.
+#
+# @param string - Tool to call.
+#
+sub callToolModuleHelpInfo
+{
+    my $tool = shift;
     my $package = "Hostboot::$tool";
 
     eval("use lib '.'; use $package; return 1;") or
             die "Couldn't load tool \"$tool\":\n\t$@";
-    eval("$package->help(\\%toolOpts);");
+    my %info = eval("$package->helpInfo(\\%toolOpts);");
     die $@ if $@;
+    return %info;
 }
 
 # @sub parseToolOpts
diff --git a/src/build/debug/vpo-debug-framework.pl b/src/build/debug/vpo-debug-framework.pl
new file mode 100755
index 000000000..ab96ac7ae
--- /dev/null
+++ b/src/build/debug/vpo-debug-framework.pl
@@ -0,0 +1,686 @@
+#!/usr/bin/perl
+#  IBM_PROLOG_BEGIN_TAG
+#  This is an automatically generated prolog.
+#
+#  $Source: src/build/debug/vpo-debug-framework.pl $
+#
+#  IBM CONFIDENTIAL
+#
+#  COPYRIGHT International Business Machines Corp. 2012
+#
+#  p1
+#
+#  Object Code Only (OCO) source materials
+#  Licensed Internal Code Source Materials
+#  IBM HostBoot Licensed Internal Code
+#
+#  The source code for this program is not published or other-
+#  wise divested of its trade secrets, irrespective of what has
+#  been deposited with the U.S. Copyright Office.
+#
+#  Origin: 30
+#
+#  IBM_PROLOG_END
+# @file vpo-debug-framework.pl
+# @brief Implementation of the common debug framework for running in vpo.
+#
+
+use strict;
+use warnings;
+
+use POSIX;
+use Getopt::Long;
+use Pod::Usage;
+use File::Temp ('tempfile');
+use File::Basename;
+
+use Hostboot::_DebugFramework;
+
+#-----------
+# Constants
+#-----------
+use constant CACHELINESIZE => 128;
+use constant CACHELINEMASK => 0xFFFFFF80;
+use constant NUMTHREADS => 8;
+
+#------------------------------------------------------------
+# Common options for the different tools in VPO environment
+#------------------------------------------------------------
+my %optionInfo = (
+    "--test" => "Use the hbicore_test.syms file instead of the default.",
+	"--img-path=<path>" => "The path to the \"img\" directory where the syms file, etc is located.",
+	"--out-path=<path>" => "The path to the directory where the output will be saved.",
+    "--debug" => "Enable debug tracing.",
+	"-k#" => "The cage to act on.",
+	"-n#" => "The node to act on.",
+	"-s#" => "The slot to act on.",
+	"-p#" => "The chip position to act on.",
+	"-c#" => "The core/chipUnit to act on.",
+);
+
+#--------------------------------------------------------------------------------
+# MAIN
+#--------------------------------------------------------------------------------
+my $name = "vpo-debug-framework.pl";
+my $self = ($0 =~ m/$name/);   #flag showing whether script invoked using a different name; i.e symlink
+my $tool = "";
+my $testImage = 0;
+my $outPath = "";
+my $outFile = "";
+my $toolOptions = "";
+my $cfgHelp = 0;
+my $cfgMan = 0;
+my $toolHelp = 0;
+my $debug = 0;
+my @ecmdOpt = ("-c3");
+my @threadState = ();
+my $l2Flushed = 0;
+my $fh;
+
+my $imgPath = "";
+my $hbDir = $ENV{'HBDIR'};
+if (defined ($hbDir))
+{
+    if ($hbDir ne "")
+    {
+        $imgPath = "$hbDir/";
+    }
+}
+
+Getopt::Long::Configure ("bundling");
+
+if ($self)
+{
+    GetOptions("tool:s" => \$tool,
+               "tool-options:s" => \$toolOptions,
+               "test" => \$testImage,
+               "img-path:s" => \$imgPath,
+               "out-path:s" => \$outPath,
+               "debug" => \$debug,
+               "help" => \$cfgHelp,
+               "toolhelp" => \$toolHelp,
+               "man" => \$cfgMan,
+               "k=i" => \&processEcmdOpts,
+               "n=i" => \&processEcmdOpts,
+               "s=i" => \&processEcmdOpts,
+               "p=i" => \&processEcmdOpts,
+               "c=i" => \&processEcmdOpts) || pod2usage(-verbose => 0);
+    pod2usage(-verbose => 1) if ($cfgHelp && $self);
+    pod2usage(-verbose => 2) if ($cfgMan && $self);
+    pod2usage(-verbose => 0) if (($tool eq "") && $self);
+
+    if ($toolHelp)
+    {
+        callToolModuleHelp($tool);
+        exit;
+    }
+}
+else
+{
+    Getopt::Long::Configure ("pass_through");
+
+    GetOptions("test" => \$testImage,
+               "img-path:s" => \$imgPath,
+               "out-path:s" => \$outPath,
+               "debug" => \$debug,
+               "help" => \$cfgHelp,
+               "man" => \$cfgMan,
+               "k=i" => \&processEcmdOpts,
+               "n=i" => \&processEcmdOpts,
+               "s=i" => \&processEcmdOpts,
+               "p=i" => \&processEcmdOpts,
+               "c=i" => \&processEcmdOpts);
+
+    #Determine the tool module.
+    determineToolModule();
+
+    if ($cfgHelp || $cfgMan)
+    {
+        displayToolModuleHelp();
+        exit;
+    }
+
+    # Determine the options for the tool module
+    determineToolModuleOpts();
+}
+
+# Determine the full image path.
+$imgPath = determineImagePath($imgPath);
+
+# Determine the output file
+$outFile = determineOutputFile();
+
+if ($outFile ne "")
+{
+    unlink $outFile if (-e $outFile);
+    open $fh, ">>$outFile" or die "ERROR: cannot open $outFile";
+    binmode $fh;
+}
+
+my $flag = "-quiet";
+if ($debug)
+{
+    $flag = "";
+}
+
+# Save original thread states
+saveThreadStates();
+
+# Parse tool options and call module.
+parseToolOpts($toolOptions);
+callToolModule($tool);
+
+# Restore thread states
+restoreThreadStates();
+
+print "\n\nData saved to $outFile\n\n";
+
+# Close the output file
+close $fh if ($outFile ne "");
+
+
+#--------------------------------------------------------------------------------
+# SUBROUTINES
+#--------------------------------------------------------------------------------
+
+sub usage
+{
+    pod2usage(-verbose => 2);
+}
+
+sub processEcmdOpts
+{
+    my ($opt, $value) = @_;
+
+    if ($opt eq "c")
+    {
+        #remove default value
+        shift(@ecmdOpt);
+    }
+
+    push(@ecmdOpt, "-$opt$value");
+}
+
+# @sub determineOutputFile
+#
+# Determine the file to save the output to.
+#
+sub determineOutputFile
+{
+    if ($outPath eq "")
+    {
+        $outPath = ".";
+    }
+
+    return "$outPath/hb-$tool.output";
+}
+
+# @sub userDisplay
+#
+# Display parameters to the user.
+#
+# @param varargs - Items to display to the user.
+#
+sub userDisplay
+{
+    foreach my $value (@_)
+    {
+        print $value;
+        print $fh $value if ($outFile ne "");
+    }
+}
+
+# Flush the L2 cache
+# This is needed in order to dump L3 quickly
+#
+sub flushL2
+{
+    if (0 == $l2Flushed)
+    {
+        #stop instructions
+        stopInstructions("all");
+
+        my $command = "/afs/awd.austin.ibm.com/projects/eclipz/lab/p8/compiled_procs/procs/p8_l2_flush_wrap.x86 ";
+        $command .= "@ecmdOpt $flag";
+        die "ERROR: cannot flush L2" if (system("$command") != 0);
+
+        $l2Flushed = 1;
+    }
+}
+
+
+# @return The blob of data requested.
+# @sub readData
+#
+# Reads a data blob from L3
+#
+# @param integer - Address to read at.
+# @param size - Size (in bytes) to read.
+#
+# @return The blob of data requested.
+#
+sub readData
+{
+    my $addr = shift;
+    my $size = shift;
+
+    #flushL2
+    flushL2();
+
+    #Compute the # of cache lines
+    my $offset = $addr % CACHELINESIZE;
+    my $numCacheLines = ceil($size / CACHELINESIZE);
+    if (($offset + $size) > ($numCacheLines * CACHELINESIZE))
+    {
+        $numCacheLines += 1;
+    }
+
+    #Read the cache lines from L3 and save to temp file
+    my (undef, $fname) = tempfile("tmpXXXXX");
+    my $command = sprintf ("p8_dump_l3 %x $numCacheLines -f $fname -b @ecmdOpt",
+                            $addr);
+
+    if ($debug)
+    {
+        print "addr $addr, size $size, offset $offset\n";
+        print "$command\n";
+    }
+    die "ERROR: cannot read L3" if (system("$command") != 0);
+
+    #Extract just the data requested from the cache lines read
+    open FILE, $fname or die "ERROR: $fname not found : $!";
+    binmode FILE;
+    my $result = "";
+    seek FILE, $offset, SEEK_SET or die "ERROR: Couldn't seek to $offset in $fname: $!\n";
+    read FILE, $result, $size;
+    close (FILE);
+    unlink($fname);
+
+    return $result;
+}
+
+# @sub writeData
+# @brief write a blob of data to L3
+sub writeData
+{
+    my $addr = shift;
+    my $size = shift;
+    my $value = shift;
+
+    #Compute the # of cache lines
+    my $base = $addr & CACHELINEMASK;
+    my $offset = $addr % CACHELINESIZE;
+    my $numCacheLines = ceil($size / CACHELINESIZE);
+    if (($offset + $size) > ($numCacheLines * CACHELINESIZE))
+    {
+        $numCacheLines += 1;
+    }
+
+    if ($debug)
+    {
+        my $value2 = unpack("H*", $value);
+        print "addr $addr, size $size, value $value2\n";
+        print "base $base, offset $offset, numCacheLines $numCacheLines\n";
+    }
+
+    #read the cachelines from L3 & save to temp file
+    my ($fh, $fname) = tempfile("tmpXXXXX");
+    binmode $fh;
+    print $fh (readData($base, $numCacheLines * CACHELINESIZE));
+    if ($debug)
+    {
+        print "data read\n";
+        system("xxd $fname");
+    }
+
+    #modify the cachelines
+    seek $fh, $offset, SEEK_SET or die "ERROR: Couldn't seek to $offset in $fname: $!\n";
+    print $fh $value;
+    close ($fh);
+    if ($debug)
+    {
+        print "data modify\n";
+        system("xxd $fname");
+    }
+
+    #write the cachelines
+    my $command = sprintf("/afs/awd/projects/eclipz/lab/p8/gsiexe/p8_load_l3 -f $fname -o 0x%x -b @ecmdOpt",
+                          $base);
+    die "ERROR: cannot write L3" if (system("$command") != 0);
+
+    unlink($fname);
+
+    if ($debug)
+    {
+        ($fh, $fname) = tempfile("tmpXXXXX");
+        binmode $fh;
+        print $fh (readData($base, $numCacheLines * CACHELINESIZE));
+        print "data written\n";
+        system("xxd $fname");
+        close ($fh);
+        unlink($fname);
+    }
+
+    return;
+}
+
+# Stop instructions
+sub stopInstructions
+{
+    my $thread = shift;
+
+    #todo Change to a hostboot dir where a copy of the tool will be kept
+    #Stopping all threads
+    my $command = "/afs/awd/projects/eclipz/lab/p8/u/karm/ekb/eclipz/chips/p8/working/procedures/utils/p8_thread_control.x86";
+    $command .= " @ecmdOpt -stop -t$thread $flag";
+
+    if ($debug)
+    {
+        print "$command\n";
+    }
+
+    die "ERROR: cannot stop instructions" if (system("$command") != 0);
+}
+
+# Start instructions
+sub startInstructions
+{
+    my $thread = shift;
+
+    #todo Change to a hostboot dir where a copy of the tool will be kept
+    #Starting all threads
+    my $command = "/afs/awd/projects/eclipz/lab/p8/u/karm/ekb/eclipz/chips/p8/working/procedures/utils/p8_thread_control.x86";
+    $command .= " @ecmdOpt -start -t$thread $flag";
+
+    if ($debug)
+    {
+        print "$command\n";
+    }
+
+    die "ERROR: cannot start instructions" if (system("$command") != 0);
+
+    #Need to flush L2 the next time we read data from L3
+    $l2Flushed = 0;
+}
+
+# Query thread state
+# @brief query whether thread state is quiesced or running
+sub queryThreadState
+{
+    my $thread = shift;
+
+    #todo Change to a hostboot dir where a copy of the tool will be kept
+    my $command = "/afs/awd/projects/eclipz/lab/p8/u/karm/ekb/eclipz/chips/p8/working/procedures/utils/p8_thread_control.x86";
+    $command .= " @ecmdOpt -query -t$thread $flag";
+    my $result = `$command`;
+
+    if ($debug)
+    {
+        print"query result:\n $result\n";
+    }
+
+    if ($result =~ m/Quiesced/)
+    {
+        return "Quiesced";
+    }
+    return "Running";
+}
+
+# Save thread states
+# @brief Save the thread states
+sub saveThreadStates
+{
+    for (my $i = 0; $i < NUMTHREADS; $i++)
+    {
+        push (@threadState, queryThreadState($i));
+    }
+}
+
+# Restore thread states
+# @brief Restore the thread states
+sub restoreThreadStates
+{
+    for (my $i = 0; $i < NUMTHREADS; $i++)
+    {
+        my $curState = queryThreadState($i);
+        if ($threadState[$i] ne $curState)
+        {
+            if ("Quiesced" eq $curState)
+            {
+                startInstructions($i);
+            }
+            else
+            {
+                stoptInstructions($i);
+            }
+        }
+    }
+}
+
+# @sub executeInstrCycles
+# @brief Tell the simulator to run for so many clock cycles
+sub executeInstrCycles
+{
+    my $flag = "-quiet";
+    if ($debug)
+    {
+        $flag = "";
+    }
+
+    #start instructions
+    startInstructions("all");
+
+    # run clock cycles
+    my $cycles = shift;
+    $cycles = $cycles * 100;   #increase cycles since VBU takes longer
+    my $command = "simclock $cycles $flag";
+    print "$command\n";
+    die "ERROR: cannot run clock cycles" if (system("$command") != 0);
+}
+
+# @sub readyForInstructions
+# @brief Check whether we can run instructions
+# @returns 0 - Not ready or 1 - Ready
+sub readyForInstructions
+{
+    # always return Ready
+    return 1;
+}
+
+# @sub getImgPath
+#
+# Return file-system path to .../img/ subdirectory containing debug files.
+#
+sub getImgPath
+{
+    return $imgPath;
+}
+
+# @sub getIsTest
+#
+# Return boolean to determine if tools should look at test debug files or
+# normal debug files.
+#
+sub getIsTest
+{
+    return $testImage;
+}
+
+#--------------------------------------------------------------------------------
+# The following routines are used when this script is invoked using a different
+# name, i.e. a symlink.
+#--------------------------------------------------------------------------------
+
+# @sub determineToolModule
+#
+# Determine the tool module.
+#
+sub determineToolModule
+{
+    if (!$self)
+    {
+        my ($fname, $dirs, $suffix) = fileparse($0, qr{\..*});
+        my @list = split('-', $fname);
+        $tool = ucfirst($list[1]);          #Make sure first letter is upper case
+    }
+    #print "tool $tool\n";
+}
+
+# @sub determineToolModuleOpts
+#
+# Determine the tool module options.
+#
+sub determineToolModuleOpts
+{
+    my $numArgs = $#ARGV + 1;
+
+    if (!$self && $numArgs)
+    {
+        foreach my $arg (@ARGV)
+        {
+            $arg =~ s/^-+//;
+            if (($arg =~ m/=/) && ($arg =~ m/ /))
+            {
+                #put quotes around it
+                my @list = split('=', $arg);
+                $list[1] = "'".$list[1]."'";
+                $arg = join("=", @list);
+            }
+
+            $toolOptions .= " $arg";
+        }
+    }
+
+    if ($debug)
+    {
+        print "toolOptions $toolOptions\n";
+    }
+}
+
+# @sub displayToolModuleHelp
+#
+# Display usage info for the specific tool plus the common tool options.
+#
+sub displayToolModuleHelp
+{
+    if (!$self)
+    {
+        my %info = callToolModuleHelpInfo($tool);
+
+        print "\nTool: $tool\n";
+
+        for my $i ( 0 .. $#{ $info{intro} } )
+        {
+            print "\t$info{intro}[$i]\n";
+        }
+
+        print "\nOptions:\n";
+        if (defined $info{options})
+        {
+            for my $key ( keys %{$info{options}} )
+            {
+                print "\t--$key\n";
+
+                for my $i (0 .. $#{ $info{options}{$key} } )
+                {
+                    print "\t\t$info{options}{$key}[$i]\n";
+                }
+            }
+        }
+
+        for my $key ( keys %optionInfo )
+        {
+            print "\t$key\n";
+            print "\t\t$optionInfo{$key}\n";
+        }
+
+        if (defined $info{notes})
+        {
+            print "\n";
+            for my $i (0 .. $#{ $info{notes} } )
+            {
+                print "$info{notes}[$i]\n";
+            }
+        }
+    }
+}
+
+__END__
+
+=head1 NAME
+
+vpo-debug-framework.pl
+
+=head1 SYNOPSIS
+
+vpo-debug-framework.pl [options] --tool=<module>
+
+=head1 OPTIONS
+
+=over 8
+
+=item B<--tool>=MODULE
+
+Identify the tool module to execute.
+
+=item B<--tool-options>="OPTIONS"
+
+List of arguments to pass to the tool as options.
+
+=item B<--toolhelp>
+
+Displays the help message for a specific debug tool.
+
+=item B<--test>
+
+Use the hbicore_test.syms file instead of the default.
+
+=item B<--img-path>=PATH
+
+The path to the "img" directory where the syms file, etc is located.
+User can also set the env variable HBDIR to the path of the "img"
+directory instead of using this option.
+
+=item B<--out-path>=PATH
+
+The path to the directory where the output will be saved.
+
+=item B<--debug>
+
+Enable debug tracing.
+
+=item B<-k>=CAGE #
+
+The cage to act on.
+
+=item B<-n>=NODE #
+
+The node to act on.
+
+=item B<-s>=SLOT #
+
+The slot to act on.
+
+=item B<-p>=CHIP #
+
+The chip position to act on.
+
+=item B<-c>=CORE #
+
+The core/chipUnit to act on.
+
+=item B<--help>
+
+Print a brief help message and exits.
+
+=item B<--man>
+
+Prints the manual page and exits.
+
+=back
+
+=head1 DESCRIPTION
+
+Executes a debug tool module.
+
+=cut
diff --git a/src/build/tools/cpfiles.pl b/src/build/tools/cpfiles.pl
index 520c6a1c1..57ea256c5 100755
--- a/src/build/tools/cpfiles.pl
+++ b/src/build/tools/cpfiles.pl
@@ -26,14 +26,14 @@
 # Purpose:  This perl script needs to be executed from the
 # git repository.  It will copy all relevant files, including
 # scripts, hbotStringFile, .list, .syms and .bin files needed for debug
-# to the user specified directory.
+# or to release Hostboot code to the user specified directory.
 #
 # Author: CamVan Nguyen 07/07/2011
 #
 
 #
 # Usage:
-# cpfiles.pl <path>
+# cpfiles.pl <path> [--test] [--release | --vpo | --simics]
 
 
 #------------------------------------------------------------------------------
@@ -55,41 +55,44 @@ sub printUsage;
 #------------------------------------------------------------------------------
 
 #List of files to copy.  Path is relative to git repository.
-my @files = ("src/build/tools/hb-parsedump.pl",
-             "src/build/simics/hb-simdebug.py",
-             "src/build/simics/post_model_hook.simics",
-             "src/build/debug/Hostboot",
-             "src/build/debug/simics-debug-framework.pl",
-             "src/build/debug/simics-debug-framework.py",
-             "img/errlparser", 
-             "img/hbotStringFile",
-             "img/hbicore.syms",
-             "img/hbicore_test.syms",
-             "img/hbicore.bin",
-             "img/hbicore_test.bin",
-             "img/hbicore.list",
-             "img/hbicore_test.list",
-             "img/hbicore_extended.bin",
-             "img/hbicore_test_extended.bin",
-             "img/hbicore.bin.modinfo",
-             "img/hbicore_test.bin.modinfo",
-             "img/pnor.toc",
-             "img/simics_SALERNO_targeting.bin",
-             "img/simics_VENICE_targeting.bin",
-             "img/vbu_targeting.bin",
-             "img/isteplist.csv",
+# r = copy to Hostboot release dir
+# s = copy to simics sandbox
+# v = copy to vpo/vbu dir
+my %files = ("src/build/tools/hb-parsedump.pl" => "rsv",
+             "src/build/simics/hb-simdebug.py" => "s",
+             "src/build/simics/post_model_hook.simics" => "s",
+             "src/build/debug/Hostboot" => "rsv",
+             "src/build/debug/simics-debug-framework.pl" => "s",
+             "src/build/debug/simics-debug-framework.py" => "s",
+             "src/build/debug/vpo-debug-framework.pl" => "rv",
+             "src/build/debug/hb-dump-debug" => "rsv",
+             "src/build/vpo/hb-dump" => "rv",
+             "src/build/vpo/hb-istep" => "rv",
+             "src/build/vpo/hb-virtdebug.pl" => "rv",
+             "src/build/vpo/VBU_Cacheline.pm" => "rv",
+             "src/build/hwpf/prcd_compile.tcl" => "r",
+             "img/errlparser" => "rsv",
+             "img/hbotStringFile" => "rsv",
+             "img/hbicore.syms" => "rsv",
+             "img/hbicore_test.syms" => "rsv",
+             "img/hbicore.bin" => "rsv",
+             "img/hbicore_test.bin" => "rsv",
+             "img/hbicore.list" => "rsv",
+             "img/hbicore_test.list" => "rsv",
+             "img/hbicore_extended.bin" => "rsv",
+             "img/hbicore_test_extended.bin" => "rsv",
+             "img/hbicore.bin.modinfo" => "rsv",
+             "img/hbicore_test.bin.modinfo" => "rsv",
+             "img/pnor.toc" => "rsv",
+             "img/simics_SALERNO_targeting.bin" => "rs",
+             "img/simics_VENICE_targeting.bin" => "rs",
+             "img/vbu_targeting.bin" => "rv",
+             "img/isteplist.csv" => "rsv",
+             "src/usr/hwpf/hwp/fapiTestHwp.C" => "r",
+             "src/include/usr/hwpf/hwp/fapiTestHwp.H" => "r",
+             "src/usr/hwpf/hwp/initfiles/sample.initfile" => "r",
              );
              
-# copy vpo files into working dir for AWAN
-my @vpofiles = ("src/build/vpo/hb-dump",
-                "src/build/vpo/hb-errl",
-                "src/build/vpo/hb-istep",
-                "src/build/vpo/hb-printk",
-                "src/build/vpo/hb-trace", 
-                "src/build/vpo/hb-virtdebug.pl",
-                "src/build/vpo/VBU_Cacheline.pm",   
-             );
-
 #Directories in base git repository
 my @gitRepoDirs = ("img",
                     "obj",
@@ -108,7 +111,7 @@ my $numArgs = $#ARGV + 1;
 
 my $test = 0;   #Flag to overwrite hbicore.<syms|bin|list> with the test versions
 my $inDir = ""; #User specified directory to copy files to
-my $vpo = 0;    # copy extra vpo files to inDir
+my $env = "s";  #Flag to indicate which environment; simics by default
 
 if ($numArgs > 3)
 {
@@ -132,10 +135,20 @@ else
             #Set flag to copy hbicore_test.<syms|bin> to hbcore_test.<syms|bin>
             $test = 1;
         }
+        elsif ($_ eq "--release")
+        {
+            #Set flag to indicate environment
+            $env = "r";
+        }
         elsif ($_ eq "--vpo")
         {
-            #Set flag to copy list of vpo files to $inDir
-            $vpo = 1;
+            #Set flag to indicate environment
+            $env = "v";
+        }
+        elsif ($_ eq "--simics")
+        {
+            #Set flag to indicate environment
+            $env = "s";
         }
         else
         {
@@ -203,14 +216,6 @@ else
 }
 
 #------------------------------------------------------------------------------
-# If vpo flag is set, add the vpo files to the @files array
-#------------------------------------------------------------------------------
-if ( $vpo )
-{
-    push( @files, @vpofiles );
-}
-
-#------------------------------------------------------------------------------
 # Get the base dir of the git repository
 #------------------------------------------------------------------------------
 my $cwd = getcwd();
@@ -239,16 +244,18 @@ my $copyDir = "";
 chdir $gitRepo;
 #print "cwd: ", getcwd()."\n";
 
-foreach (@files)
+# There is no guarantee of the order of the elements of a hash (%files), so
+# delete the files first so we don't accidentally delete them after copying when
+# the --test option is used.
+while ( my ($key, $value) = each(%files) )
 {
-    $command = "";
+    #Skip file if not correct env
+    next if (!($value =~ m/$env/));
 
-    my($filename, $directories, $suffix) = fileparse($_, qr{\..*});
-    #print "$filename, $directories, $suffix\n";
-
-    my $recursive = (-d $_) ? "-r" : "";
+    my($filename, $directories, $suffix) = fileparse($key, qr{\..*});
+    my $recursive = (-d $key) ? "-r" : "";
 
-    #Copy .bin to the img dir
+    #Is file in img dir?
     if (($suffix eq ".bin") ||
 	($suffix eq ".toc"))
     {
@@ -262,37 +269,57 @@ foreach (@files)
     #Delete the old file first (handles copying over symlinks)
     $command = sprintf("rm -f %s %s/%s%s", $recursive, 
                                            $copyDir, $filename, $suffix);
-    if ($command ne "")
-    {
-        print "$command\n";
-        `$command`;
-    }
+    print "$command\n";
+    die if (system("$command") != 0);
+}
+
+# Now copy files
+while ( my ($key, $value) = each(%files) )
+{
+    #Skip file if not correct env
+    next if (!($value =~ m/$env/));
+
     $command = "";
 
+    my($filename, $directories, $suffix) = fileparse($key, qr{\..*});
+    #print "$filename, $directories, $suffix\n";
+
+    my $recursive = (-d $key) ? "-r" : "";
+
+    #Copy .bin to the img dir
+    if (($suffix eq ".bin") ||
+	($suffix eq ".toc"))
+    {
+        $copyDir = $imgDir;
+    }
+    else
+    {
+        $copyDir = $simicsDir;
+    }
 
     #Check if user wants to copy test versions to hbicore.<syms|bin|list>
-    if ($test == 1)
+    if (($test == 1) && ("r" ne $env))
     {
         #Copy test versions to hbicore.<syms|bin|list>
         if ($filename eq "hbicore_test")
         {
             $command = sprintf("cp %s %s %s", $recursive,
-                                              $_, $copyDir."/hbicore".$suffix);
+                                              $key, $copyDir."/hbicore".$suffix);
         }
         elsif ($filename eq "hbicore_test_extended")
         {
             $command = sprintf("cp %s %s %s", $recursive,
-                                              $_, $copyDir."/hbicore_extended".$suffix);
+                                              $key, $copyDir."/hbicore_extended".$suffix);
         }
         elsif ($filename ne "hbicore" and $filename ne "hbicore_extended")
         {
             $command = sprintf("cp %s %s %s", $recursive,
-                                              $_, $copyDir);
+                                              $key, $copyDir);
         }
     }
     else
     {
-        $command = sprintf("cp %s %s %s", $recursive, $_, $copyDir);
+        $command = sprintf("cp %s %s %s", $recursive, $key, $copyDir);
     }
 
     # Copy the file
@@ -300,27 +327,56 @@ foreach (@files)
     {
         print "$command\n";
         `$command`;
-	if( $? != 0 )
-	{
-	    print "ERROR : exiting\n";
-	    exit(-1);
-	}
+        if( $? != 0 )
+        {
+            print "ERROR : exiting\n";
+            exit(-1);
+        }
     }
-
 }
 
-# create a sym-link to the appropriate targeting binary
-print "Linking in simics_".$machine.".targeting.bin\n";
-$command = sprintf("ln -sf %s/simics_%s_targeting.bin %s/targeting.bin", $imgDir, $machine, $imgDir );
-print "$command\n";
-`$command`;
-if( $? != 0 )
+if ("s" eq $env) #simics
 {
-    print "ERROR : exiting\n";
-    exit(-1);
+    # create a sym-link to the appropriate targeting binary
+    print "Linking in simics_".$machine.".targeting.bin\n";
+    $command = sprintf("ln -sf %s/simics_%s_targeting.bin %s/targeting.bin", $imgDir, $machine, $imgDir );
+    print "$command\n";
+    `$command`;
+    if( $? != 0 )
+    {
+        print "ERROR : exiting\n";
+        exit(-1);
+    }
 }
+else #release or vpo
+{
+    # create sym-links for vpo-debug-framework.pl for each perl module
+    # in the debug framework
+    my $debugFrameworkDir = "src/build/debug/Hostboot";
+    opendir (my $dh, $debugFrameworkDir) or
+        die "Can't open dir '$debugFrameworkDir': $!";
+    my @files = readdir $dh;
+    closedir $dh;
+
+    #need to create symlinks for old tool names as well
+    push(@files, "errl");
+    push(@files, "printk");
+    push(@files, "trace");
+
+    foreach (@files)
+    {
+        # filter out any file prefixed with '_' or '.'
+        next if ($_ =~ m/^[._]/);
 
+        my($filename, $directories, $suffix) = fileparse($_, qr{\..*});
 
+        # create sym-link
+        $filename = "hb-$filename";
+        $command = "ln -sf $inDir/vpo-debug-framework.pl $inDir/$filename";
+        print "$command\n";
+        die if (system("$command") != 0);
+    }
+}
 
 chdir $cwd;
 #print "cwd: ", getcwd()."\n";
@@ -335,17 +391,20 @@ chdir $cwd;
 #------------------------------------------------------------------------------
 sub printUsage()
 {
-    print ("\nUsage: cpfiles.pl [--help] | [<path>] [--test]\n\n");
+    print ("\nUsage: cpfiles.pl [--help] | [<path>] [--test]\n");
+    print ("                   [--release| --vpo| --simics]\n\n");
     print ("  This program needs to be executed from the git repository.\n");
     print ("  It will copy all relevant files, scripts, hbotStringFile,\n");
-    print ("  .list, .syms and .bin files needed for debug to one of two\n");
-    print ("  locations:\n");
+    print ("  .list, .syms and .bin files needed for debug or release of\n");
+    print ("  Hostboot code to one of two locations:\n");
     print ("      1.  <path> if one is specified by the user\n");
     print ("      2.  if <path> is not specified, then the files will be\n");
     print ('          copied to the path specified by env variable $SANDBOXBASE'."\n");
     print ("          if it is defined.\n\n");
     print ("  --help: prints usage information\n");
     print ("  --test: Copy hbicore_test.<syms|bin|list> to hbicore.<syms|bin|list>\n");
-    print ("  --vpo:  Copy files in src/build/vpo to support vpo operation\n");
+    print ("  --release: Copy files needed to release Hostboot code\n");
+    print ("  --vpo: Copy files needed to debug in vpo\n");
+    print ("  --simics: <default> Copy files needed to debug in simics\n");
 }
 
diff --git a/src/build/vpo/hb-errl b/src/build/vpo/hb-errl
deleted file mode 100755
index 64f7cc9f4..000000000
--- a/src/build/vpo/hb-errl
+++ /dev/null
@@ -1,196 +0,0 @@
-#!/usr/bin/perl
-#  IBM_PROLOG_BEGIN_TAG
-#  This is an automatically generated prolog.
-#
-#  $Source: src/build/vpo/hb-errl $
-#
-#  IBM CONFIDENTIAL
-#
-#  COPYRIGHT International Business Machines Corp. 2011
-#
-#  p1
-#
-#  Object Code Only (OCO) source materials
-#  Licensed Internal Code Source Materials
-#  IBM HostBoot Licensed Internal Code
-#
-#  The source code for this program is not published or other-
-#  wise divested of its trade secrets, irrespective of what has
-#  been deposited with the U.S. Copyright Office.
-#
-#  Origin: 30
-#
-#  IBM_PROLOG_END
-
-#
-# Purpose:  This perl script works on VBU and will dump error log buffer
-#
-# Author: CamVan Nguyen
-# Last Updated: 09/07/2011
-#
-
-#------------------------------------------------------------------------------
-# Specify perl modules to use
-#------------------------------------------------------------------------------
-use strict;
-use warnings;
-use POSIX;
-
-
-#------------------------------------------------------------------------------
-# Forward Declaration
-#------------------------------------------------------------------------------
-sub printUsage;
-
-
-#==============================================================================
-# MAIN
-#==============================================================================
-
-#------------------------------------------------------------------------------
-# Parse optional input arguments
-#------------------------------------------------------------------------------
-my $numArgs = $#ARGV + 1;
-#print "num args = $numArgs\n";
-#print "argument list = @ARGV\n";
-
-my $hbDir = $ENV{'HBDIR'};
-if (defined ($hbDir))
-{
-    unless ($hbDir ne "")
-    {
-        $hbDir = '.';             #Set to current directory
-    }
-}
-else
-{
-    $hbDir = '.';                 #Set to current directory
-}
-
-my $dumpErrlList = 0;
-my $dumpErrlDtl = 0;
-my @errlOpt;
-for (my $i=0; $i<$numArgs; $i++)
-{
-    if (($ARGV[$i] eq "--help") || ($ARGV[$i] eq "-h"))
-    {
-        #Print command line help
-        printUsage();
-        exit (0);
-    }
-    elsif ($ARGV[$i] eq "--in")
-    {
-        if (($i + 1) >= $numArgs)
-        {
-            die "No value given for --in parameter.\n";
-        }
-        $i++;
-        $hbDir = $ARGV[$i];
-    }
-    elsif ($ARGV[$i] eq "--out")
-    {
-        if (($i + 1) >= $numArgs)
-        {
-            die "No value given for --out parameter.\n";
-        }
-        $i++;
-    }
-    elsif (($ARGV[$i] eq "-d") && ($dumpErrlList == 0))
-    {
-        $dumpErrlDtl = 1;
-
-        #save the error log option & remove it from @ARGV
-        push(@errlOpt, $ARGV[$i]);
-        splice(@ARGV, $i, 1);
-        $numArgs--;
-        $i--;
-
-        last if (($i + 1) >= $numArgs);
-        $i++;
-
-        if (substr($ARGV[$i], 0, 1) eq '-')
-        {
-            $i--;
-        }
-        else
-        {
-            if (($ARGV[$i] =~ /all/i) || isdigit($ARGV[$i]))
-            {
-                #save the error log option & remove it from @ARGV
-                push(@errlOpt, $ARGV[$i]);
-                splice(@ARGV, $i, 1);
-                $numArgs--;
-                $i--;
-            }
-            else
-            {
-                die "ERROR:  Enter logid or 'all'"
-            }
-        }
-    }
-    elsif (($ARGV[$i] eq "-l") && ($dumpErrlDtl == 0))
-    {
-        $dumpErrlList = 1;
-
-        #save the error log option & remove it from @ARGV
-        push(@errlOpt, $ARGV[$i]);
-        splice(@ARGV, $i, 1);
-        $numArgs--;
-        $i--;
-    }
-    elsif (($ARGV[$i] ne "--test") && ($ARGV[$i] !~ m/^-[cknsp]\d+/))
-    {
-        print "Invalid argument entered:  $ARGV[$i]\n";
-        printUsage();
-        exit(1);
-    }
-}
-
-
-#------------------------------------------------------------------------------
-# Dump the error log(s)
-#------------------------------------------------------------------------------
-my $command = "$hbDir/hb-virtdebug.pl --errl @errlOpt @ARGV";
-system($command);
-
-
-#==============================================================================
-# SUBROUTINES
-#==============================================================================
-
-#------------------------------------------------------------------------------
-# Print command line help
-#------------------------------------------------------------------------------
-sub printUsage()
-{
-    print ("\nUsage: hb-errl [--help] | [-l | -d [<logid>|all]]\n");
-    print ("               [--in <path to .syms file, hb-virtdebug.pl & errlparser>]\n");
-    print ("               [--out <path to save output data>]\n");
-    print ("               [--test] [-k#] [-n#] [-s#] [-p#] [-c#]\n\n");
-    print ("  This program retrieves the error log(s) from L3.\n");
-    print ("  User should copy the relevant .syms file, hb-virtdebug.pl & errlparser\n");
-    print ("  to the current directory or set the env variable HBDIR to the path\n");
-    print ("  of the files.\n\n");
-    print ("  --help            Prints usage information.\n");
-    print ("  --in              Overrides the automatically detected .syms file,\n");
-    print ("                    hb-virtdebug.pl & errlparser in HBDIR or the current directory.\n");
-    print ("                    This program will search for the files in the ");
-    print ("following order:\n");
-    print ("                        1.  from the path specified by the user\n");
-    print ("                        2.  from HBDIR if it is defined\n");
-    print ("                        3.  from the current directory\n");
-    print ("  --out             Directory where the output data will be saved.\n");
-    print ("                    Default path is the current directory.\n");
-    print ("  --test            Use the hbicore_test.syms file vs the hbicore.syms file\n");
-    print ("  -l                Dumps a listing of all the error logs\n");
-    print ("  -d <logid>        Dumps detailed data of the specified error log.\n");
-    print ("  -d [all]          Dumps detailed data of all error logs\n");
-    print ("  -k#               Specify which cage to act on (default = 0).\n");
-    print ("  -n#               Specify which node to act on (default = 0).\n");
-    print ("  -s#               Specify which slot to act on (default = 0).\n");
-    print ("  -p#               Specify which chip position to act on (default = 0).\n");
-    print ("  -c#               Specify which core/chipUnit to act on (default = 3).\n");
-    print ("\n  NOTE:  This program will not work if user has not stopped instructions\n");
-    print ("         prior to running this program.\n");
-}
-
diff --git a/src/build/vpo/hb-printk b/src/build/vpo/hb-printk
deleted file mode 100755
index 95ad98e26..000000000
--- a/src/build/vpo/hb-printk
+++ /dev/null
@@ -1,144 +0,0 @@
-#!/usr/bin/perl
-#  IBM_PROLOG_BEGIN_TAG
-#  This is an automatically generated prolog.
-#
-#  $Source: src/build/vpo/hb-printk $
-#
-#  IBM CONFIDENTIAL
-#
-#  COPYRIGHT International Business Machines Corp. 2011
-#
-#  p1
-#
-#  Object Code Only (OCO) source materials
-#  Licensed Internal Code Source Materials
-#  IBM HostBoot Licensed Internal Code
-#
-#  The source code for this program is not published or other-
-#  wise divested of its trade secrets, irrespective of what has
-#  been deposited with the U.S. Copyright Office.
-#
-#  Origin: 30
-#
-#  IBM_PROLOG_END
-
-#
-# Purpose:  This perl script works on VBU and will dump kernel printk buffer
-#
-# Author: CamVan Nguyen
-# Last Updated: 09/07/2011
-#
-
-#------------------------------------------------------------------------------
-# Specify perl modules to use
-#------------------------------------------------------------------------------
-use strict;
-use warnings;
-
-
-#------------------------------------------------------------------------------
-# Forward Declaration
-#------------------------------------------------------------------------------
-sub printUsage;
-
-
-#==============================================================================
-# MAIN
-#==============================================================================
-
-#------------------------------------------------------------------------------
-# Parse optional input arguments
-#------------------------------------------------------------------------------
-my $numArgs = $#ARGV + 1;
-#print "num args = $numArgs\n";
-#print "argument list = @ARGV\n";
-
-my $hbDir = $ENV{'HBDIR'};
-if (defined ($hbDir))
-{
-    unless ($hbDir ne "")
-    {
-        $hbDir = '.';             #Set to current directory
-    }
-}
-else
-{
-    $hbDir = '.';                 #Set to current directory
-}
-
-for (my $i=0; $i<$numArgs; $i++)
-{
-    if (($ARGV[$i] eq "--help") || ($ARGV[$i] eq "-h"))
-    {
-        #Print command line help
-        printUsage();
-        exit (0);
-    }
-    elsif ($ARGV[$i] eq "--in")
-    {
-        if (($i + 1) >= $numArgs)
-        {
-            die "No value given for --in parameter.\n";
-        }
-        $i++;
-        $hbDir = $ARGV[$i];
-    }
-    elsif ($ARGV[$i] eq "--out")
-    {
-        if (($i + 1) >= $numArgs)
-        {
-            die "No value given for --out parameter.\n";
-        }
-        $i++;
-    }
-    elsif (($ARGV[$i] ne "--test") && ($ARGV[$i] !~ m/^-[cknsp]\d+/))
-    {
-        print "Invalid argument entered:  $ARGV[$i]\n";
-        printUsage();
-        exit(1);
-    }
-}
-
-
-#------------------------------------------------------------------------------
-# Dump the kernel printk buffer
-#------------------------------------------------------------------------------
-my $command = "$hbDir/hb-virtdebug.pl --printk @ARGV";
-system($command);
-
-
-#==============================================================================
-# SUBROUTINES
-#==============================================================================
-
-#------------------------------------------------------------------------------
-# Print command line help
-#------------------------------------------------------------------------------
-sub printUsage()
-{
-    print ("\nUsage: hb-printk [--help] | [--in <path to .syms file & hb-virtdebug.pl>]\n");
-    print ("                 [--out <path to save output data>]\n");
-    print ("                 [--test] [-k#] [-n#] [-s#] [-p#] [-c#]\n\n");
-    print ("  This program retrieves the kernel printk buffer from L3.\n");
-    print ("  User should copy the relevant .syms file & hb-virtdebug.pl to the\n");
-    print ("  current directory or set the env variable HBDIR to the path of the files.\n\n");
-    print ("  --help            Prints usage information.\n");
-    print ("  --in              Overrides the automatically detected .syms file &\n");
-    print ("                    hb-virtebug.pl in HBDIR or the current directory.\n");
-    print ("                    This program will search for the files in the ");
-    print ("following order:\n");
-    print ("                        1.  from the path specified by the user\n");
-    print ("                        2.  from HBDIR if it is defined\n");
-    print ("                        3.  from the current directory\n");
-    print ("  --out             Directory where the output data will be saved.\n");
-    print ("                    Default path is the current directory.\n");
-    print ("  --test            Use the hbicore_test.syms file vs the hbicore.syms file\n");
-    print ("  -k#               Specify which cage to act on (default = 0).\n");
-    print ("  -n#               Specify which node to act on (default = 0).\n");
-    print ("  -s#               Specify which slot to act on (default = 0).\n");
-    print ("  -p#               Specify which chip position to act on (default = 0).\n");
-    print ("  -c#               Specify which core/chipUnit to act on (default = 3).\n");
-    print ("\n  NOTE:  This program will not work if user has not stopped instructions\n");
-    print ("         prior to running this program.\n");
-}
-
diff --git a/src/build/vpo/hb-trace b/src/build/vpo/hb-trace
deleted file mode 100755
index 5b3556c80..000000000
--- a/src/build/vpo/hb-trace
+++ /dev/null
@@ -1,158 +0,0 @@
-#!/usr/bin/perl
-#  IBM_PROLOG_BEGIN_TAG
-#  This is an automatically generated prolog.
-#
-#  $Source: src/build/vpo/hb-trace $
-#
-#  IBM CONFIDENTIAL
-#
-#  COPYRIGHT International Business Machines Corp. 2011
-#
-#  p1
-#
-#  Object Code Only (OCO) source materials
-#  Licensed Internal Code Source Materials
-#  IBM HostBoot Licensed Internal Code
-#
-#  The source code for this program is not published or other-
-#  wise divested of its trade secrets, irrespective of what has
-#  been deposited with the U.S. Copyright Office.
-#
-#  Origin: 30
-#
-#  IBM_PROLOG_END
-
-#
-# Purpose:  This perl script works on VBU and will dump the component trace
-# buffers.
-#
-# Author: CamVan Nguyen
-# Last Updated: 09/07/2011
-#
-
-#------------------------------------------------------------------------------
-# Specify perl modules to use
-#------------------------------------------------------------------------------
-use strict;
-use warnings;
-
-
-#------------------------------------------------------------------------------
-# Forward Declaration
-#------------------------------------------------------------------------------
-sub printUsage;
-
-
-#==============================================================================
-# MAIN
-#==============================================================================
-
-#------------------------------------------------------------------------------
-# Parse optional input arguments
-#------------------------------------------------------------------------------
-my $numArgs = $#ARGV + 1;
-#print "num args = $numArgs\n";
-#print "argument list = @ARGV\n";
-
-my $hbDir = $ENV{'HBDIR'};
-if (defined ($hbDir))
-{
-    unless ($hbDir ne "")
-    {
-        $hbDir = '.';             #Set to current directory
-    }
-}
-else
-{
-    $hbDir = '.';                 #Set to current directory
-}
-
-# Skip the component names
-my $i = 0;
-for ($i=0; $i<$numArgs; $i++)
-{
-    last if (substr($ARGV[$i], 0, 1) eq '-')
-}
-
-# Parse the other arguments
-for ( ;$i<$numArgs; $i++)
-{
-    if (($ARGV[$i] eq "--help") || ($ARGV[$i] eq "-h"))
-    {
-        #Print command line help
-        printUsage();
-        exit (0);
-    }
-    elsif ($ARGV[$i] eq "--in")
-    {
-        if (($i + 1) >= $numArgs)
-        {
-            die "No value given for --in parameter.\n";
-        }
-        $i++;
-        $hbDir = $ARGV[$i];
-    }
-    elsif ($ARGV[$i] eq "--out")
-    {
-        if (($i + 1) >= $numArgs)
-        {
-            die "No value given for --out parameter.\n";
-        }
-        $i++;
-    }
-    elsif (($ARGV[$i] ne "--test") && ($ARGV[$i] !~ m/^-[cknsp]\d+/))
-    {
-        print "Invalid argument entered:  $ARGV[$i]\n";
-        printUsage();
-        exit(1);
-    }
-}
-
-
-#------------------------------------------------------------------------------
-# Dump the component trace buffer(s)
-#------------------------------------------------------------------------------
-my $command = "$hbDir/hb-virtdebug.pl --trace @ARGV";
-system($command);
-
-
-#==============================================================================
-# SUBROUTINES
-#==============================================================================
-
-#------------------------------------------------------------------------------
-# Print command line help
-#------------------------------------------------------------------------------
-sub printUsage()
-{
-    print ("\nUsage: hb-trace [--help] | [<compName1> <compName2> ...]\n");
-    print ("                [--in <path to .syms file, hbotStringFile & hb-virtdebug.pl>]\n");
-    print ("                [--out <path to save output data>]\n");
-    print ("                [--test] [-k#] [-n#] [-s#] [-p#] [-c#]\n\n");
-    print ("  This program retrieves the component trace buffer(s) from L3.\n");
-    print ("  User should copy the relevant .syms file, hbotStringFile & hb-virtdebug.pl\n");
-    print ("  to the current directory or set the env variable HBDIR to the path of ");
-    print ("the files.\n\n");
-    print ("  User should also set the env variable PATH to include the path to the ");
-    print ("fsp-trace program.\n\n");
-    print ("  --help            Prints usage information.\n");
-    print ("  --in              Overrides the automatically detected .syms file,\n");
-    print ("                    hbotStringFile & hb-virtdebug.pl in HBDIR or the\n");
-    print ("                    current directory.\n");
-    print ("                    This program will search for the files in the ");
-    print ("following order:\n");
-    print ("                        1.  from the path specified by the user\n");
-    print ("                        2.  from HBDIR if it is defined\n");
-    print ("                        3.  from the current directory\n");
-    print ("  --out             Directory where the output data will be saved.\n");
-    print ("                    Default path is the current directory.\n");
-    print ("  --test            Use the hbicore_test.syms file vs the hbicore.syms file\n");
-    print ("  -k#               Specify which cage to act on (default = 0).\n");
-    print ("  -n#               Specify which node to act on (default = 0).\n");
-    print ("  -s#               Specify which slot to act on (default = 0).\n");
-    print ("  -p#               Specify which chip position to act on (default = 0).\n");
-    print ("  -c#               Specify which core/chipUnit to act on (default = 3).\n");
-    print ("\n  NOTE:  This program will not work if user has not stopped instructions\n");
-    print ("         prior to running this program.\n");
-}
-
diff --git a/src/build/vpo/hb-virtdebug.pl b/src/build/vpo/hb-virtdebug.pl
index 19dc38d3f..f468c199c 100755
--- a/src/build/vpo/hb-virtdebug.pl
+++ b/src/build/vpo/hb-virtdebug.pl
@@ -49,6 +49,7 @@ use constant TRAC_DEFAULT_BUFFER_SIZE => 0x0800;
 use constant CACHE_LINE_SIZE => 128;
 use constant TRAC_BUFFER_SIZE_OFFSET => 20;
 use constant TRAC_BUFFER_SIZE_SIZE => 4;
+use constant NUMTHREADS => 8;
 
 
 #------------------------------------------------------------------------------
@@ -89,6 +90,7 @@ my @symsLines;                    #Array to store the .syms file data
 my $outDir = getcwd();            #Default = current working directory
 my @ecmdOpt;                      #Array of ecmd options
 my $core = "3";                   #Default is core 3
+my @threadState = ();             #Array to store the thread states
 
 my $hbDir = $ENV{'HBDIR'};
 if (defined ($hbDir))
@@ -267,11 +269,10 @@ if (!$dumpAll)
 }
 
 #------------------------------------------------------------------------------
-# Output reminder to stop instructions
+# Save the original thread states and stop instructions
 #------------------------------------------------------------------------------
-print "\n++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n";
-print "\nREMINDER:  User need to stop instructions prior to running this program.\n";
-print "\n++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n";
+saveThreadStates();
+stopInstructions("all");
 
 #------------------------------------------------------------------------------
 #Flush L2 - this step is needed in order to dump L3 quickly
@@ -279,8 +280,8 @@ print "\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 my $command = "";
 $command = "/afs/awd.austin.ibm.com/projects/eclipz/lab/p8/compiled_procs/procs/p8_l2_flush_wrap.x86 ";
 $command .= "@ecmdOpt -quiet";
-print "$command\n";
-die if (system("$command") != 0);
+#print "$command\n";
+die "ERROR: cannot flush L2" if (system("$command") != 0);
 
 
 #------------------------------------------------------------------------------
@@ -321,16 +322,16 @@ if ($dumpPrintk)
 
         if (-s $string)
         {
-	    #Extract and save just the kernel printk buffer
-	    $buffer = readStringBinFile($string, $offset);
+            #Extract and save just the kernel printk buffer
+            $buffer = readStringBinFile($string, $offset);
 
-	    writeBinFile($string, $buffer);
+            writeBinFile($string, $buffer);
 
-	    #Output to screen
-	    print "\nKernel printk buffer:";
-	    print "\n=====================\n\n$buffer\n";
-	    print "\n=====================\n\n";
-	    print "Data saved to file $string\n\n";
+            #Output to screen
+            print "\nKernel printk buffer:";
+            print "\n=====================\n\n$buffer\n";
+            print "\n=====================\n\n";
+            print "Data saved to file $string\n\n";
         }
         else
         {
@@ -551,8 +552,8 @@ if ($dumpAll)
     #Dump L3 to file
     my $hbDumpFile = "$outDir/hbdump.$timeStamp";
     $command = "p8_dump_l3 0 65536 -f $hbDumpFile -b @ecmdOpt";
-    print "$command\n";
-    die if (system("$command") != 0);
+    #print "$command\n";
+    die "ERROR: cannot dump L3" if (system("$command") != 0);
 
     #Check if hbDumpFile exists and is not empty
     if (-s "$hbDumpFile")
@@ -567,6 +568,10 @@ if ($dumpAll)
     }
 }
 
+#------------------------------------------------------------------------------
+# Restore the original thread states
+#------------------------------------------------------------------------------
+restoreThreadStates();
 
 
 
@@ -575,6 +580,92 @@ if ($dumpAll)
 #==============================================================================
 
 #------------------------------------------------------------------------------
+# Stop instructions
+#------------------------------------------------------------------------------
+sub stopInstructions
+{
+    my $thread = shift;
+
+    #todo Change to a hostboot dir where a copy of the tool will be kept
+    #Stopping all threads
+    my $command = "/afs/awd/projects/eclipz/lab/p8/u/karm/ekb/eclipz/chips/p8/working/procedures/utils/p8_thread_control.x86";
+    $command .= " @ecmdOpt -stop -t$thread -quiet";
+    die "ERROR: cannot stop instructions" if (system("$command") != 0);
+}
+
+#------------------------------------------------------------------------------
+# Start instructions
+#------------------------------------------------------------------------------
+sub startInstructions
+{
+    my $thread = shift;
+
+    #todo Change to a hostboot dir where a copy of the tool will be kept
+    #Starting all threads
+    my $command = "/afs/awd/projects/eclipz/lab/p8/u/karm/ekb/eclipz/chips/p8/working/procedures/utils/p8_thread_control.x86";
+    $command .= " @ecmdOpt -start -t$thread -quiet";
+    die "ERROR: cannot start instructions" if (system("$command") != 0);
+}
+
+#------------------------------------------------------------------------------
+# Query thread state
+# @brief query whether thread state is quiesced or running
+#------------------------------------------------------------------------------
+sub queryThreadState
+{
+    my $thread = shift;
+    #print "thread $thread\n";
+
+    #todo Change to a hostboot dir where a copy of the tool will be kept
+    my $command = "/afs/awd/projects/eclipz/lab/p8/u/karm/ekb/eclipz/chips/p8/working/procedures/utils/p8_thread_control.x86";
+    $command .= " @ecmdOpt -query -t$thread --quiet";
+    my $result = `$command`;
+    #print "result:\n $result";
+    if ($result =~ m/Quiesced/)
+    {
+	    #print "Thread $thread is quiesced\n";
+        return "Quiesced";
+    }
+    #print "Thread $thread is running\n";
+    return "Running";
+}
+
+#------------------------------------------------------------------------------
+# Save thread states
+# @brief Save the thread states
+#------------------------------------------------------------------------------
+sub saveThreadStates
+{
+    for (my $i = 0; $i < NUMTHREADS; $i++)
+    {
+        push (@threadState, queryThreadState($i));
+    }
+}
+
+#------------------------------------------------------------------------------
+# Restore thread states
+# @brief Restore the thread states
+#------------------------------------------------------------------------------
+sub restoreThreadStates
+{
+    for (my $i = 0; $i < NUMTHREADS; $i++)
+    {
+        my $curState = queryThreadState($i);
+        if ($threadState[$i] ne $curState)
+        {
+            if ("Quiesced" eq $curState)
+            {
+                startInstructions($i);
+            }
+            else
+            {
+                stopInstructions($i);
+            }
+        }
+    }
+}
+
+#------------------------------------------------------------------------------
 # Parse the .syms data to find the relevant address and size for the data
 # requested.
 #------------------------------------------------------------------------------
@@ -720,7 +811,5 @@ sub printUsage()
     print ("  -s#               Specify which slot to act on (default = 0)\n");
     print ("  -p#               Specify which chip position to act on (default = 0)\n");
     print ("  -c#               Specify which core/chipUnit to act on (default = 3)\n");
-    print ("\n  NOTE:  This program will not work if user has not stopped instructions\n");
-    print ("         prior to running this program.\n");
 }