Fix continuous trace function losing traces in high traffic condition.

Change-Id: I9444e222e89e44017bf3039a09ecb28e9409a389
RTC: 39164
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/917
Tested-by: Jenkins Server
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

10 files changed, 551 insertions, 59 deletions

diff --git a/src/build/debug/Hostboot/ContTrace.pm b/src/build/debug/Hostboot/ContTrace.pm
index a9ba5eadb..1559199be 100755
--- a/src/build/debug/Hostboot/ContTrace.pm
+++ b/src/build/debug/Hostboot/ContTrace.pm
@@ -28,8 +28,12 @@ package Hostboot::ContTrace;
 use Exporter;
 our @EXPORT_OK = ('main');
 
+use constant MAX_NUM_CONT_TRACE_CTL_STRUCT => 2;
+use constant CONT_TRACE_CTL_STRUCT_SIZE => 16;
+use constant CONT_TRACE_ENABLE_FLAG_OFFSET => MAX_NUM_CONT_TRACE_CTL_STRUCT * CONT_TRACE_CTL_STRUCT_SIZE;
 use constant MAX_NUM_CONT_TRACE_BUFFERS => 2;
 use constant DDWORD_SIZE => 8;
+use constant WORD_SIZE => 4;
 use constant TRIG_BIT => 0x8000000000000000;
 
 use File::Temp ('tempfile');
@@ -46,6 +50,12 @@ sub main
         $args->{"fsp-trace"} = "fsp-trace";
     }
 
+    my $dbgMsg = 0;
+    if (defined $args->{"debug"})
+    {
+        $dbgMsg = 1;
+    }
+
     my $fsptrace_options = "";
     if (defined $args->{"with-file-names"})
     {
@@ -58,59 +68,184 @@ sub main
         $last = 1;
     }
 
-    my ($symAddr, $symSize) =
+    my $on2off = 0;
+    my $symAddr = 0;
+    my $symSize = 0;
+
+    if (defined $args->{"enable-cont-trace"})
+    {
+        my $new_enable = $args->{"enable-cont-trace"};
+        $new_enable = $new_enable ? 2 : 0;
+        ($symAddr, $symSize) =
+                      ::findSymbolAddress("TRACE::g_cont_trace_trigger_info");
+        if (not defined $symAddr)
+        {
+            ::userDisplay "Cannot find symbol.\n"; die;
+        }
+        my $enable = ::read64($symAddr + CONT_TRACE_ENABLE_FLAG_OFFSET);
+        if ($dbgMsg)
+        {
+            ::userDisplay("current Cont Trace Enable Flag = $enable\n");
+        }
+        if (($enable < 2) && ($new_enable == 2))
+        {
+            # truncate tracMERG to 0
+            system( "cp /dev/null tracMERG" );
+            ::write64($symAddr + CONT_TRACE_ENABLE_FLAG_OFFSET, $new_enable);
+            if ($dbgMsg)
+            {
+                $new_enable = ::read64($symAddr+CONT_TRACE_ENABLE_FLAG_OFFSET);
+                ::userDisplay("new Cont Trace Enable Flag = $new_enable\n");
+            }
+            return;
+        }
+        elsif (($enable == 2) && ($new_enable == 0))
+        {
+            $on2off = 1;
+            $last = 0;
+        }
+        else
+        {
+            return;
+        }
+    }
+
+    my $trigger = "simGETFAC B0.C0.S0.P0.E8.TPC.FSI.FSI_MAILBOX.FSXCOMP." .
+                  "FSXLOG.LBUS_MAILBOX.Q_GMB2E0.NLC.L2 32";
+    $trigger = `$trigger`;
+    $trigger =~ s/.*\n0xr(.*)\n.*/$1/g;
+    $trigger =~ s/\n//g;
+    if ($dbgMsg)
+    {
+        ::userDisplay("$trigger...\n");
+        my $cycles = `simgetcurrentcycle`;
+        $cycles =~ s/\n//g;
+        $cycles =~ s/.*is ([0-9]*).*/$1/g;
+        ::userDisplay("$cycles\n");
+        $cycles = `date`;
+        ::userDisplay("$cycles");
+    }
+    if (($trigger !~ /[1-9a-fA-F]+/) && ($last == 0))
+    {
+        if ($on2off)
+        {
+            ::write64($symAddr + CONT_TRACE_ENABLE_FLAG_OFFSET, 0);
+            if ($dbgMsg)
+            {
+                $on2off = ::read64($symAddr + CONT_TRACE_ENABLE_FLAG_OFFSET);
+                ::userDisplay("new Cont Trace Enable Flag = $on2off\n");
+            }
+        }
+        if ($dbgMsg)
+        {
+            my $cycles = `date`;
+            ::userDisplay("$cycles");
+        }
+        return;
+    }
+
+    ($symAddr, $symSize) =
                       ::findSymbolAddress("TRACE::g_cont_trace_trigger_info");
     if (not defined $symAddr) { ::userDisplay "Cannot find symbol.\n"; die; }
 
-    my ($fh,$fname) = tempfile();
-    binmode($fh);
+    my @fh;
+    my @fname;
+    ($fh[0],$fname[0]) = tempfile();
+    binmode($fh[0]);
+    ($fh[1],$fname[1]) = tempfile();
+    binmode($fh[1]);
 
     # read the g_cont_trace_trigger_info structure
     my $result = ::readData($symAddr, $symSize);
 
     my $addrOff = 0;
     my $lenOff = $addrOff + DDWORD_SIZE;
+    my $seqOff = $lenOff + WORD_SIZE;
     my $foundBuffer = 0;
+    my @seqNum;
 
     for (my $i = 0; $i < MAX_NUM_CONT_TRACE_BUFFERS; $i++)
     {
         # get the pointer to the continuous trace buffer
         my $buffAddr = substr $result, $addrOff, DDWORD_SIZE;
         $buffAddr= hex (unpack('H*',$buffAddr));
-        my $buffLen = substr $result, $lenOff, DDWORD_SIZE;
+        my $buffLen = substr $result, $lenOff, WORD_SIZE;
         $buffLen= hex (unpack('H*',$buffLen));
-        #::userDisplay("Trigger [".$i."] = $buffAddr\n");
-        #::userDisplay("Length  [".$i."] = $buffLen\n");
+        $seqNum[$i] = substr $result, $seqOff, WORD_SIZE;
+        $seqNum[$i]= hex (unpack('H*',$seqNum[$i]));
+        if ($dbgMsg)
+        {
+            ::userDisplay("Trigger [".$i."] = $buffAddr\n");
+            ::userDisplay("Length  [".$i."] = $buffLen\n");
+            ::userDisplay("SeqNum  [".$i."] = $seqNum[$i]\n");
+        }
 
+        my $fhandle = $fh[$i];
         # If trigger bit is set, or last call and buffer has trace data
         if ((0 != ($buffAddr & TRIG_BIT)) || (($last == 1) && ($buffLen > 1)))
         {
-            $foundBuffer = 1;
+            $foundBuffer |= (1 << $i);
             $buffAddr &= ~TRIG_BIT;
-            print $fh (::readData($buffAddr, $buffLen));
+            print $fhandle (::readData($buffAddr, $buffLen));
 
             # reset trigger bit
             ::write64($symAddr + $addrOff, $buffAddr);
 
             # reset count to 1
-            ::write64($symAddr + $lenOff, 1);
+            ::write32($symAddr + $lenOff, 1);
         }
 
         # increment to next element in g_cont_trace_trigger_info.triggers[]
         $addrOff += (2 * DDWORD_SIZE);
         $lenOff = $addrOff + DDWORD_SIZE;
+        $seqOff = $lenOff + WORD_SIZE;
     }
 
-    if ($foundBuffer)
+    if ($dbgMsg)
     {
-        open TRACE, ($args->{"fsp-trace"}." -s ".
-                    ::getImgPath()."hbotStringFile $fsptrace_options $fname |");
-        while (my $line = <TRACE>)
+        my $cycles = `simgetcurrentcycle`;
+        $cycles =~ s/\n//g;
+        $cycles =~ s/.*is ([0-9]*).*/$1/g;
+        ::userDisplay("$cycles\n");
+    }
+
+    if ($on2off)
+    {
+        ::write64($symAddr + CONT_TRACE_ENABLE_FLAG_OFFSET, 0);
+        if ($dbgMsg)
         {
-            ::userDisplay $line;
+            $on2off = ::read64($symAddr + CONT_TRACE_ENABLE_FLAG_OFFSET);
+            ::userDisplay("new Cont Trace Enable Flag = $on2off\n");
         }
     }
 
+    if (($foundBuffer == 3) && ($seqNum[1] < $seqNum[0]))
+    {
+        my $tmp = $fname[0];
+        $fname[0] =$fname[1];
+        $fname[1] = $tmp;
+    }
+
+    for (my $i = 0; $i < MAX_NUM_CONT_TRACE_BUFFERS; $i++)
+    {
+        if (($foundBuffer & (1 << $i)))
+        {
+            #my $cmd = "cp " . $fname[$i] . " tracMERG." . $seqNum[$i];
+            #system ( $cmd );
+            open TRACE, ($args->{"fsp-trace"}." -s ".
+               ::getImgPath()."hbotStringFile $fsptrace_options $fname[$i] |");
+            while (my $line = <TRACE>)
+            {
+                ::userDisplay $line;
+            }
+        }
+    }
+
+    if ($dbgMsg)
+    {
+        my $cycles = `date`;
+        ::userDisplay("$cycles");
+    }
 }
 
 sub helpInfo
@@ -123,6 +258,8 @@ sub helpInfo
                     "with-file-names" => ["Trace statements will include file name of place the",
                                           "trace was defined."],
                     "last" => ["Shutdown call to offload remaining traces."],
-                   },
+                    "enable-cont-trace=<1|0>" => ["Turn on|off continuous trace"],
+                    "debug" => ["Turn on debug messages"],
+                   }
     );
 }
diff --git a/src/build/debug/simics-debug-framework.py b/src/build/debug/simics-debug-framework.py
index 9453440a6..39a186666 100755
--- a/src/build/debug/simics-debug-framework.py
+++ b/src/build/debug/simics-debug-framework.py
@@ -342,6 +342,8 @@ def magic_instruction_callback(user_arg, cpu, arg):
         SIM_break_simulation( "Simulation stopped. (hap 7007)"  )
 
     if arg == 7055:   # MAGIC_CONTINUOUS_TRACE 
+        # Set execution environment flag to 0
+        writeLongLong(contTraceTrigInfo+32,0)
         # Continuous trace.
         # Residing at tracBinaryInfoAddr is the pointer to the tracBinary buffer
         pTracBinaryBuffer = readLongLong(tracBinaryInfoAddr)
@@ -378,6 +380,13 @@ for line in open('hbicore.syms'):
         words=line.split(",")
         tracBinaryInfoAddr=int(words[1],16)
         break
+# Find the address of the g_cont_trace_trigger_info and save it in
+# contTraceTrigInfo
+for line in open('hbicore.syms'):
+    if "g_cont_trace_trigger_info" in line:
+        words=line.split(",")
+        contTraceTrigInfo=int(words[1],16)
+        break
 
 # Continuous trace: Clear these files.
 rc = os.system( "rm -f tracMERG" )
diff --git a/src/build/vpo/do_sprint b/src/build/vpo/do_sprint
index 1d6690eb6..8e4a2af87 100755..100644
--- a/src/build/vpo/do_sprint
+++ b/src/build/vpo/do_sprint
@@ -176,6 +176,7 @@ sim_clocks=${AUTOVBU_SIMCLOCKS-2000000}
 loopstart=${AUTOVBU_START-0}
 loopmax=`expr $loopstart '+' ${AUTOVBU_LOOPS-2000}`
 lines=${AUTOVBU_PRINTK_LINES-50}
+sample=${AUTOVBU_PRINTK_SAMPLE-30}
 
 # If we requested checkpoints after loops or isteps, force a checkpoint after the hbitest as well
 test "$chkpt_after_loops" = "--chkpt_after_loops" && chkpt_after_hbitest="--chkpt_after_hbitest"
@@ -397,6 +398,7 @@ if [ "$start_after_hbitest" != "--start_after_hbitest" ]; then
 	
 	
 		echo >$printk_tmp1
+		cat /dev/null >printk_out
 	
 		if [ $loopcnt -ne 0 -a -d $AUTOVBU_TEST_DIR/trace_after_loop${loopcnt} ]; then
 			echo
@@ -420,12 +422,16 @@ if [ "$start_after_hbitest" != "--start_after_hbitest" ]; then
 		echo "       export AUTOCHKPT_VERSION=$AUTOCHKPT_VERSION"
 		echo
 	
+                $HB_TOOLS/hb-ContTrace --enable-cont-trace=1 --mute > /dev/null
+
 		while [ ! -f "$testloop_stop" -a $loopcnt -lt $loopmax ]; do
 			loopcnt=`expr $loopcnt + 1`
 	
 			simclock $sim_clocks  -quiet || exit 1
 			cur_cia=`GET_CIA`
 			echo "$loopcnt:	$cur_cia"
+                        $HB_TOOLS/hb-ContTrace --mute > /dev/null
+                        cat hb-ContTrace.output >> tracMERG
 			if [ "$use_hb_trace" = "--use_hb_trace" ]; then
 				p8_ins_stop
 				$HB_TOOLS/hb-trace $test_arg --out $printk_tmp3 2>&1 | egrep -v "^FAPI DBG>:|^FAPI IMP>:|Using deprecated ReturnCode function to assign integer" >$printk_tmp2
@@ -436,13 +442,19 @@ if [ "$start_after_hbitest" != "--start_after_hbitest" ]; then
 			elif [ "$use_dump_l3" = "--use_dump_l3" ]; then
 				DUMP_L3_PRINTK ${addr} ${lines} >$printk_tmp2
 			else
-				p8_ins_stop
-				$HB_TOOLS/hb-printk $test_arg --out $printk_tmp3 2>&1 | egrep -v "^FAPI DBG>:|^FAPI IMP>:|Using deprecated ReturnCode function to assign integer" >$printk_tmp2
-				p8_ins_start
+                           if [ `expr $loopcnt % $sample` -eq 0 ];
+                           then
+                               $HB_TOOLS/hb-printk --mute > /dev/null
+                               diff hb-Printk.output printk_out | sed -e 's/< //g' -e '1d'
+                               mv hb-Printk.output printk_out
+                           fi
+                        fi
+                        if [ -f $printk_tmp2 ];
+                        then
+                                echo >>$printk_tmp2
+                                diff -a $printk_tmp1 $printk_tmp2 | egrep -a "^> " | sed -e 's/[Ee][Rr][Rr][Oo][Rr]/ERR*R/g' -e 's/FAIL/F*IL/g'
+                                mv $printk_tmp2 $printk_tmp1
 			fi
-			echo >>$printk_tmp2
-			diff -a $printk_tmp1 $printk_tmp2 | egrep -a "^> " | sed -e 's/[Ee][Rr][Rr][Oo][Rr]/ERR*R/g' -e 's/FAIL/F*IL/g'
-			mv $printk_tmp2 $printk_tmp1
 	
 			grep 'HypeEmu: Illegal instruction' $printk_tmp1 >/dev/null
 			if [ $? -eq 0 ]; then
@@ -491,6 +503,10 @@ if [ "$start_after_hbitest" != "--start_after_hbitest" ]; then
 				simecho "Renew Tokens: `date`" -quiet
 			fi
 		done
+                $HB_TOOLS/hb-ContTrace --mute > /dev/null
+                cat hb-ContTrace.output >> tracMERG
+                $HB_TOOLS/hb-ContTrace --mute --last > /dev/null
+                cat hb-ContTrace.output >> tracMERG
 		date
 		rm -f $printk_tmp2 $printk_tmp1
 	
diff --git a/src/build/vpo/hb-istep b/src/build/vpo/hb-istep
index 6bbcae7a3..81d96e190 100755
--- a/src/build/vpo/hb-istep
+++ b/src/build/vpo/hb-istep
@@ -132,6 +132,12 @@ my  $g_SeqNum   =   int(rand(64));
 my  $pgmDir  =   `dirname $0`;
 chomp( $pgmDir );
 
+my $hbToolsDir = $ENV{'HB_TOOLS'};
+if ( ! defined( $hbToolsDir) || ( $hbToolsDir eq "" ) )
+{
+    $hbToolsDir = $pgmDir;         ##  Set to tool directory
+}     
+
 my $hbDir = $ENV{'HB_IMGDIR'};
 if ( ! defined( $hbDir) || ( $hbDir eq "" ) )
 {
@@ -487,6 +493,8 @@ sub getSyncStatus( )
         ##  sequence number to see if it has changed.  rinse and repeat.
         ##  Note:  RunClocks will start instructions 
         VBU_Cacheline::RunClocks();
+        system ("$hbToolsDir/hb-ContTrace --mute > /dev/null");
+        system ("cat hb-ContTrace.output >> tracMERG");
 
         ##  Stop instructions, flush L2
         VBU_Cacheline::P8_Ins_Stop();
@@ -742,6 +750,8 @@ sub setMode( $ )
         ##  advance HostBoot code by a certain # of cycles, then check the 
         ##  sequence number to see if it has changed.  rinse and repeat.
         VBU_Cacheline::RunClocks();
+        system ("$hbToolsDir/hb-ContTrace --mute > /dev/null");
+        system ("cat hb-ContTrace.output >> tracMERG");
 
         ##  Stop instructions, flush L2
          VBU_Cacheline::P8_Ins_Stop();
diff --git a/src/include/usr/hbotcompid.H b/src/include/usr/hbotcompid.H
index 721b7c510..1c359a8cc 100644
--- a/src/include/usr/hbotcompid.H
+++ b/src/include/usr/hbotcompid.H
@@ -210,6 +210,14 @@ const compId_t HBMDIA_COMP_ID = 0x1400;
 const char HBMDIA_COMP_NAME[] = "mdia";
 //@}
 
+/** @name TRACE
+ *  Trace component
+ */
+//@{
+const compId_t HBTRACE_COMP_ID = 0x1500;
+const char HBTRACE_COMP_NAME[] = "trace";
+//@}
+
 /** @name RESERVED
  *  Reserved component ID. x3100 is the component ID
  *  of FipS ERRL component. Due to our use of
diff --git a/src/include/usr/trace/trace.H b/src/include/usr/trace/trace.H
index b8656ee3e..0997c60f7 100644
--- a/src/include/usr/trace/trace.H
+++ b/src/include/usr/trace/trace.H
@@ -39,6 +39,7 @@
 #include <util/singleton.H>
 #include <sys/sync.h>
 #include <stdarg.h>
+#include <list>
 
 
 /******************************************************************************/
@@ -131,6 +132,9 @@ namespace TRACE
 class Trace;
 typedef Singleton<Trace> theTrace;
 
+// Forward declaration.
+class TraceDaemon;
+
 /**
  *  @brief  Trace Singleton Class
  *
@@ -149,12 +153,12 @@ public:
     static Trace& getTheInstance();
 
     /**
-     *  @brief  Initialize a trace buffer. 
+     *  @brief  Initialize a trace buffer.
      *
      *  Size is capped at 2KB. You can request larger, but
-     *  the code in src/usr/trace/trace.C imposes 
+     *  the code in src/usr/trace/trace.C imposes
      *  a maximum size of 2KB. Sizes smaller than 2KB
-     *  will save space.  
+     *  will save space.
      *
      *  @param [out] o_td Trace descriptor to initialize
      *  @param [in] i_comp Component name for trace buffer
@@ -219,23 +223,23 @@ public:
      *  first query the size of the buffer by calling with the desired
      *  buffer name and with o_data null and i_bufferSize
      *  zero. The value returned will be the full buffer size.  Caller
-     *  allocates the buffer and calls again. 
-     *  
+     *  allocates the buffer and calls again.
+     *
      *  The buffer provided can be less than the full size of the desired
      *  buffer. In that case, this function will copy as many of the most
-     *  recent traces into the output buffer as will fit. The buffer must 
-     *  be big enough to hold a trace buffer header (40 bytes). 
+     *  recent traces into the output buffer as will fit. The buffer must
+     *  be big enough to hold a trace buffer header (40 bytes).
      *
-     *  i_bufferSize may be larger that the desired trace buffer. 
+     *  i_bufferSize may be larger that the desired trace buffer.
      *
-     *  @param [in]  i_pName      name of trace buffer 
+     *  @param [in]  i_pName      name of trace buffer
      *  @param [out] o_data       pointer to output buffer
      *  @param [in]  i_bufferSize size of output buffer in bytes
      *
      *  @return Count of bytes copied, or if given null parameters,
      *  the size of the buffer. Returns zero for error, perhaps the
      *  component name/trace buffer name is not found, or perhaps
-     *  the size of the provided buffer is unreasonable.  
+     *  the size of the provided buffer is unreasonable.
      */
     uint64_t getBuffer( const char * i_pName,
                         void *       o_data,
@@ -244,7 +248,7 @@ public:
 
 #if !defined(__HIDDEN_TRACEIF_CLEARBUFFER)
 private:
-#endif    
+#endif
 
 
 
@@ -390,7 +394,7 @@ private:
 
     /**
      *  @brief  This function manages the usage of the two ping-pong buffers
-     *          for handling the continuous trace support under VPO/VBU. 
+     *          for handling the continuous trace support under VPO/VBU.
      *
      *  @param [in] i_cbRequired number of bytes needed for the trace entry
      *              to be added to the active continuous trace buffer
@@ -403,10 +407,16 @@ private:
     mutex_t iv_trac_mutex;
 
     // Controls writing to tracBinary
-    bool iv_ContinuousTrace; 
+    bool iv_ContinuousTrace;
 
     // VPO/VBU continuous trace active buffer index
     uint8_t iv_CurBuf;
+
+    // sequence number
+    uint32_t iv_seqNum;
+
+    // Trace Daemon for messages.
+    TraceDaemon* iv_daemon;
 };
 
 } // namespace TRACE
diff --git a/src/usr/trace/makefile b/src/usr/trace/makefile
index c025957c0..fe7cf2d92 100644
--- a/src/usr/trace/makefile
+++ b/src/usr/trace/makefile
@@ -23,7 +23,7 @@
 ROOTPATH = ../../..
 MODULE = trace
 
-OBJS = trace.o assert.o
+OBJS = trace.o tracedaemon.o assert.o
 
 SUBDIRS = test.d
 
diff --git a/src/usr/trace/trace.C b/src/usr/trace/trace.C
index e43c8661a..398f6c418 100644
--- a/src/usr/trace/trace.C
+++ b/src/usr/trace/trace.C
@@ -54,8 +54,7 @@
 #include <assert.h>
 
 #include <trace/trace.H>
-
-
+#include "tracedaemon.H"
 
 
 /******************************************************************************/
@@ -125,21 +124,24 @@ mixed_trace_info_t g_tracBinaryInfo[2];
 
 struct vpo_con_trigger_t
 {
-    uint64_t trig;  // bit0 = trigger signalling bit1:63 = trace buffer addr
-    uint64_t len;   // length of trace buffer with valid trace data.
+    volatile uint64_t trig;  // bit0 = trig signalling bit1:63 = trace buff addr
+    uint32_t len;   // length of trace buffer with valid trace data.
+    uint32_t seq;
 };
 
 struct vpo_cont_support_t
 {
     vpo_con_trigger_t triggers[2];
-    uint64_t disable;  // clear by VPO script to enable continous trace
+    uint64_t enable;   // VPO script sets it to 2
+                       // SIMICS hap handler sets it to 0
+                       // Compiler sets it to 1 for Mbox
 };
 
-// This structure is monitored by VPO script. The disable variable is set
-// at compile time to 1. The VPO script clears the disable variable at thei
+// This structure is monitored by VPO script. The enable variable is set
+// at compile time to 1. The VPO script set the enable variable at the
 // start to enable the continuous trace support for VPO. It then montiors the
 // trigger active bit of each buffer and take action.
-vpo_cont_support_t g_cont_trace_trigger_info = { { { 0, 0 }, { 0, 0 } }, 1 };
+vpo_cont_support_t g_cont_trace_trigger_info = { { {0,0,0}, {0,0,0} }, 1 };
 
 const uint64_t TRIGGER_ACTIVE_BIT = 0x8000000000000000;
 
@@ -190,6 +192,10 @@ Trace::Trace()
                       reinterpret_cast<uint64_t>(g_tracBinaryInfo[i].pBuffer);
     }
 
+    // if this code is running under simics, call the hap handler to set
+    // g_cont_trace_trigger_info.enable to 0. Otherwise, this will be noop
+    MAGIC_INSTRUCTION(MAGIC_CONTINUOUS_TRACE);
+
     // tracBINARY buffer appending is always on.
     // TODO figure a way to control continuous trace on/off, perhaps
     // unregister the hap handler for it.
@@ -197,6 +203,11 @@ Trace::Trace()
 
     // select buffer0 initially
     iv_CurBuf = 0;
+    // initialize seq number
+    iv_seqNum = 0;
+
+    // Create the daemon.
+    iv_daemon = new TraceDaemon();
 }
 
 /******************************************************************************/
@@ -204,7 +215,8 @@ Trace::Trace()
 /******************************************************************************/
 Trace::~Trace()
 {
-
+    // Delete the daemon.
+    delete iv_daemon;
 }
 
 
@@ -344,23 +356,66 @@ void Trace::initValuesBuffer( trace_desc_t *o_buf,
 // ManageContTraceBuffers
 // This function manages the usage of the two ping-pong buffers for handling
 // the continuous trace support.
+//
+// 1) Under VPO/VBU (g_cont_trace_trigger_info.enable = 2)
+//    The handshake between Hostboot code and VPO/VBU script is shown belows
+//                        _______________________
+//    Trigger_Avtive ____/                       \__________
+//                           _________________________
+//    ScomReg_Active _______/                         \______
+//
+//    Hostboot code (this function) sets Trigger_Active, then ScomReg_Active,
+//    VPO/VBU script detects ScomReg_Active active, off-load the trigger buffer,
+//    and clears Trigger_Active. Hostboot code detects a trigger-state buffer
+//    with Trigger_Active cleared, reset ScomReg_Active. Two-level trigger is
+//    desired. The ScomReg trigger allows VPO script to sample at no expense
+//    of simclocks, thus avoiding spending extra simclocks associated with the
+//    flushing of L2/L3 to read the memory trigger.
+//
+// 2) Under Simics with hap handler (g_cont_trace_trigger_info.enable = 0)
+//    The handshake between Hostboot code and the hap handler is shown belows
+//                        _______________________
+//    Trigger_Avtive ____/                       \__________
+//
+//    Hostboot code (this function) sets Trigger_Active, then invokes the hap
+//    handler. The hap handler off-loads the trigger buffer, and clears the
+//    Trigger_Active.
+//
+// 3) Under Simics/HW with FSP mbox (g_cont_trace_trigger_info.enable = 1)
+//                        _______________________
+//    Trigger_Avtive ____/                       \__________
+//
+//    Hostboot code (this function) sets Trigger_Active, then schedule a thread
+//    to use mbox DMA MSG to off-load the trigger buffer to FSP, and clears the
+//    Trigger_Active.
+//
 /******************************************************************************/
 void Trace::ManageContTraceBuffers(uint64_t i_cbRequired)
 {
     uint8_t l_AltBuf = (iv_CurBuf + 1) % 2;
+    bool needScomReset = false;
 
     // Reset TriggerActive if the buffer has been offloaded by VPO
     // script when running under VBU Awan environment
-    for (size_t i = 0; (!(g_cont_trace_trigger_info.disable)) && (i < 2); i++)
+    for (size_t i = 0; (g_cont_trace_trigger_info.enable > 1) && (i < 2); i++)
     {
         if ((g_tracBinaryInfo[i].TriggerActive != 0) &&
              (!(g_cont_trace_trigger_info.triggers[i].trig &
                                                          TRIGGER_ACTIVE_BIT)))
         {
             g_tracBinaryInfo[i].TriggerActive = 0;
+            needScomReset = true;
         }
     }
 
+    if (needScomReset)
+    {
+        msg_t* l_msg = msg_allocate();
+        l_msg->type = TraceDaemon::UPDATE_SCRATCH_REG;
+        l_msg->data[0] = 0;
+        msg_send(iv_daemon->iv_msgQ, l_msg);
+    }
+
     // we should never have the current buffer in the trigger state
     assert (g_tracBinaryInfo[iv_CurBuf].TriggerActive == 0);
 
@@ -372,19 +427,38 @@ void Trace::ManageContTraceBuffers(uint64_t i_cbRequired)
         // current buffer entering trigger state
         g_tracBinaryInfo[iv_CurBuf].TriggerActive = 1;
 
-        if (!(g_cont_trace_trigger_info.disable))
+        if (g_cont_trace_trigger_info.enable > 1)
         {
+            if (g_tracBinaryInfo[l_AltBuf].TriggerActive == 1)
+            {
+                // If the alternate buffer's trigger is active, wait for a
+                // chance to get offloaded before it is reused.
+                uint64_t l_wait = 0x100000;
+                while (g_cont_trace_trigger_info.triggers[l_AltBuf].trig &
+                             TRIGGER_ACTIVE_BIT)
+                {
+                    if (--l_wait == 0)
+                    {
+                        break;
+                    }
+                }
+                // If alternate buffer has been offloaded, exit trigger state.
+                if (l_wait != 0)
+                {
+                    g_tracBinaryInfo[l_AltBuf].TriggerActive = 0;
+                }
+            }
+
+            g_cont_trace_trigger_info.triggers[iv_CurBuf].seq = iv_seqNum++;
             // Turn on the current buffer's trigger
             g_cont_trace_trigger_info.triggers[iv_CurBuf].trig |=
                                                            TRIGGER_ACTIVE_BIT;
-            // If the alternate buffer's trigger is active and
-            // the buffer will now be reused, so reset the trigger
-            if (g_cont_trace_trigger_info.triggers[l_AltBuf].trig &
-                                                           TRIGGER_ACTIVE_BIT)
-            {
-                g_cont_trace_trigger_info.triggers[l_AltBuf].trig &=
-                                                          ~TRIGGER_ACTIVE_BIT;
-            }
+
+            msg_t* l_msg = msg_allocate();
+            l_msg->type = TraceDaemon::UPDATE_SCRATCH_REG;
+            l_msg->data[0] = 0x13579BDF00000000;
+            l_msg->data[0] += (iv_seqNum * 0x100000000);
+            msg_send(iv_daemon->iv_msgQ, l_msg);
         }
 
         // If the alternate buffer is in trigger state, move it out of
@@ -395,9 +469,23 @@ void Trace::ManageContTraceBuffers(uint64_t i_cbRequired)
             g_tracBinaryInfo[l_AltBuf].TriggerActive = 0;
         }
         // Now switching to alternate buffer and reset the usage count
+        uint8_t l_cur = iv_CurBuf;
+        uint64_t l_len = g_tracBinaryInfo[l_cur].cbUsed;
         iv_CurBuf = l_AltBuf;
         g_tracBinaryInfo[iv_CurBuf].cbUsed = 1;
 
+        // For FSP mbox method.
+        if (g_cont_trace_trigger_info.enable == 1)
+        {
+            msg_t* l_msg = msg_allocate();
+            l_msg->type = TraceDaemon::SEND_TRACE_BUFFER;
+            l_msg->data[1] = TRAC_BINARY_SIZE;
+            l_msg->extra_data = malloc(TRAC_BINARY_SIZE);
+            memcpy( l_msg->extra_data, g_tracBinaryInfo[l_cur].pBuffer, l_len );
+            msg_send(iv_daemon->iv_msgQ, l_msg);
+            g_tracBinaryInfo[l_cur].TriggerActive = 0;
+        }
+
         MAGIC_INSTRUCTION(MAGIC_CONTINUOUS_TRACE);
     }
 }
@@ -462,8 +550,7 @@ void Trace::_trace_adal_write_all(trace_desc_t *io_td,
     // Sum the sizes of the items in i_args in order to know how big to
     // allocate the entry.
 
-    const size_t fmt_len = strlen(_fmt);
-    for (size_t i = 0; i <= fmt_len; i++)
+    for (size_t i = 0; i <= strlen(_fmt); i++)
     {
         if ('%' == _fmt[i])
         {
@@ -693,7 +780,7 @@ void Trace::_trace_adal_write_all(trace_desc_t *io_td,
             g_tracBinaryInfo[iv_CurBuf].cbUsed += l_cbRequired;
 
             // maintain the buffer's actually used bytes for VPO script
-            if ((!g_cont_trace_trigger_info.disable))
+            if (g_cont_trace_trigger_info.enable > 1)
             {
                 g_cont_trace_trigger_info.triggers[iv_CurBuf].len =
                                            g_tracBinaryInfo[iv_CurBuf].cbUsed;
@@ -847,7 +934,7 @@ void Trace::_trace_adal_write_bin(trace_desc_t *io_td,const trace_hash_val i_has
             g_tracBinaryInfo[iv_CurBuf].cbUsed += l_cbRequired;
 
             // maintain the buffer's actually used bytes for VPO script
-            if ((!g_cont_trace_trigger_info.disable))
+            if (g_cont_trace_trigger_info.enable > 1)
             {
                 g_cont_trace_trigger_info.triggers[iv_CurBuf].len =
                                             g_tracBinaryInfo[iv_CurBuf].cbUsed;
@@ -1390,7 +1477,6 @@ void Trace::clearAllBuffers()
 
 
 
-
 #if 0
 /******************************************************************************/
 // resetBuf - TODO
diff --git a/src/usr/trace/tracedaemon.C b/src/usr/trace/tracedaemon.C
new file mode 100644
index 000000000..c4bd4e56c
--- /dev/null
+++ b/src/usr/trace/tracedaemon.C
@@ -0,0 +1,124 @@
+//  IBM_PROLOG_BEGIN_TAG
+//  This is an automatically generated prolog.
+//
+//  $Source: src/usr/trace/tracedaemon.C $
+//
+//  IBM CONFIDENTIAL
+//
+//  COPYRIGHT International Business Machines Corp. 2012
+//
+//  p1
+//
+//  Object Code Only (OCO) source materials
+//  Licensed Internal Code Source Materials
+//  IBM HostBoot Licensed Internal Code
+//
+//  The source code for this program is not published or other-
+//  wise divested of its trade secrets, irrespective of what has
+//  been deposited with the U.S. Copyright Office.
+//
+//  Origin: 30
+//
+//  IBM_PROLOG_END
+#include "tracedaemon.H"
+#include <sys/task.h>
+#include <targeting/common/commontargeting.H>
+#include <vfs/vfs.H>
+#include <devicefw/driverif.H>
+#include <errl/errlentry.H>
+#include <errl/errlmanager.H>
+
+namespace TRACE
+{
+
+TraceDaemon::TraceDaemon() :
+    iv_pMaster(TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL)
+{
+    iv_msgQ = msg_q_create();
+    task_create(TraceDaemon::start, this);
+}
+
+TraceDaemon::~TraceDaemon()
+{
+        // Send message to shutdown daemon thread.
+    msg_t* l_msg = msg_allocate();
+    l_msg->type = DAEMON_SHUTDOWN;
+    msg_sendrecv(iv_msgQ, l_msg);
+    msg_free(l_msg);
+
+        // Release message queue.
+    msg_q_destroy(iv_msgQ);
+}
+
+void TraceDaemon::start(void* i_self)
+{
+    reinterpret_cast<TraceDaemon *>(i_self)->run();
+};
+
+void TraceDaemon::run()
+{
+    msg_t* l_msg = NULL;
+
+    // Main daemon loop.
+    while(1)
+    {
+        // Get message from client.
+        l_msg = msg_wait(iv_msgQ);
+
+        // Switch based on message type.
+        switch(l_msg->type)
+        {
+            case UPDATE_SCRATCH_REG:
+                updateScratchReg(l_msg->data[0]);
+                break;
+
+            case SEND_TRACE_BUFFER: // TODO.
+                    // Delete buffer for now.
+                free(l_msg->extra_data);
+                break;
+
+            case DAEMON_SHUTDOWN:
+                    // Respond to message and exit.
+                msg_respond(iv_msgQ, l_msg);
+                task_end();
+                break;
+        };
+
+        if (msg_is_async(l_msg))
+        {
+            // Delete async messages.
+            msg_free(l_msg);
+        }
+        else
+        {
+            // Respond to sync messages.
+            msg_respond(iv_msgQ, l_msg);
+        }
+    }
+
+}
+
+void TraceDaemon::updateScratchReg(uint64_t i_value)
+{
+    // Find master processor target.
+    if (iv_pMaster == TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL)
+    {
+        if (VFS::module_is_loaded("libtargeting.so") &&
+            TARGETING::targetService().isInitialized())
+        {
+            TARGETING::targetService().masterProcChipTargetHandle(iv_pMaster);
+        }
+    }
+
+    // Write scratch register to requested value.
+    size_t l_size = sizeof(uint64_t);
+    errlHndl_t l_errl = deviceWrite(iv_pMaster, &i_value, l_size,
+                                    DEVICE_SCOM_ADDRESS(MB_SCRATCH_REGISTER_0));
+
+    if (l_errl)
+    {
+        errlCommit(l_errl, HBTRACE_COMP_ID);
+    }
+}
+
+};
diff --git a/src/usr/trace/tracedaemon.H b/src/usr/trace/tracedaemon.H
new file mode 100644
index 000000000..6c4006701
--- /dev/null
+++ b/src/usr/trace/tracedaemon.H
@@ -0,0 +1,92 @@
+//  IBM_PROLOG_BEGIN_TAG
+//  This is an automatically generated prolog.
+//
+//  $Source: src/usr/trace/tracedaemon.H $
+//
+//  IBM CONFIDENTIAL
+//
+//  COPYRIGHT International Business Machines Corp. 2012
+//
+//  p1
+//
+//  Object Code Only (OCO) source materials
+//  Licensed Internal Code Source Materials
+//  IBM HostBoot Licensed Internal Code
+//
+//  The source code for this program is not published or other-
+//  wise divested of its trade secrets, irrespective of what has
+//  been deposited with the U.S. Copyright Office.
+//
+//  Origin: 30
+//
+//  IBM_PROLOG_END
+
+#ifndef __TRACE_DAEMON_H
+#define __TRACE_DAEMON_H
+
+#include <sys/msg.h>
+#include <targeting/common/target.H>
+
+namespace TRACE
+{
+    /** @class TraceDaemon
+     *  @brief Encapsulates the daemon functionality of trace.
+     *
+     *  The main-line trace paths can send messages to this code to request
+     *  actions, such as buffer offload (by mailbox or signalling a scratch
+     *  register), to be done or (TODO) the FSP can send messages to it.
+     */
+    class TraceDaemon
+    {
+        public:
+            /** Default Constructor
+             *
+             *  Initializes class and starts daemon thread.
+             */
+            TraceDaemon();
+
+            /** Default Destructor
+             *
+             *  Shuts down daemon thread and releases mailbox queue.
+             */
+            ~TraceDaemon();
+
+            /** Message types supported by the trace daemon. */
+            enum SUPPORTED_MSG_TYPES
+            {
+                UPDATE_SCRATCH_REG,     //< Update cont-trace scratch reg.
+                SEND_TRACE_BUFFER,      //< Send buffer to FSP.
+
+                DAEMON_SHUTDOWN,        //< Shutdown daemon thread.
+            };
+
+            // Make trace class a friend so it can get the message queue.
+            friend class Trace;
+
+        protected:
+                /** Message Queue */
+            msg_q_t iv_msgQ;
+
+        private:
+                /** Target for master processor */
+            TARGETING::Target* iv_pMaster;
+
+                /** SCOM address of scratch register. */
+            static const uint32_t MB_SCRATCH_REGISTER_0 = 0x00050038;
+
+            /** @brief Function to start daemon thread (using task_create).
+             *  @param[in] Pointer to self.
+             */
+            static void start(void*);
+
+            /** @brief Main daemon loop. */
+            void run();
+
+            /** @brief Update a scratch register with the desired value.
+             *  @param[in] i_value - Value to write to scratch register.
+             */
+            void updateScratchReg(uint64_t i_value);
+    };
+};
+
+#endif