Log traces to error logs in HBRT

This enables buffer tracing at hostboot runtime.
Will add these traces to runtime errors for better debug

Change-Id: I795bb7deafdd02adea4588ebf8dfb11cbce116a0
RTC:172770
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/48084
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Martin Gloff <mgloff@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

9 files changed, 1049 insertions, 24 deletions

diff --git a/src/usr/trace/buffer.H b/src/usr/trace/buffer.H
index b73d7b00b..a2b53d628 100644
--- a/src/usr/trace/buffer.H
+++ b/src/usr/trace/buffer.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2012,2015                        */
+/* Contributors Listed Below - COPYRIGHT 2012,2017                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -31,16 +31,20 @@
 
 #include <trace/interface.H>
 #include <sys/sync.h>
+#include "daemon/daemon.H"
 
+#ifndef __HOSTBOOT_RUNTIME
 namespace TRACEDAEMON { class Daemon; } // Forward declaration.
+#endif
 
 namespace TRACE
 {
         // Forward declarations.
     class BufferPage;
     class Entry;
+#ifndef __HOSTBOOT_RUNTIME
     class DaemonIf;
-
+#endif
     /** @class Buffer
      *  @brief Class to manage the front-side (client) buffers
      *
@@ -62,6 +66,7 @@ namespace TRACE
                 UNLIMITED = UINT32_MAX
             };
 
+#ifndef __HOSTBOOT_RUNTIME
             /** Constructor.
              *
              *  @param[in] i_daemon - Daemon interface for this buffer.
@@ -69,7 +74,15 @@ namespace TRACE
              *                          before 'claimEntry' blocks.
              */
             Buffer(DaemonIf* i_daemon, size_t i_maxPages = 4);
-
+#else
+            /** Constructor.
+             *
+             *  @param[in] i_daemon   - Runtime "daemon" for this buffer.
+             *  @param[in] i_maxPages - Maximum number of pages to consume
+             *                          before 'claimEntry' blocks.
+             */
+            Buffer(TRACEDAEMON::Daemon * i_daemon, size_t i_maxPages = 4);
+#endif
             /** @brief Claim an entry from the buffer to write data to.
              *
              *  @param[in] i_comp - Component which will own entry.
@@ -104,6 +117,7 @@ namespace TRACE
 
             BufferPage* volatile iv_firstPage; //< Most recent page.
 
+#ifndef __HOSTBOOT_RUNTIME
             /** @union locklessCounter
              *
              *  This class is mostly lockless in that multiple trace entries
@@ -141,6 +155,13 @@ namespace TRACE
 
             DaemonIf* iv_daemon; //< Daemon interface.
 
+            void _producerEnter();      //< Enter client section.
+            void _producerExit();       //< Exit client section.
+            void _consumerEnter();      //< Enter daemon section.
+            void _consumerExit();       //< Exit daemon section.
+#else
+            TRACEDAEMON::Daemon * iv_daemon;  // Daemon
+#endif
             /** @brief Claim front-side buffer pages to be merged into the
              *         common buffer.
              *
@@ -186,13 +207,11 @@ namespace TRACE
                                 Entry* i_condActVal = NULL,
                             Entry** i_addr = NULL, Entry* i_val = NULL);
 
-            void _producerEnter();      //< Enter client section.
-            void _producerExit();       //< Exit client section.
-            void _consumerEnter();      //< Enter daemon section.
-            void _consumerExit();       //< Exit daemon section.
 
             friend class BufferTest;
+
             friend class TRACEDAEMON::Daemon;
+
     };
 }
 
diff --git a/src/usr/trace/bufferpage.C b/src/usr/trace/bufferpage.C
index 00b9ff447..c6cfb57e0 100644
--- a/src/usr/trace/bufferpage.C
+++ b/src/usr/trace/bufferpage.C
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012,2014              */
+/* Contributors Listed Below - COPYRIGHT 2012,2017                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -25,7 +27,11 @@
 
 #include <limits.h>
 #include <stdlib.h>
+#include <string.h>
+
+#ifndef __HOSTBOOT_RUNTIME
 #include <kernel/pagemgr.H>
+#endif
 
 namespace TRACE
 {
@@ -60,7 +66,11 @@ namespace TRACE
     {
         BufferPage* page = NULL;
 
+#ifndef __HOSTBOOT_RUNTIME
         page = reinterpret_cast<BufferPage*>(PageManager::allocatePage());
+#else
+        page = reinterpret_cast<BufferPage*>(malloc(PAGESIZE));
+#endif
         memset(page, '\0', PAGESIZE);
 
         if (i_common)
@@ -73,7 +83,15 @@ namespace TRACE
 
     void BufferPage::deallocate(BufferPage* i_page)
     {
+#ifndef __HOSTBOOT_RUNTIME
         PageManager::freePage(i_page);
+#else
+        if (i_page != nullptr)
+        {
+           free(i_page);
+           i_page = nullptr;
+        }
+#endif
     }
 
 }
diff --git a/src/usr/trace/daemon/daemon.H b/src/usr/trace/daemon/daemon.H
index 57e908696..46af37f35 100644
--- a/src/usr/trace/daemon/daemon.H
+++ b/src/usr/trace/daemon/daemon.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2012,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2012,2017                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -66,6 +66,10 @@ namespace TRACEDAEMON
              */
             static void* start(void*);
 
+       #ifdef __HOSTBOOT_RUNTIME
+            void signal_trace_daemon(void);
+       #endif
+
         private:
 
             /** @fn execute
diff --git a/src/usr/trace/interface.C b/src/usr/trace/interface.C
index c7856e712..f7d8e68ff 100644
--- a/src/usr/trace/interface.C
+++ b/src/usr/trace/interface.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2012,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2012,2017                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -64,6 +64,10 @@ namespace TRACE
             i_size = KILOBYTE;
         }
 
+#ifdef __HOSTBOOT_RUNTIME
+        // Run all Runtime traces to same buffer
+        i_bufferType = BUFFER_FAST;
+#endif
         (*o_td) =
             Singleton<ComponentList>::instance().getDescriptor(i_comp,
                                                                i_size,
diff --git a/src/usr/trace/runtime/makefile b/src/usr/trace/runtime/makefile
index 2c37db85f..c7307e0cd 100644
--- a/src/usr/trace/runtime/makefile
+++ b/src/usr/trace/runtime/makefile
@@ -5,7 +5,9 @@
 #
 # OpenPOWER HostBoot Project
 #
-# COPYRIGHT International Business Machines Corp. 2013,2014
+# Contributors Listed Below - COPYRIGHT 2013,2017
+# [+] International Business Machines Corp.
+#
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,7 +29,14 @@ MODULE = trace_rt
 OBJS += interface.o
 OBJS += assert.o
 OBJS += compdesc.o
+OBJS += complist.o
 OBJS += rt_service.o
+OBJS += bufferpage.o
+OBJS += rt_daemon.o
+OBJS += rt_buffer.o
+
 
 VPATH += ..
+VPATH += ../daemon
+
 include $(ROOTPATH)/config.mk
diff --git a/src/usr/trace/runtime/rt_buffer.C b/src/usr/trace/runtime/rt_buffer.C
new file mode 100644
index 000000000..7f95d7035
--- /dev/null
+++ b/src/usr/trace/runtime/rt_buffer.C
@@ -0,0 +1,382 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/usr/trace/runtime/rt_buffer.C $                           */
+/*                                                                        */
+/* OpenPOWER HostBoot Project                                             */
+/*                                                                        */
+/* Contributors Listed Below - COPYRIGHT 2017                             */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+#include "../buffer.H"
+#include "../bufferpage.H"
+#include "../entry.H"
+#include "../compdesc.H"
+#include "../daemonif.H"
+
+#include <assert.h>
+#include <limits.h>
+#include <string.h>
+#include <util/align.H>
+
+#include <arch/ppc.H>
+
+
+
+namespace TRACE
+{
+    Buffer::Buffer(TRACEDAEMON::Daemon * i_daemon, size_t i_maxPages) :
+        iv_pagesAlloc(0), iv_pagesMax(i_maxPages),
+        iv_firstPage(NULL), iv_daemon(i_daemon)
+    {
+        assert(i_maxPages > 0);
+    }
+
+
+    Entry* Buffer::claimEntry(ComponentDesc* i_comp, size_t i_size)
+    {
+        // Bump up the needed size to include the entry itself and
+        // necessary alignment.
+        // (Alignment is needed so that Entry's members can be atomically
+        //  updated).
+        i_size = ALIGN_8(i_size + sizeof(Entry));
+
+        // If an entry is bigger than this amount, it can never be satisfied.
+        if (i_size > (PAGESIZE - sizeof(BufferPage)))
+        {
+            return NULL;
+        }
+
+        Entry* l_entry = NULL;
+
+        // Now we begin the search for an entry.
+        do
+        {
+            BufferPage* first = iv_firstPage;
+
+            // Attempt to claim from the current page first.
+            if (first)
+            {
+                l_entry = first->claimEntry(i_size);
+                if (l_entry)
+                {
+                    // Found one, great!  We're done.
+                    break;
+                }
+            }
+
+            // Wasn't enough space, so try to allocate a new page.
+            uint64_t pagesAllocated = iv_pagesAlloc;
+            if (pagesAllocated >= iv_pagesMax)
+            {
+                // Clean up old pages and consolidate traces
+                iv_daemon->signal_trace_daemon();
+
+                // A page might be allocated now, start over.
+                continue;
+            }
+
+            // Atomically update page count.
+            uint64_t newPagesAllocated = pagesAllocated + 1;
+            if (!__sync_bool_compare_and_swap(&iv_pagesAlloc,
+                                              pagesAllocated,
+                                              newPagesAllocated))
+            {
+                // Someone beat us to updating the count, so start over.
+                // (another thread is also trying to allocate a page, let them
+                //  do it).
+                continue;
+            }
+
+            // Successfully updated the count so allocate the new page.
+            BufferPage* newPage = BufferPage::allocate();
+            newPage->prev = first;
+
+            // Now we have a page allocated, claim our entry first and then
+            // hook it up to master list.
+            l_entry = newPage->claimEntry(i_size);
+
+            iv_firstPage = newPage;
+
+            // If there was a page already, update its next pointer to point
+            // back at this new page.
+            if (first)
+            {
+                if (!__sync_bool_compare_and_swap(&first->next,
+                                                  NULL,
+                                                  newPage))
+                {
+                    // We were the first one to update iv_firstPage, so
+                    // first->next should have been NULL and nobody was
+                    // suppose to touch it.
+                    assert(false);
+                }
+            }
+
+            // And since we allocated a page, wake up the daemon if we
+            // allocated the last page or if there are more than 4 pages
+            // and we are an infinite buffer.  (4 pages is arbitrary).
+            static const size_t SIGNAL_AFTER_N_PAGES_ALLOCATED = 4;
+
+            if ((pagesAllocated == iv_pagesMax) ||
+                ((pagesAllocated >= SIGNAL_AFTER_N_PAGES_ALLOCATED) &&
+                 (iv_pagesMax == UNLIMITED)))
+            {
+                // Clean up old pages and consolidate traces
+                iv_daemon->signal_trace_daemon();
+            }
+        } while(!l_entry);
+
+        // Update component name and entry size.
+        l_entry->comp = i_comp;
+        l_entry->size = i_size - sizeof(Entry);
+
+        return l_entry;
+    }
+
+    BufferPage* Buffer::claimPages()
+    {
+        BufferPage* page = iv_firstPage;
+
+        iv_firstPage = NULL;
+        iv_pagesAlloc = 0;
+
+        // Rewind to beginning of buffer.
+        int pageCnt = 0;
+        if (page)
+        {
+            pageCnt++;
+            while(page->prev) {
+                page = page->prev;
+                pageCnt++;
+            }
+        }
+
+        return page;
+    }
+
+    void Buffer::commitEntry(Entry* i_entry)
+    {
+        // Read the component from the entry itself (added as part of claiming).
+        ComponentDesc* l_comp = i_entry->comp;
+
+        Entry* l_savedNext = NULL;
+
+        // Lockless loop to update component linked-list.
+        do
+        {
+            // Update our entry's "next" pointer.
+            // Note: Our next pointer could change out from under us by the
+            // daemon's replaceEntry function, but the component iv_first
+            // cannot change by the daemon until we _producerExit, therefore
+            // we need to save the original next pointer for the atomic update
+            // of l_comp->iv_first below.
+            l_savedNext = i_entry->next = l_comp->iv_first;
+
+            // If there is an entry, update its "prev" pointer to this entry.
+            if (l_savedNext)
+            {
+                if (!__sync_bool_compare_and_swap(&l_savedNext->prev,
+                                                  NULL,
+                                                  i_entry))
+                {
+                    // Failed lockless update, try again.
+                    continue;
+                }
+            }
+            // Otherwise, we need to update the component list since this is
+            // the first entry.
+            else
+            {
+                if (!__sync_bool_compare_and_swap(&l_comp->iv_last,
+                                                  NULL,
+                                                  i_entry))
+                {
+                    // Failed lockless update, try again.
+                    continue;
+                }
+            }
+
+            // Successful at our updates, break out.
+            break;
+
+        } while(1);
+
+        // We just added the newest entry, so update component list.
+        l_comp->iv_first = i_entry;
+
+        // Atomically increment component size.
+        __sync_add_and_fetch(&l_comp->iv_curSize, i_entry->size);
+
+        // Mark entry as committed.
+        i_entry->committed = 1;
+    }
+
+    size_t Buffer::getTrace(ComponentDesc* i_comp, void* o_data, size_t i_size)
+    {
+        char* l_data = reinterpret_cast<char*>(o_data);
+        size_t l_size = 0;
+        size_t l_entries = 0;
+
+        // If either the pointer is null or the buffer is 0, we're just trying
+        // to determine the size of the buffer.
+        bool determineSize = ((o_data == NULL) || (i_size == 0));
+
+        if (determineSize)
+        {
+            i_size = UINT64_MAX;
+        }
+
+        trace_buf_head_t* header = NULL;
+
+        // If we're actually extracting, add the fsp-trace buffer header.
+        if(!determineSize)
+        {
+            if (i_size < sizeof(trace_buf_head_t))
+                return 0;
+
+            header = reinterpret_cast<trace_buf_head_t*>(&l_data[l_size]);
+            memset(header, '\0', sizeof(trace_buf_head_t));
+
+            header->ver = TRACE_BUF_VERSION;
+            header->hdr_len = sizeof(trace_buf_head_t);
+            header->time_flg = TRACE_TIME_REAL;
+            header->endian_flg = 'B'; // Big Endian.
+            memcpy(&header->comp[0], &i_comp->iv_compName, TRAC_COMP_SIZE);
+        }
+        l_size += sizeof(trace_buf_head_t);
+
+
+        size_t l_totalSize = l_size;
+        Entry* entry = i_comp->iv_first;
+        size_t l_entriesToExtract = 0;
+
+        do
+        {
+            if ((!entry) || (!entry->comp))
+            {
+                break;
+            }
+
+            // First walk the list backwards to find everything that will fit.
+            while(1)
+            {
+                // fsp-trace buffer entries have an extra word of size at the
+                // end.  That is where the sizeof(uint32_t) comes from...
+
+                if ((l_totalSize + entry->size + sizeof(uint32_t)) <= i_size)
+                {
+                    l_totalSize += entry->size + sizeof(uint32_t);
+                    l_entriesToExtract++;
+
+                    if ((entry->next) &&
+                        (entry->next->comp))
+                    {
+                        entry = entry->next;
+                        continue;
+                    }
+                }
+                else // This entry was too big to fit, so roll back one.
+                {
+                    entry = entry->prev;
+                }
+                break;
+            }
+
+            // If we didn't find anything that fit, leave.
+            if (l_totalSize == l_size)
+            {
+                break;
+            }
+
+            // If we're just trying to find the size, we're done.
+            if(determineSize)
+            {
+                l_size = l_totalSize;
+                break;
+            }
+
+            // Now we can actually copy all the entries...
+            while(entry != nullptr)
+            {
+                // Copy entry data.
+                memcpy(&l_data[l_size], &entry->data[0],entry->size);
+                l_size += entry->size;
+
+                // Copy entry size.
+                uint32_t entry_size = entry->size + sizeof(uint32_t);
+                memcpy(&l_data[l_size], &entry_size, sizeof(uint32_t));
+                l_size += sizeof(uint32_t);
+
+                l_entries++;
+
+                if (l_entries == l_entriesToExtract)
+                {
+                    break;
+                }
+                else
+                {
+                    entry = entry->prev;
+                }
+            };
+
+        }
+        while(0);
+
+        // Update header.
+        if (header)
+        {
+            header->size = l_size;
+            header->next_free = l_size;
+            header->te_count = l_entries;
+        }
+
+        return l_size;
+    }
+
+    bool Buffer::consumerOp(Entry** i_condAddr, Entry* i_condVal,
+                            Entry** i_condActAddr, Entry* i_condActVal,
+                            Entry** i_addr, Entry* i_val)
+    {
+        bool rc = true;
+
+        // Primitive #1.
+        if (NULL != i_condAddr)
+        {
+            if (i_condVal == *i_condAddr)
+            {
+                if (NULL != i_condActAddr)
+                {
+                    *i_condActAddr = i_condActVal;
+                }
+            }
+            else
+            {
+                rc = false;
+            }
+        }
+
+        // Primitive #2.
+        if (NULL != i_addr)
+        {
+            *i_addr = i_val;
+        }
+
+        return rc;
+    }
+
+}
diff --git a/src/usr/trace/runtime/rt_daemon.C b/src/usr/trace/runtime/rt_daemon.C
new file mode 100644
index 000000000..0e8300f95
--- /dev/null
+++ b/src/usr/trace/runtime/rt_daemon.C
@@ -0,0 +1,369 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/usr/trace/runtime/rt_daemon.C $                           */
+/*                                                                        */
+/* OpenPOWER HostBoot Project                                             */
+/*                                                                        */
+/* Contributors Listed Below - COPYRIGHT 2017                             */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+#include "../daemon/daemon.H"
+#include "../daemonif.H"
+#include "../service.H"
+#include "../buffer.H"
+#include "../bufferpage.H"
+#include "../entry.H"
+#include "../compdesc.H"
+#include "../debug.H"
+
+#include <errl/hberrltypes.H>
+#include <string.h>
+#include <util/align.H>
+
+
+namespace TRACEDAEMON
+{
+    using namespace TRACE;
+
+    Daemon::Daemon() : iv_service(nullptr), iv_totalPruned(0)
+    {
+        iv_first = iv_last = BufferPage::allocate(true);
+    }
+
+    Daemon::~Daemon()
+    {
+        assert(0, "No need to destruct the trace Daemon");
+    }
+
+    void Daemon::signal_trace_daemon(void)
+    {
+        // Put here to avoid build issue instead of in class constructor
+        // iv_service won't be used until these functions are called
+        iv_service = Service::getGlobalInstance();
+
+        // Collect trace entries from client.
+        collectTracePages();
+
+        // Reduce buffer space in daemon-side buffer.
+        pruneTraceEntries();
+
+        // combine trace buffer pages to remove pruned entries
+        coalescePages();
+    }
+
+
+    void Daemon::collectTracePages()
+    {
+        // Collect buffer pages from front-end.
+        BufferPage* srcPages[BUFFER_COUNT];
+        for (size_t i = 0; i < BUFFER_COUNT; i++)
+        {
+            if (iv_service->iv_buffers[i] == nullptr)
+            {
+                iv_curPages[i] = srcPages[i] = nullptr;
+            }
+            else
+            {
+                iv_curPages[i] = srcPages[i] =
+                    iv_service->iv_buffers[i]->claimPages();
+            }
+            iv_curOffset[i] = 0;
+        }
+
+        uint32_t entry_number = 0;
+        // Process buffer pages.
+        do
+        {
+            size_t whichBuffer = BUFFER_COUNT;
+            Entry* whichEntry = nullptr;
+            uint64_t minTimeStamp = UINT64_MAX;
+
+            // Find the entry with the earliest timestamp.
+            for (size_t i = 0; i < BUFFER_COUNT; i++)
+            {
+                if (nullptr == iv_curPages[i]) continue;
+
+                Entry* entry =
+                    reinterpret_cast<Entry*>(
+                        &((&(iv_curPages[i]->data[0]))[iv_curOffset[i]])
+                    );
+
+                trace_bin_entry_t* binEntry =
+                    reinterpret_cast<trace_bin_entry_t*>(
+                        &(entry->data[0])
+                    );
+
+
+                uint64_t curTimeStamp =
+                    TWO_UINT32_TO_UINT64(binEntry->stamp.tbh,
+                                         binEntry->stamp.tbl);
+
+                if (curTimeStamp < minTimeStamp )
+                {
+                    whichBuffer = i;
+                    whichEntry = entry;
+                    minTimeStamp = curTimeStamp;
+                }
+            }
+
+            // Did not find another entry, our work is done.
+            if (whichBuffer == BUFFER_COUNT)
+            {
+                break;
+            }
+
+            if ((whichEntry == nullptr) || (whichEntry->comp == nullptr))
+            {
+                break;
+            }
+
+            // Increment pointers to next buffer entry.
+            iv_curOffset[whichBuffer] += whichEntry->size + sizeof(Entry);
+            if (iv_curOffset[whichBuffer] >= iv_curPages[whichBuffer]->usedSize)
+            {
+                iv_curPages[whichBuffer] = iv_curPages[whichBuffer]->next;
+                iv_curOffset[whichBuffer] = 0;
+            }
+
+            // Allocate a new back-end entry.
+            Entry* mainBuffEntry = nullptr;
+            while (nullptr ==
+                (mainBuffEntry =
+                    iv_last->claimEntry(whichEntry->size + sizeof(Entry))))
+            {
+                BufferPage* n = BufferPage::allocate(true);
+
+                // Remember iv_last = Last newest combined-buffer page.
+                // Set n to the newest page, and push current iv_last back
+                n->next = iv_last;
+                iv_last->prev = n;
+                iv_last = n;
+                iv_last->prev = nullptr;
+            }
+
+            // Move entry from front-end buffer to back-end.
+            replaceEntry(whichEntry, mainBuffEntry);
+
+            entry_number++;
+        } while(1);
+
+        // Release pages.
+        for (size_t i = 0; i < BUFFER_COUNT; i++)
+        {
+            while(srcPages[i])
+            {
+                BufferPage* tmp = srcPages[i]->next;
+                BufferPage::deallocate(srcPages[i]);
+                srcPages[i] = tmp;
+            }
+        }
+    }
+
+
+    void Daemon::replaceEntry(Entry* from, Entry* to)
+    {
+        do
+        {
+            // Copy entry content to new entry.
+            memcpy(to,
+                   from,
+                   from->size + sizeof(Entry));
+
+            // Update next object's pointer.
+            if (to->next)
+            {
+                to->next->prev = to;
+            }
+            else
+            {
+                if (to->comp != nullptr)
+                {
+                    to->comp->iv_last = to;
+                }
+            }
+
+            // Update prev object's pointer.
+            //     Buffer ensures that an entry's "next" is written before
+            //     the "next->prev", so we can be certain that if to->prev
+            //     then to->prev->next is finalized.
+            if (to->prev)
+            {
+                to->prev->next = to;
+            }
+            else // If there is no previous, this is the first (most recent)
+                 // for the component, so update the component object.
+            {
+                if (to->comp != nullptr)
+                {
+                    Buffer* b =
+                        iv_service->iv_buffers[to->comp->iv_bufferType];
+
+                    // Need to toggle the consumer lock on this one, so use
+                    // the consumerOp to move the compoment->first from
+                    // 'from' to 'to'.
+                    //
+                    // If it fails (first != from anymore) then retry this
+                    // whole sequence.
+                    if (!b->consumerOp(&to->comp->iv_first, from,
+                                       &to->comp->iv_first, to))
+                    {
+                        continue;
+                    }
+                }
+            }
+
+            // Successfully updated everything, break from loop.
+            break;
+
+        } while (1);
+    }
+
+
+    void Daemon::pruneTraceEntries(bool i_all)
+    {
+        ComponentList::List::iterator component;
+
+        size_t pruned = 0;
+
+        // Iterate through the components...
+        bool more = iv_service->iv_compList->first(component);
+        while(more)
+        {
+            Entry* entry = component->iv_last;
+            Entry* orig_entry = entry;
+
+            // Invalidate entries until the component is small enough.
+            while((entry) &&
+                    ((component->iv_curSize > component->iv_maxSize) ||
+                     i_all)
+                 )
+            {
+                if (!reinterpret_cast<BufferPage*>(
+                        ALIGN_PAGE_DOWN(
+                            reinterpret_cast<uint64_t>(entry)))->commonPage)
+                {
+                    break;
+                }
+
+                entry->comp = nullptr; // Invalidate entry.
+
+                __sync_sub_and_fetch(&component->iv_curSize, entry->size);
+                pruned += entry->size;
+
+                entry = entry->prev;
+            }
+
+            if (entry != orig_entry)
+            {
+
+                // Break chain of linked list.
+                if (entry != nullptr)
+                {
+                    entry->next = nullptr;
+                }
+
+                // Update component pointers.
+                Buffer* b =
+                    iv_service->iv_buffers[component->iv_bufferType];
+
+                // consumerOp pseudo-code:
+                //    if (entry == nullptr) component->first = nullptr;
+                //    component->last = entry;
+                b->consumerOp(&entry, nullptr,
+                              &component->iv_first, nullptr,
+                              &component->iv_last, entry);
+            }
+
+            // Get next component.
+            more = iv_service->iv_compList->next(component);
+        }
+
+        // Record size of pruned entries in a global.
+        if (pruned)
+        {
+            iv_totalPruned += pruned;
+        }
+    }
+
+    void Daemon::coalescePages()
+    {
+        // Skip if there hasn't been enough entries pruned to make this
+        // worth our while.
+        if (iv_totalPruned < PAGESIZE) { return; }
+
+        iv_totalPruned = 0;
+
+        // Allocate a new back-end page for the coalesced entries.
+        BufferPage* outputPage = BufferPage::allocate(true);
+        BufferPage* originalOutputPage = outputPage;
+
+        // Get the first page from the original back-end buffer.
+        BufferPage* currentPage = iv_first;
+
+        // Iterate through the back-end pages.
+        while(currentPage != nullptr)
+        {
+            // Look at all the entries on the back-end pages.
+            size_t offset = 0;
+            while (offset < currentPage->usedSize)
+            {
+                Entry* entry =
+                    reinterpret_cast<Entry*>(&currentPage->data[offset]);
+
+                if (nullptr != entry->comp) // Ensure entry is valid.
+                {
+                    Entry* newEntry = nullptr;
+
+                    // Allocate space on new back-end pages.
+                    while (nullptr == (newEntry =
+                        outputPage->claimEntry(entry->size + sizeof(Entry))))
+                    {
+                        BufferPage* newPage = BufferPage::allocate(true);
+
+                        newPage->next = outputPage;
+                        outputPage->prev = newPage;
+                        outputPage = newPage;
+                    }
+
+                    // Move entry to new back-end page.
+                    replaceEntry(entry, newEntry);
+                }
+
+                offset += entry->size + sizeof(Entry);
+            }
+
+            currentPage = currentPage->prev;
+        }
+
+        BufferPage* oldPage = iv_first;
+
+        // Update back-end buffer pointers to point to new back-end pages.
+        iv_last = outputPage;
+        iv_first = originalOutputPage;
+
+        // Delete the old back-end pages.
+        while(oldPage)
+        {
+            BufferPage* temp = oldPage->prev;
+            BufferPage::deallocate(oldPage);
+            oldPage = temp;
+        }
+
+    }
+
+}
diff --git a/src/usr/trace/runtime/rt_service.C b/src/usr/trace/runtime/rt_service.C
index ecbb26f2a..476e6052d 100644
--- a/src/usr/trace/runtime/rt_service.C
+++ b/src/usr/trace/runtime/rt_service.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2013,2015                        */
+/* Contributors Listed Below - COPYRIGHT 2013,2017                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -24,13 +24,20 @@
 /* IBM_PROLOG_END_TAG                                                     */
 #include "../service.H"
 #include "../compdesc.H"
+#include "../buffer.H"
+#include "../entry.H"
 #include <limits.h>
 #include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <ctype.h>
 #include <string.h>
+#include <sys/time.h>
+#include <sys/task.h>
+#include <sys/sync.h>
+#include <util/align.H>
 #include <runtime/interface.h>
+#include <util/singleton.H>
 
 namespace TRACE
 {
@@ -39,12 +46,19 @@ namespace TRACE
 
     Service::Service()
     {
+        // Only use one "fake" daemon for runtime and one valid buffer
+        // BUFFER_FAST skips some blocking calls and is limited in pages
+        iv_daemon = new TRACEDAEMON::Daemon();
+        iv_buffers[BUFFER_SLOW] = nullptr;
+        iv_buffers[BUFFER_FAST] = new Buffer(iv_daemon);
+
+        iv_compList = &(Singleton<ComponentList>::instance());
     }
 
     Service::~Service()
     {
         // No need to destruct the service.
-        assert(0);
+        assert(0, "No need to destruct the Service");
     }
 
     void Service::writeEntry(ComponentDesc* i_td,
@@ -85,12 +99,155 @@ namespace TRACE
         {
             g_hostInterfaces->puts(output);
         }
+
+        //////////////////////////////////////////////
+        // Try to use buffers
+        //////////////////////////////////////////////
+        do
+        {
+            // Get the right buffer for this component.
+            Buffer* l_buffer = iv_buffers[i_td->iv_bufferType];
+
+            if (l_buffer == nullptr)
+            {
+                break;
+            }
+
+            // Copy the current time.
+            trace_entry_stamp_t l_time;
+            _createTimeStamp(&l_time);
+
+            // Sizes for trace entry.
+            uint32_t l_size = 0;
+            uint32_t l_num_args = 0;
+
+            // Maps of arguments to special types.
+            uint64_t l_str_map = 0, l_char_map = 0, l_double_map = 0;
+
+            va_list l_args;
+            va_copy(l_args, i_args);
+
+            // Parse the fmt list to determine the types/sizes of each
+            // argument.
+            while(NULL != (i_fmt = strchr(i_fmt, '%')))
+            {
+                i_fmt++;
+
+                switch (*i_fmt)
+                {
+                    case '%':
+                        break;
+
+                    case 's': // string.
+                        l_str_map |= (1 << l_num_args);
+                        l_num_args++;
+                        l_size += ALIGN_4(strlen(va_arg(l_args, char*)) + 1);
+                        break;
+
+                    case 'c': // character.
+                        l_char_map |= (1 << l_num_args);
+                        l_num_args++;
+                        va_arg(l_args, uint32_t);
+                        l_size += sizeof(uint32_t);
+                        break;
+
+                    case 'e': // doubles.
+                    case 'f':
+                    case 'g':
+                        l_double_map |= (1 << l_num_args);
+                        l_num_args++;
+                        va_arg(l_args, double);
+                        l_size += sizeof(double);
+                        break;
+
+                    default: // uint64_t-sized argument.
+                        l_num_args++;
+                        va_arg(l_args, uint64_t);
+                        l_size += sizeof(uint64_t);
+                        break;
+                }
+                i_fmt++;
+            }
+
+            va_end(l_args);
+
+            // Ensure we don't have too many arguments.
+            if (l_num_args > TRAC_MAX_ARGS)
+            {
+                // Simply reducing the number of arguments and continuing
+                // causes FSP trace to crash.  See defect 864438.
+                //l_num_args = TRAC_MAX_ARGS;
+                break;
+            }
+
+            // Claim an entry from the buffer.
+            uint32_t l_realSize = ALIGN_4(sizeof(trace_bin_entry_t) + l_size);
+            Entry* l_entry = l_buffer->claimEntry(i_td, l_realSize);
+            if (nullptr == l_entry)
+            {
+                break;
+            }
+
+            // Copy contents into the 'fsp-trace'-style structure.
+            trace_bin_entry_t* l_binEntry =
+                reinterpret_cast<trace_bin_entry_t*>(&l_entry->data[0]);
+
+            l_binEntry->stamp = l_time;
+            l_binEntry->head.length = l_size;
+            l_binEntry->head.tag = TRACE_COMP_TRACE;
+            l_binEntry->head.hash = i_hash;
+            l_binEntry->head.line = i_line;
+
+            // Copy arguments into the 'fsp-trace' entry's data section.
+            char* l_dataOut = reinterpret_cast<char*>(&l_binEntry->data[0]);
+            for (size_t i = 0; i < l_num_args; i++)
+            {
+                // String.
+                if (l_str_map & (1 << i))
+                {
+                    char* str = va_arg(i_args, char*);
+                    uint32_t strSize = strlen(str);
+
+                    memcpy(l_dataOut, str, strSize+1);
+
+                    l_dataOut += ALIGN_4(strSize + 1);
+                }
+                // Single character.
+                else if (l_char_map & (1 << i))
+                {
+                    *(reinterpret_cast<uint32_t*>(l_dataOut)) =
+                        va_arg(i_args, uint32_t);
+                    l_dataOut += sizeof(uint32_t);
+                }
+                // Doubles.
+                else if (l_double_map & (1 << i))
+                {
+                    *(reinterpret_cast<double*>(l_dataOut)) =
+                        va_arg(i_args, double);
+
+                    l_dataOut += sizeof(double);
+                }
+                // All others (as uint64_t's).
+                else
+                {
+                    *(reinterpret_cast<uint64_t*>(l_dataOut)) =
+                        va_arg(i_args, uint64_t);
+
+                    l_dataOut += sizeof(uint64_t);
+                }
+            }
+
+            // "Commit" entry to buffer.
+            l_buffer->commitEntry(l_entry);
+
+        } while(0);
     }
 
+
     void Service::writeBinEntry(ComponentDesc* i_td,
                                 trace_hash_val i_hash,
                                 const char * i_fmt,
-                                uint32_t i_ine,
+                                uint32_t i_line,
                                 const void* i_ptr,
                                 uint32_t i_size,
                                 uint32_t i_type)
@@ -189,14 +346,54 @@ namespace TRACE
             pos += 16;
         }
 
+        // Runtime traces
+        do
+        {
+            // Get the right buffer for this component.
+            Buffer* l_buffer = iv_buffers[i_td->iv_bufferType];
+            if(l_buffer == nullptr)
+            {
+                break;
+            }
+
+            // Copy the current time.
+            trace_entry_stamp_t l_time;
+            _createTimeStamp(&l_time);
+
+            // Claim an entry from the buffer.
+            uint32_t l_realSize = ALIGN_4(sizeof(trace_bin_entry_t) + i_size);
+            Entry* l_entry = l_buffer->claimEntry(i_td, l_realSize);
+            if (nullptr == l_entry)
+            {
+                break;
+            }
+
+            // Copy contents into the 'fsp-trace'-style structure.
+            trace_bin_entry_t* l_binEntry =
+                reinterpret_cast<trace_bin_entry_t*>(&l_entry->data[0]);
+
+            l_binEntry->stamp = l_time;
+            l_binEntry->head.length = i_size;
+            l_binEntry->head.tag = TRACE_FIELDBIN;
+            l_binEntry->head.hash = i_hash;
+            l_binEntry->head.line = i_line;
+
+            // Copy bin-data into the 'fsp-trace' entry's data section.
+            memcpy(&l_binEntry->data[0], i_ptr, i_size);
+
+            // "Commit" entry to buffer.
+            l_buffer->commitEntry(l_entry);
+
+        } while(0);
+
     }
 
     size_t Service::getBuffer(ComponentDesc* i_comp,
                               void * o_data,
                               size_t i_size)
     {
-        // No way to get the buffer back in runtime.
-        return 0;
+        return
+            iv_buffers[i_comp->iv_bufferType]->getTrace(i_comp,o_data,i_size);
     }
 
     void Service::flushBuffers()
@@ -204,4 +401,19 @@ namespace TRACE
         // No-op in runtime.
     }
 
+    void Service::_createTimeStamp(trace_entry_stamp_t *o_entry)
+    {
+        timespec_t curTime;
+
+        clock_gettime(CLOCK_MONOTONIC, &curTime);
+
+        o_entry->tbh = curTime.tv_sec;
+        o_entry->tbl = curTime.tv_nsec;
+        o_entry->tid = task_gettid();
+    }
+
+    Service* Service::getGlobalInstance()
+    {
+        return &(Singleton<Service>::instance());
+    }
 }
diff --git a/src/usr/trace/service.H b/src/usr/trace/service.H
index 001242530..afe2c18f7 100644
--- a/src/usr/trace/service.H
+++ b/src/usr/trace/service.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2012,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2012,2017                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -167,20 +167,28 @@ namespace TRACE
             friend class TRACEDAEMON::Daemon;
 
         private:
-#ifndef __HOSTBOOT_RUNTIME
                 /** Front-size buffers */
             Buffer* iv_buffers[BUFFER_COUNT]; // slow / fast buffers.
-                /** Interface to signal daemon. */
-            DaemonIf* iv_daemon;
-                /** List of component descriptors. */
-            ComponentList* iv_compList;
 
-                /** Get the singleton instance. */
-            static Service* getGlobalInstance();
                 /** Copy the current time into the timestamp. */
             void _createTimeStamp(trace_entry_stamp_t* o_entry);
+
+                /** Get the singleton instance. */
+
+            static Service* getGlobalInstance();
+
+                /** List of component descriptors. */
+            ComponentList* iv_compList;
+
+#ifndef __HOSTBOOT_RUNTIME
+                /** Interface to signal daemon. */
+            DaemonIf* iv_daemon;
+
                 /** tracelite enabled **/
             uint8_t iv_traceLite;
+#else
+                /** Runtime daemon **/
+            TRACEDAEMON::Daemon * iv_daemon;
 #endif
     };
 }