Debug improvements for exceptions and OOM hangs

There are two main changes in this commit:
1) Forcing an assert if we cannot allocate pages after
10,000 attempts to yield.
2) Adding a backtrace for a lot of exception paths.

Change-Id: I755ada753b78abed56e553f7c669f0f98ae68700
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/58224
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Prachi Gupta <pragupta@us.ibm.com>
Reviewed-by: Thi N. Tran <thi@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>

8 files changed, 109 insertions, 16 deletions

diff --git a/src/include/arch/ppc.H b/src/include/arch/ppc.H
index f63b5db2f..053661e1d 100644
--- a/src/include/arch/ppc.H
+++ b/src/include/arch/ppc.H
@@ -434,10 +434,11 @@ ALWAYS_INLINE
 inline void MAGIC_INSTRUCTION(int _n)
 {
     register int n = _n;
+    isync();
     asm volatile("rlwimi %0,%0,0,%1,%2"       \
         :: "i" (((n) >> 8) & 0x1f),           \
         "i" (((n) >> 4) & 0xf),               \
-        "i" ((((n) >> 0) & 0xf) | 16));
+        "i" ((((n) >> 0) & 0xf) | 16));       \
 }
 
 // Arguments to MAGIC_INSTRUCTION().
diff --git a/src/include/kernel/misc.H b/src/include/kernel/misc.H
index 4c570423a..26fde6528 100644
--- a/src/include/kernel/misc.H
+++ b/src/include/kernel/misc.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2011,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2011,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -178,7 +178,7 @@ namespace KernelMisc
      */
     void populate_cache_lines(uint64_t* i_start, uint64_t* i_end);
 
-    /** @fn set
+    /** @fn set scratch reg
      *
      *  @brief Update value of scratch register to specified data.
      *
@@ -195,5 +195,17 @@ namespace KernelMisc
      */
     void updateScratchReg(MMIO_Scratch_Register scratch_addr, uint64_t data);
 
+    /** @fn Log backtrace to printk
+     *
+     *  @brief Collect the backtrace for the given task and print an
+     *         abbreviated version to the printk buffer.
+     *
+     *  This can be used in task-crash paths for FFDC
+     *
+     *  @param[in] task_t i_task - Task to operate against,
+     *             if i_task==nullptr then call will use user-space interfaces
+     */
+    void printkBacktrace(task_t* i_task);
+
 };
 #endif
diff --git a/src/kernel/exception.C b/src/kernel/exception.C
index cf2a35c81..ca05bf3a1 100644
--- a/src/kernel/exception.C
+++ b/src/kernel/exception.C
@@ -37,6 +37,7 @@
 #include <kernel/terminate.H>
 #include <kernel/hbterminatetypes.H>
 #include <kernel/kernel_reasoncodes.H>
+#include <kernel/misc.H>
 
 
 namespace ExceptionHandles
@@ -62,7 +63,8 @@ void kernel_execute_prog_ex()
     }
     if (!handled)
     {
-        printk("Program exception, killing task %d\n", t->tid);
+        printk( "Program exception, killing task %d, SRR0=0x%lX, SRR1=0x%lX\n",
+                t->tid, getSRR0(), getSRR1() );
         MAGIC_INSTRUCTION(MAGIC_BREAK_ON_ERROR);
         TaskManager::endTask(t, NULL, TASK_STATUS_CRASHED);
     }
@@ -107,7 +109,7 @@ void kernel_execute_data_storage()
                "Exception Type: %lx\n"
                "Instruction where it occurred: %p\n",
                t->tid, getDAR(), getDSISR(), t->context.nip);
-        MAGIC_INSTRUCTION(MAGIC_BREAK_ON_ERROR);
+        KernelMisc::printkBacktrace(t);
         TaskManager::endTask(t, NULL, TASK_STATUS_CRASHED);
     }
 }
diff --git a/src/kernel/machchk.C b/src/kernel/machchk.C
index 2a96b5896..776ede0ce 100644
--- a/src/kernel/machchk.C
+++ b/src/kernel/machchk.C
@@ -149,7 +149,7 @@ void setCheckstopData(uint64_t i_xstopAddr, uint64_t i_xstopData)
     g_xstopRegPtr = reinterpret_cast<uint64_t*>(i_xstopAddr
                                                 |VmmManager::FORCE_PHYS_ADDR);
     g_xstopRegValue = i_xstopData;
-    printk( "Set MchChk Xstop: %p=%.16lX\n", g_xstopRegPtr, g_xstopRegValue );
+    printk( "Arm MchChk Xstop: %p=%.16lX\n", g_xstopRegPtr, g_xstopRegValue );
 
     // Now that the machine check handler can do the xscom we
     //  can set MSR[ME]=1 to enable the regular machine check
diff --git a/src/kernel/misc.C b/src/kernel/misc.C
index b602ed707..f6aa69bda 100644
--- a/src/kernel/misc.C
+++ b/src/kernel/misc.C
@@ -585,6 +585,51 @@ namespace KernelMisc
         writeScratchReg(l_scratch_addr, data);
     };
 
+    /**
+     *  @brief Collect the backtrace for the given task and print an
+     */
+    void printkBacktrace(task_t* i_task)
+    {
+        uint64_t* l_frame = nullptr;
+        uint32_t l_tid = 0;
+        bool l_kernelSpace = true;
+        if( i_task == nullptr ) //user-space
+        {
+            l_kernelSpace = false;
+            printk("U:");
+            l_frame = static_cast<uint64_t*>(framePointer());
+            l_tid = task_gettid();
+        }
+        else //kernel-space
+        {
+            printk("K:");
+            l_frame = reinterpret_cast<uint64_t*>( i_task->context.gprs[1] );
+            l_tid = i_task->tid;
+        }
+
+        printk("Backtrace for %d:\n  ", l_tid );
+        printkd("frame=%p\n",l_frame);isync();
+        while (l_frame != NULL)
+        {
+            printkd("\nf=%p\n",l_frame); isync();
+            if( l_kernelSpace )
+            {
+                uint64_t* frame_p = reinterpret_cast<uint64_t*>
+                  (VmmManager::findPhysicalAddress( reinterpret_cast<uint64_t>
+                                                    (l_frame) ));
+                printkd("frame_p=%p\n",frame_p); isync();
+                l_frame = frame_p;
+            }
+            if( (0 != *l_frame) && (0 != l_frame[2]) )
+            {
+                printk( "<-0x%lX", l_frame[2] );
+            }
+
+            l_frame = reinterpret_cast<uint64_t*>(*l_frame);
+        }
+        printk("\n");
+    }
+
 
 };
 
diff --git a/src/kernel/pagemgr.C b/src/kernel/pagemgr.C
index f5c4d406a..42545470a 100644
--- a/src/kernel/pagemgr.C
+++ b/src/kernel/pagemgr.C
@@ -37,6 +37,7 @@
 #include <assert.h>
 #include <kernel/memstate.H>
 #include <kernel/bltohbdatamgr.H>
+#include <kernel/misc.H>
 
 
 size_t PageManager::cv_coalesce_count = 0;
@@ -155,7 +156,7 @@ void* PageManager::allocatePage(size_t n, bool userspace)
     // In non-kernel mode, make a system-call to allocate in kernel-mode.
     if (!KernelMisc::in_kernel_mode())
     {
-        size_t attempts = 0;
+        size_t l_attempts = 0;
         while (NULL == page)
         {
             page = _syscall1(Systemcalls::MM_ALLOC_PAGES,
@@ -165,11 +166,14 @@ void* PageManager::allocatePage(size_t n, bool userspace)
             // will eventually free up (ex. VMM flushes).
             if (NULL == page)
             {
-                attempts++;
-                if( attempts == 10000 ) //arbitrarily huge number
+                l_attempts++;
+                if( l_attempts == 10000 )
                 {
-                    printk("Cannot allocate %ld pages\n", n);
+                    printk( "Cannot allocate %ld pages to %d!\n",
+                            n, task_gettid() );
                     MAGIC_INSTRUCTION(MAGIC_BREAK_ON_ERROR);
+                    KernelMisc::printkBacktrace(nullptr);
+                    task_crash();
                 }
                 task_yield();
             }
diff --git a/src/usr/testcore/kernel/taskwaittest.H b/src/usr/testcore/kernel/taskwaittest.H
index 2d4a39772..a929dcd9a 100644
--- a/src/usr/testcore/kernel/taskwaittest.H
+++ b/src/usr/testcore/kernel/taskwaittest.H
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2014              */
+/* Contributors Listed Below - COPYRIGHT 2011,2018                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -168,7 +170,7 @@ class TaskWaitTest : public CxxTest::TestSuite
 
         static void* TaskThatCrashes(void* unused)
         {
-            printk("Test case: Expect to see uncaught exception! ");
+            printk("TaskThatCrashes: Expect to see uncaught exception! ");
             *(int64_t*)(0) = 0xDEADC0DE;
             return NULL;
         }
diff --git a/src/usr/testcore/kernel/vmmbasetest.H b/src/usr/testcore/kernel/vmmbasetest.H
index 5a3c0d0f8..c4d42116e 100644
--- a/src/usr/testcore/kernel/vmmbasetest.H
+++ b/src/usr/testcore/kernel/vmmbasetest.H
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2014              */
+/* Contributors Listed Below - COPYRIGHT 2011,2018                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -44,7 +46,7 @@ class VmmBaseTest : public CxxTest::TestSuite
         {
             int status;
 
-            printk("Test case: Expect to see uncaught exception! ");
+            printk("testNullAccess1: Expect to see uncaught exception! ");
             tid_t child = task_create(readFromNULL, NULL);
 
             if ((child != task_wait_tid(child, &status, NULL)) ||
@@ -53,7 +55,7 @@ class VmmBaseTest : public CxxTest::TestSuite
                 TS_FAIL("Write to NULL not caught.");
             }
 
-            printk("Test case: Expect to see uncaught exception! ");
+            printk("testNullAccess2: Expect to see uncaught exception! ");
             child = task_create(writeToNULL, NULL);
             if ((child != task_wait_tid(child, &status, NULL)) ||
                 (status != TASK_STATUS_CRASHED))
@@ -66,7 +68,7 @@ class VmmBaseTest : public CxxTest::TestSuite
         {
             int status;
 
-            printk("Test case: Expect to see uncaught exception! ");
+            printk("testWriteToKernelCode: Expect to see uncaught exception! ");
             tid_t child = task_create(writeToKernelCode, NULL);
             if ((child != task_wait_tid(child, &status, NULL)) ||
                 (status != TASK_STATUS_CRASHED))
@@ -134,6 +136,21 @@ class VmmBaseTest : public CxxTest::TestSuite
             }
         }
 
+        void testHugeMalloc()
+        {
+            int status;
+
+            printk("testHugeMalloc: Expect to see uncaught exception! ");isync();
+
+            tid_t child = task_create(bigMalloc, NULL);
+
+            if ((child != task_wait_tid(child, &status, NULL)) ||
+                (status != TASK_STATUS_CRASHED))
+            {
+                TS_FAIL("testHugeMalloc> Giant malloc didn't crash.");
+            }
+        }
+
     private:
 
         static void* readFromNULL(void* unused)
@@ -180,6 +197,16 @@ class VmmBaseTest : public CxxTest::TestSuite
             return NULL;
         }
 
+        static void* bigMalloc(void* unused)
+        {
+            uint8_t* ptr = (uint8_t*)malloc(VMM_MEMORY_SIZE);
+            for( size_t x=0; x<(VMM_MEMORY_SIZE); x+=MEGABYTE )
+            {
+                ptr[x] = x;
+            }
+            return NULL;
+        }
+
 };
 msg_q_t VmmBaseTest::iv_mq = msg_q_create();