PNOR ECC UE could cause HB VFS to crash/hang

Change-Id: I786930cf35f7b573576135c186e5b468d8066ddf
RTC: 102119
CQ: SW255041
Backport: release-fips810
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/10042
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>
Tested-by: Jenkins Server
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

2 files changed, 247 insertions, 15 deletions

diff --git a/src/usr/vfs/vfsrp.C b/src/usr/vfs/vfsrp.C
index 96290a757..8f2cc471c 100644
--- a/src/usr/vfs/vfsrp.C
+++ b/src/usr/vfs/vfsrp.C
@@ -39,6 +39,7 @@
 #include <pnor/pnorif.H>
 #include <util/align.H>
 #include <errl/errludstring.H>
+#include <errl/errlmanager.H>
 
 using namespace VFS;
 
@@ -87,24 +88,45 @@ void* VfsRp::msg_handler(void * unused)
 
 // ----------------------------------------------------------------------------
 
-void* VfsRp::load_unload(void * i_msg)
+void * VfsRp::vfsWatcher(void * unused)
+{
+    Singleton<VfsRp>::instance()._vfsWatcher();
+    return NULL;
+}
+
+// ----------------------------------------------------------------------------
+
+void* VfsRp::loadUnloadMonitored(void * i_msg)
 {
     task_detach();
-    Singleton<VfsRp>::instance()._load_unload((msg_t*)i_msg);
+    Singleton<VfsRp>::instance()._loadUnloadMonitored((msg_t*)i_msg);
     return NULL;
 }
 
 // ----------------------------------------------------------------------------
 
-void* VfsRp::exec(void * i_msg)
+void* VfsRp::loadUnload(void * i_msg)
+{
+    Singleton<VfsRp>::instance()._loadUnload((msg_t*)i_msg);
+    return NULL;
+}
+
+// ----------------------------------------------------------------------------
+
+void* VfsRp::execMonitored(void * i_msg)
 {
     task_detach();
-    Singleton<VfsRp>::instance()._exec((msg_t*)i_msg);
+    Singleton<VfsRp>::instance()._execMonitored((msg_t*)i_msg);
     return NULL;
 }
 
 // ----------------------------------------------------------------------------
 
+void* VfsRp::exec(void * i_msg)
+{
+    Singleton<VfsRp>::instance()._exec((msg_t*)i_msg);
+    return NULL;
+}
 /**
  * Initialze the vfs resource provider
  */
@@ -132,10 +154,10 @@ errlHndl_t VfsRp::_init()
         {
             // Note: permissions are set elsewhere
 
-            // Start msg_handler
+            // Start msg_handler task watcher
             //  NOTE: This would be a weak consistancy issues if
             //  task_create were not a system call.
-            task_create(VfsRp::msg_handler, NULL);
+            task_create(VfsRp::vfsWatcher, NULL);
         }
         else
         {
@@ -167,10 +189,80 @@ errlHndl_t VfsRp::_init()
 
 // ----------------------------------------------------------------------------
 
+void VfsRp::_vfsWatcher()
+{
+
+    while(1)
+    {
+        tid_t tid = task_create(VfsRp::msg_handler, NULL);
+        assert( tid > 0 );
+
+        int childsts = 0;
+        void * childRc = NULL;
+
+        // The msg_handler will only return if there is a problem
+        tid_t tidRc = task_wait_tid( tid,
+                                     &childsts,
+                                     &childRc);
+
+        TRACFCOMP(g_trac_vfs, ERR_MRK
+                  "VFS msg_handler crashed. tid:%d status:%d childRc:%p. "
+                  "Restarting VFS...",
+                  tid, childsts, childRc);
+
+        /*@ errorlog tag
+         * @errortype       ERRL_SEV_CRITICAL_SYS_TERM
+         * @moduleid        VFS_WATCHER
+         * @reasoncode      VFS_TASK_CRASHED
+         * @userdata1       tidRc
+         * @userdata2       task Rc
+         *
+         * @devdesc         VFS RP Task crashed.
+         *
+         */
+        errlHndl_t err = new ERRORLOG::ErrlEntry
+            (
+             ERRORLOG::ERRL_SEV_CRITICAL_SYS_TERM, // severity
+             VFS::VFS_WATCHER,                     // moduleid
+             VFS::VFS_TASK_CRASHED,                // reason
+             (uint64_t)tidRc,                      // tid rc
+             (uint64_t)childRc,                    // child rc
+             true
+            );
+
+        // Message is only saved in iv_msg for messages from the kernel
+        if(iv_msg != NULL)
+        {
+            if(childRc == NULL) // Critical error not yet reported
+            {
+                ERRORLOG::errlCommit(err, VFS_COMP_ID);
+            }
+            else // Crit error already generated
+            {
+                delete err;
+            }
+
+            iv_msg->data[1] = -EIO;  /* I/O error */
+            iv_msg->extra_data = childRc;  /* Critical code from PNOR */
+            msg_respond(iv_msgQ, iv_msg);
+            iv_msg = NULL;
+        }
+        else //nothing to respond to - just commit error and restar task
+        {
+            ERRORLOG::errlCommit(err, VFS_COMP_ID);
+        }
+
+        // Loop and restart the vfs msg_handler task
+    }
+}
+
+// ----------------------------------------------------------------------------
+
 void VfsRp::msgHandler()
 {
     while(1)
     {
+        iv_msg = NULL;
         msg_t* msg = msg_wait(iv_msgQ);
 
         switch(msg->type)
@@ -181,7 +273,7 @@ void VfsRp::msgHandler()
                               (const char *) msg->data[0]);
 
                     // run in own task so page faults can be handled
-                    task_create(load_unload, msg);
+                    task_create(loadUnloadMonitored, msg);
                 }
                 break;
 
@@ -191,7 +283,7 @@ void VfsRp::msgHandler()
                               (const char *) msg->data[0]);
 
                     // run in own task so page faults can be handled
-                    task_create(load_unload, msg);
+                    task_create(loadUnloadMonitored, msg);
 
                 }
                 break;
@@ -202,13 +294,14 @@ void VfsRp::msgHandler()
                            (const char*)((msg_t*)msg->data[0])->data[0]);
 
                     // run in own task so page faults can be handled
-                    task_create(exec, msg);
+                    task_create(execMonitored, msg);
 
                 }
                 break;
 
             case MSG_MM_RP_READ:
                 {
+                    iv_msg = msg; // save message in case task crashes
                     uint64_t vaddr = msg->data[0]; //page aligned
                     uint64_t paddr = msg->data[1]; //page aligned
 
@@ -243,7 +336,59 @@ void VfsRp::msgHandler()
 
 // ----------------------------------------------------------------------------
 
-void VfsRp::_load_unload(msg_t * i_msg)
+void VfsRp::_loadUnloadMonitored(msg_t * i_msg)
+{
+    tid_t tid = task_create(VfsRp::loadUnload, i_msg);
+    assert( tid > 0 );
+
+    int childsts = 0;
+    void * childRc = NULL;
+
+    tid_t tidRc = task_wait_tid( tid,
+                                 &childsts,
+                                 &childRc);
+
+
+    if(childsts == TASK_STATUS_CRASHED)
+    {
+        TRACFCOMP(g_trac_vfs, ERR_MRK
+                  "VFS load/unload crashed. tid:%d status:%d childRc:%p",
+                  tid, childsts, childRc);
+
+        /*@ errorlog tag
+         * @errortype       ERRL_SEV_CRITICAL_SYS_TERM
+         * @moduleid        VFS_MODULE_LOAD_MONITOR
+         * @reasoncode      VFS_TASK_CRASHED
+         * @userdata1       tidRc
+         * @userdata2       Task Status
+         *
+         * @devdesc         VFS Task crashed.
+         *
+         */
+        errlHndl_t err = new ERRORLOG::ErrlEntry
+            (
+             ERRORLOG::ERRL_SEV_CRITICAL_SYS_TERM,     // severity
+             VFS::VFS_MODULE_LOAD_MONITOR,             // moduleid
+             VFS::VFS_TASK_CRASHED,                    // reason Code
+             (uint64_t)tidRc,                          // tid rc
+             (uint64_t)childsts,                       // task status
+             true
+            );
+
+        if(childRc != NULL) // crit elog aleady generated
+        {
+            err->setSev(ERRORLOG::ERRL_SEV_INFORMATIONAL);
+        }
+
+        // tell caller that load/unload failed
+        i_msg->data[0] = reinterpret_cast<uint64_t>(err);
+        msg_respond(iv_msgQ, i_msg);
+    }
+}
+
+// ----------------------------------------------------------------------------
+
+void VfsRp::_loadUnload(msg_t * i_msg)
 {
     errlHndl_t err = NULL;
 
@@ -378,6 +523,57 @@ void VfsRp::_load_unload(msg_t * i_msg)
 
 // ----------------------------------------------------------------------------
 
+void VfsRp::_execMonitored(msg_t * i_msg)
+{
+    tid_t tid = task_create(VfsRp::exec, i_msg);
+    assert( tid > 0 );
+
+    int childsts = 0;
+    void * childRc = NULL;
+
+    tid_t tidRc = task_wait_tid( tid,
+                                 &childsts,
+                                 &childRc);
+
+    if(childsts == TASK_STATUS_CRASHED)
+    {
+        TRACFCOMP(g_trac_vfs, ERR_MRK
+                  "VFS exec crashed. tid:%d status:%d childRc:%p",
+                  tid, childsts, childRc);
+
+        /*@ errorlog tag
+         * @errortype       ERRL_SEV_CRITICAL_SYS_TERM
+         * @moduleid        VFS_MODULE_EXEC_MONITOR
+         * @reasoncode      VFS_TASK_CRASHED
+         * @userdata1       tidRc
+         * @userdata2       task Rc
+         *
+         * @devdesc         VFS Task crashed.
+         *
+         */
+        errlHndl_t err = new ERRORLOG::ErrlEntry
+            (
+             ERRORLOG::ERRL_SEV_CRITICAL_SYS_TERM,     // severity
+             VFS::VFS_MODULE_EXEC_MONITOR,             // moduleid
+             VFS::VFS_TASK_CRASHED,                    // reason Code
+             (uint64_t)tidRc,                          // tid rc
+             (uint64_t)childRc,                        // child rc
+             true
+            );
+
+        if(childRc != NULL) // crit elog aleady generated
+        {
+            err->setSev(ERRORLOG::ERRL_SEV_INFORMATIONAL);
+        }
+
+        // tell caller that exec/startup failed
+        i_msg->data[0] = reinterpret_cast<uint64_t>(err);
+        msg_respond(iv_msgQ, i_msg);
+    }
+}
+
+// ----------------------------------------------------------------------------
+
 void VfsRp::_exec(msg_t * i_msg)
 {
     msg_t * msg1 = (msg_t *) i_msg->data[0];
diff --git a/src/usr/vfs/vfsrp.H b/src/usr/vfs/vfsrp.H
index 014aad815..febf35c4a 100644
--- a/src/usr/vfs/vfsrp.H
+++ b/src/usr/vfs/vfsrp.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2012              */
+/* COPYRIGHT International Business Machines Corp. 2011,2014              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -88,7 +88,7 @@ namespace VFS
             /**
              * Ctor
              */
-            VfsRp() : iv_msgQ(NULL), iv_pnor_vaddr(0)
+            VfsRp() : iv_msgQ(NULL), iv_msg(NULL), iv_pnor_vaddr(0)
             {
                 mutex_init(&iv_mutex);
             }
@@ -104,19 +104,37 @@ namespace VFS
             static void* msg_handler(void * unused);
 
             /**
+             * Load or Unload a module. Monitor for crash
+             * @param[in] i_msg the message
+             * @note Does not return.
+             */
+            static void* loadUnloadMonitored(void * i_msg);
+
+            /**
              * Load or Unload a module
              * @param[in] i_msg the message
+             */
+            static void* loadUnload(void * i_msg);
+
+            /**
+             * Call _start on module. Montior for crash
+             * @param[in] i_msg the message
              * @note Does not return. 
              */
-            static void* load_unload(void * i_msg);
+            static void* execMonitored(void * i_msg);
 
             /**
              * Call _start on module
              * @param[in] i_msg the message
-             * @note Does not return. 
              */
             static void* exec(void * i_msg);
 
+            /**
+             * Start the VFS resource provider message handler
+             * and watch for crashes.
+             */
+            static void* vfsWatcher(void * unused);
+
         private: // functions
 
             /**
@@ -131,10 +149,16 @@ namespace VFS
             void msgHandler();
 
             /**
+             * Load or Unload a module. Monitor for task crash
+             * @parm[in] The message
+             */
+            void _loadUnloadMonitored(msg_t * i_msg);
+
+            /**
              * Load or Unload a module
              * @param[in] i_msg the message
              */
-            void _load_unload(msg_t * i_msg);
+            void _loadUnload(msg_t * i_msg);
 
             /**
              * Call _start on module
@@ -142,9 +166,21 @@ namespace VFS
              */
             void _exec(msg_t * i_msg);
 
+            /**
+             * Call _start on Module. Montor for task crash
+             * @param[in] The message
+             */
+            void _execMonitored(msg_t * i_msg);
+
+            /**
+             * Message Handler task watcher
+             */
+            void _vfsWatcher();
+
         private: // data
 
             msg_q_t iv_msgQ;         //!< message queue
+            msg_t*  iv_msg;          //!< Current message being handled
             uint64_t iv_pnor_vaddr;  //!< virtual address of ext image in PFNOR
 
             typedef std::vector<VfsSystemModule *> ModuleList_t;