Complete virtual memory write path during shutdown sequence

Change-Id: I93a6305b88539d8cf1b41cfc4cde713fd7c19494
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/522
Tested-by: Jenkins Server
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

12 files changed, 196 insertions, 17 deletions

diff --git a/src/include/sys/misc.h b/src/include/sys/misc.h
index 8c90b2363..f30a408ca 100644
--- a/src/include/sys/misc.h
+++ b/src/include/sys/misc.h
@@ -46,11 +46,13 @@ extern "C"
 {
 #endif
 
+#ifdef __HIDDEN_SYSCALL_SHUTDOWN
 /** @fn shutdown()
  *  @brief Shutdown all CPUs (hardware threads)
  *  @param[in] i_status The status code to post
  */
-void shutdown(uint64_t i_status);
+extern "C" void shutdown(uint64_t i_status);
+#endif
 
 /** @enum ProcessorCoreType
  *  @brief Enumeration of the different supported processor cores.
diff --git a/src/include/usr/initservice/initserviceif.H b/src/include/usr/initservice/initserviceif.H
new file mode 100644
index 000000000..dabb81e68
--- /dev/null
+++ b/src/include/usr/initservice/initserviceif.H
@@ -0,0 +1,45 @@
+//  IBM_PROLOG_BEGIN_TAG
+//  This is an automatically generated prolog.
+//
+//  $Source: src/include/usr/initservice/initserviceif.H $
+//
+//  IBM CONFIDENTIAL
+//
+//  COPYRIGHT International Business Machines Corp. 2011
+//
+//  p1
+//
+//  Object Code Only (OCO) source materials
+//  Licensed Internal Code Source Materials
+//  IBM HostBoot Licensed Internal Code
+//
+//  The source code for this program is not published or other-
+//  wise divested of its trade secrets, irrespective of what has
+//  been deposited with the U.S. Copyright Office.
+//
+//  Origin: 30
+//
+//  IBM_PROLOG_END
+#ifndef __INITSERVICE_INITSERVICEIF_H
+#define __INITSERVICE_INITSERVICEIF_H
+
+#include <vmmconst.h>
+
+namespace INITSERVICE
+{
+
+/**
+ * @brief Register a block/range of vitual memory to be handled during a
+ *        shutdown.
+ *
+ * @param[in] i_vaddr - Base virtual address
+ * @param[in] i_size - Size of virtual memory from base address
+ * @param[in] i_priority - Order to handle given block(0-Lowest Priority)
+ *
+ * @return Nothing
+ */
+void registerBlock(void* i_vaddr, uint64_t i_size, BlockPriority i_priority);
+
+}
+
+#endif
diff --git a/src/include/usr/initservice/initsvcreasoncodes.H b/src/include/usr/initservice/initsvcreasoncodes.H
index 2f61cd460..1ed76efdd 100644
--- a/src/include/usr/initservice/initsvcreasoncodes.H
+++ b/src/include/usr/initservice/initsvcreasoncodes.H
@@ -72,6 +72,7 @@ enum    InitServiceModuleID
     EXTINITSVC_TASK_RETURNED_ERROR_ID   =   0x23,
     CXXTEST_TASK_RETURNED_ERROR_ID      =   0x24,
     ISTEP_RETURNED_ERROR_ID             =   0x25,
+    INITSVC_DO_SHUTDOWN_MOD_ID          =   0x26,
 
     //
     //  HWAS:  ISTEP 4
@@ -109,8 +110,7 @@ enum    InitServiceReasonCode
     INITSVC_FAILED_NO_ERRLOG    =   INITSVC_COMP_ID | 0x06,
     EXTINITSVC_FAILED_NO_ERRLOG =   INITSVC_COMP_ID | 0x07,
     CXXTEST_FAILED_NO_ERRLOG    =   INITSVC_COMP_ID | 0x08,
-
-
+    SHUTDOWN_FLUSH_FAILED       =   INITSVC_COMP_ID | 0x09,
 
 };
 
diff --git a/src/include/usr/vmmconst.h b/src/include/usr/vmmconst.h
index 51a933edf..f490f1301 100644
--- a/src/include/usr/vmmconst.h
+++ b/src/include/usr/vmmconst.h
@@ -57,6 +57,13 @@
 // image generator script 
 #define VMM_VADDR_ATTR_RP  (3ul * 1024ul * 1024ul * 1024ul)
 
+/** Virtual memory block priorities */
+enum BlockPriority
+{
+    PNOR_PRIORITY = 0, //No dependencies
+    VFS_PRIORITY  = (PNOR_PRIORITY + 1), //Dependent on PNOR
+    ATTR_PRIORITY = (PNOR_PRIORITY + 1), //Dependent on PNOR
+};
 
 /**
  * Other Constants
diff --git a/src/lib/syscall_misc.C b/src/lib/syscall_misc.C
index b5a44f80a..26ecc9bfe 100644
--- a/src/lib/syscall_misc.C
+++ b/src/lib/syscall_misc.C
@@ -20,9 +20,10 @@
 //  Origin: 30
 //
 //  IBM_PROLOG_END
+#define __HIDDEN_SYSCALL_SHUTDOWN
+
 #include <sys/misc.h>
 #include <sys/syscall.h>
-#include <sys/mmio.h>
 
 using namespace Systemcalls;
 
diff --git a/src/usr/initservice/baseinitsvc/initservice.C b/src/usr/initservice/baseinitsvc/initservice.C
index d9f1a630a..aeda3deba 100644
--- a/src/usr/initservice/baseinitsvc/initservice.C
+++ b/src/usr/initservice/baseinitsvc/initservice.C
@@ -28,6 +28,7 @@
  *  See initservice.H for details
  *
  */
+#define __HIDDEN_SYSCALL_SHUTDOWN
 
 #include    <kernel/console.H>
 #include    <sys/vfs.h>
@@ -38,6 +39,8 @@
 #include    <errl/errlentry.H>
 #include    <errl/errlmanager.H>
 #include    <sys/sync.h>
+#include    <sys/mm.h>
+#include    <vmmconst.h>
 
 #include    "initservice.H"
 #include    "initsvctasks.H"
@@ -324,8 +327,8 @@ void InitService::init( void *io_ptr )
         TRACFCOMP( g_trac_initsvc, "InitService: Committing errorlog." );
         errlCommit( l_errl, INITSVC_COMP_ID );
 
-        // Tell the kernel to shutdown.
-        shutdown( SHUTDOWN_STATUS_INITSVC_FAILED );
+        //Tell initservice to perform shutdown sequence
+        doShutdown( SHUTDOWN_STATUS_INITSVC_FAILED );
 
     }
 
@@ -349,4 +352,67 @@ InitService::InitService( )
 InitService::~InitService( )
 { }
 
+void registerBlock(void* i_vaddr, uint64_t i_size, BlockPriority i_priority)
+{
+    Singleton<InitService>::instance().registerBlock(i_vaddr,i_size,i_priority);
+}
+
+void InitService::registerBlock(void* i_vaddr, uint64_t i_size,
+                                BlockPriority i_priority)
+{
+    //Order priority from largest to smallest upon inserting
+    std::vector<regBlock_t*>::iterator regBlock_iter = iv_regBlock.begin();
+    for (; regBlock_iter!=iv_regBlock.end(); ++regBlock_iter)
+    {
+        if ((uint64_t)i_priority >= (*regBlock_iter)->priority)
+        {
+            iv_regBlock.insert(regBlock_iter,
+                               new regBlock_t(i_vaddr,i_size,
+                                              (uint64_t)i_priority));
+            regBlock_iter=iv_regBlock.begin();
+            break;
+        }
+    }
+    if (regBlock_iter == iv_regBlock.end())
+    {
+        iv_regBlock.push_back(new regBlock_t(i_vaddr,i_size,
+                                             (uint64_t)i_priority));
+    }
+}
+
+void InitService::doShutdown(uint64_t i_status)
+{
+    int l_rc = 0;
+    errlHndl_t l_err = NULL;
+    std::vector<regBlock_t*>::iterator l_rb_iter = iv_regBlock.begin();
+    //FLUSH each registered block in order
+    while (l_rb_iter!=iv_regBlock.end())
+    {
+        l_rc = mm_remove_pages(FLUSH,(*l_rb_iter)->vaddr,(*l_rb_iter)->size);
+        if (l_rc)
+        {
+            /*
+             * @errorlog tag
+             * @errortype       ERRL_SEV_CRITICAL_SYS_TERM
+             * @moduleid        INITSVC_DO_SHUTDOWN_MOD_ID
+             * @reasoncode      SHUTDOWN_FLUSH_FAILED
+             * @userdata1       returncode from mm_remove_pages()
+             * @userdata2       0
+             *
+             * @defdesc         Could not FLUSH virtual memory.
+             *
+             */
+            l_err = new ERRORLOG::ErrlEntry(
+                        ERRORLOG::ERRL_SEV_CRITICAL_SYS_TERM,
+                        INITSERVICE::INITSVC_DO_SHUTDOWN_MOD_ID,
+                        INITSERVICE::SHUTDOWN_FLUSH_FAILED,l_rc,0);
+            //Commit and attempt flushing other registered blocks
+            errlCommit( l_err, INITSVC_COMP_ID );
+            l_err = NULL;
+        }
+        l_rb_iter++;
+    }
+    shutdown(i_status);
+}
+
 } // namespace
diff --git a/src/usr/initservice/baseinitsvc/initservice.H b/src/usr/initservice/baseinitsvc/initservice.H
index a31d108c8..4d15ffeec 100644
--- a/src/usr/initservice/baseinitsvc/initservice.H
+++ b/src/usr/initservice/baseinitsvc/initservice.H
@@ -45,8 +45,8 @@
 #include <errl/errlentry.H>
 #include <initservice/initsvcreasoncodes.H>
 #include <initservice/taskargs.H>
-
 #include <initservice/initsvcstructs.H>
+#include <vmmconst.h>
 
 namespace   INITSERVICE
 {
@@ -181,6 +181,27 @@ public:
     errlHndl_t  dispatchTask( const TaskInfo    *i_ptask,
                               TaskArgs          *io_pargs ) const;
 
+    /**
+     * @brief Registry a block/range of vitual memory to be handled during a
+     *        shutdown.
+     *
+     * @param[in] i_vaddr - Base virtual address
+     * @param[in] i_size - Size of virtual memory from base address
+     * @param[in] i_priority - Order to handle given block(0-Lowest Priority)
+     *
+     * @return Nothing
+     */
+    void registerBlock(void* i_vaddr, uint64_t i_size, BlockPriority i_priority);
+
+    /**
+     * @brief Perform necessary steps, such as FLUSHing, to registered blocks.
+     *
+     * @param[in] i_status - Shutdown status to be passed along on shutdown
+     *
+     * @return Nothing
+     */
+    void doShutdown(uint64_t i_status);
+
 
 protected:
 
@@ -202,6 +223,30 @@ private:
     InitService(const InitService& i_right);
     InitService& operator=(const InitService& i_right);
 
+    /**
+     * @struct regBlock_t
+     * @brief Attributes stored for virtual memory ranges that must be handled
+     *        during a shutdown.
+     */
+    struct regBlock_t
+    {
+        //Base virtual address
+        void* vaddr;
+        //Size of virtual memory from base address
+        uint64_t size;
+        //Priority order in which to handle the given block
+        uint64_t priority;
+
+        /**
+         * @brief Constructor to initialize a registered block object
+         */
+        regBlock_t(void* i_vaddr, uint64_t i_size, uint64_t i_priority) :
+            vaddr(i_vaddr), size(i_size), priority(i_priority) {}
+    };
+
+    //Store a list of registered blocks
+    std::vector<regBlock_t*> iv_regBlock;
+
 };  //  class InitService
 
 } // namespace INITSERVICE
diff --git a/src/usr/initservice/extinitsvc/extinitsvc.C b/src/usr/initservice/extinitsvc/extinitsvc.C
index 342dad0e7..9e6402ff1 100644
--- a/src/usr/initservice/extinitsvc/extinitsvc.C
+++ b/src/usr/initservice/extinitsvc/extinitsvc.C
@@ -167,8 +167,9 @@ void ExtInitSvc::init( void *io_ptr )
 
         //  pass an error code to initsvc that we are shutting down.
         pTaskArgs->postReturnCode( TASKARGS_SHUTDOWN_RC );
-        // Tell the kernel to shutdown.
-        shutdown( SHUTDOWN_STATUS_EXTINITSVC_FAILED );
+        //Tell initservice to perform shutdown sequence
+        InitService::getTheInstance().doShutdown(
+                                      SHUTDOWN_STATUS_EXTINITSVC_FAILED);
     }
 
     TRACFCOMP( g_trac_initsvc,
@@ -294,7 +295,8 @@ void ExtInitSvc::init( void *io_ptr )
         l_shutdownStatus = SHUTDOWN_STATUS_UT_FAILED;
     }
 
-    shutdown(l_shutdownStatus);
+    //Tell initservice to perform shutdown sequence
+    InitService::getTheInstance().doShutdown(l_shutdownStatus);
 
     // return to _start(), which may end the task or die.
     return;
diff --git a/src/usr/initservice/istepdispatcher/istepdispatcher.C b/src/usr/initservice/istepdispatcher/istepdispatcher.C
index df3178f6f..ee5380c21 100644
--- a/src/usr/initservice/istepdispatcher/istepdispatcher.C
+++ b/src/usr/initservice/istepdispatcher/istepdispatcher.C
@@ -708,8 +708,8 @@ void    IStepDispatcher::runAllISteps( void * io_ptr )   const
 
         // pass an error code on to extinitsvc that we are shutting down.
         pTaskArgs->postReturnCode( TASKARGS_SHUTDOWN_RC );
-        // Tell the kernel to shutdown.
-        shutdown( SHUTDOWN_STATUS_ISTEP_FAILED );
+        //Tell initservice to perform shutdown sequence
+        InitService::getTheInstance().doShutdown( SHUTDOWN_STATUS_ISTEP_FAILED );
 
     }
 
diff --git a/src/usr/pnor/pnorrp.C b/src/usr/pnor/pnorrp.C
index 0aa4421e9..e39f49758 100644
--- a/src/usr/pnor/pnorrp.C
+++ b/src/usr/pnor/pnorrp.C
@@ -32,7 +32,7 @@
 #include <string.h>
 #include <sys/mm.h>
 #include <errno.h>
-
+#include <initservice/initserviceif.H>
 
 // Trace definition
 trace_desc_t* g_trac_pnor = NULL;
@@ -193,11 +193,13 @@ void PnorRP::initDaemon()
             break;
         }
 
+        //Register this memory range to be FLUSHed during a shutdown.
+        INITSERVICE::registerBlock(reinterpret_cast<void*>(BASE_VADDR),
+                                   TOTAL_SIZE,PNOR_PRIORITY);
 
         // Need to set permissions to R/W
-	rc = mm_set_permission((void*) BASE_VADDR,
-			       TOTAL_SIZE ,
-			       WRITABLE);
+        rc = mm_set_permission((void*) BASE_VADDR,TOTAL_SIZE,
+                               WRITABLE | WRITE_TRACKED);
 
 
 
diff --git a/src/usr/targeting/attrrp.C b/src/usr/targeting/attrrp.C
index 80ab93ab1..8c2d43731 100644
--- a/src/usr/targeting/attrrp.C
+++ b/src/usr/targeting/attrrp.C
@@ -27,10 +27,12 @@
 #include <errno.h>
 #include <string.h>
 #include <algorithm>
+#include <vmmconst.h>
 
 #include <targeting/targreasoncodes.H>
 #include "attrrp.H"
 #include "trace.H"
+#include <initservice/initserviceif.H>
 
 using namespace INITSERVICE;
 using namespace ERRORLOG;
@@ -392,6 +394,13 @@ namespace TARGETING
 
                     case SECTION_TYPE_PNOR_RW:
                         l_perm = WRITABLE | WRITE_TRACKED;
+                        /*
+                         * Register this memory range to be FLUSHed during
+                         * a shutdown.
+                         */
+                        INITSERVICE::registerBlock(
+                            reinterpret_cast<void*>(iv_sections[i].vmmAddress),
+                            iv_sections[i].size,ATTR_PRIORITY);
                         break;
 
                     case SECTION_TYPE_HEAP_PNOR_INIT:
diff --git a/src/usr/testcore/kernel/vmmpagetest.H b/src/usr/testcore/kernel/vmmpagetest.H
index 0edbd78f3..a6679cc81 100644
--- a/src/usr/testcore/kernel/vmmpagetest.H
+++ b/src/usr/testcore/kernel/vmmpagetest.H
@@ -204,7 +204,7 @@ class vmmpagetest: public CxxTest::TestSuite
            (*(volatile uint64_t *)(iv_va+4*PAGESIZE)) = 0x34343434;
 
 
-            printk("\n%lx\n", (*(volatile uint64_t *)(iv_va+4*PAGESIZE)));
+            printkd("\n%lx\n", (*(volatile uint64_t *)(iv_va+4*PAGESIZE)));
             //printkd(" Successfully read from a WRITABLE page\n");
 
          }