Extract hb_image_exist mask on multinode MPIPL

CQ: SW248937
Change-Id: I5a7e901b7923cd489795c28dde00538a03ff6dc5
Reviewed-on: http://gfw160.aus.stglabs.ibm.com:8080/gerrit/8955
Reviewed-by: Douglas R. Gilbert <dgilbert@us.ibm.com>
Tested-by: Jenkins Server
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

5 files changed, 124 insertions, 3 deletions

diff --git a/src/include/usr/initservice/initsvcreasoncodes.H b/src/include/usr/initservice/initsvcreasoncodes.H
index fdf6d9358..3e1abc040 100644
--- a/src/include/usr/initservice/initsvcreasoncodes.H
+++ b/src/include/usr/initservice/initsvcreasoncodes.H
@@ -66,6 +66,7 @@ enum    InitServiceReasonCode
     ISTEP_FAILED_DUE_TO_DECONFIG =   INITSVC_COMP_ID | 0x0a,
     //termination_rc
     SHUTDOWN_REQUESTED_BY_FSP    =   INITSVC_COMP_ID | 0x0b,
+    ISTEP_INVALID_ON_MPIPL       =   INITSVC_COMP_ID | 0x0c,
 };
 
 enum InitServiceUserDetailDataSubSection
diff --git a/src/include/usr/intr/intr_reasoncodes.H b/src/include/usr/intr/intr_reasoncodes.H
index 3cdce388d..629f23637 100644
--- a/src/include/usr/intr/intr_reasoncodes.H
+++ b/src/include/usr/intr/intr_reasoncodes.H
@@ -41,6 +41,7 @@ namespace INTR
         MOD_INTR_SYNC_NODES          = 0x09, /**< intrrp.C : IntrRp::syncNodes() */
         MOD_INTR_SYNC_ADDNODE        = 0x0A, /**< intrrp.C : IntrRp::addHbNodeToMpiplSyncArea */
         MOD_INTR_ADDHBNODE           = 0x0B, /**< intrrp.C : INTR::addHbNode */
+        MOD_INTR_EXTRACTNODEINFO     = 0x0C, /**< intrrp.C : INTR::extractHbNodeInfo */
     };
 
     enum IntrReasonCode
diff --git a/src/usr/initservice/istepdispatcher/istepdispatcher.C b/src/usr/initservice/istepdispatcher/istepdispatcher.C
index 05e3f44b1..1f9fc1099 100644
--- a/src/usr/initservice/istepdispatcher/istepdispatcher.C
+++ b/src/usr/initservice/istepdispatcher/istepdispatcher.C
@@ -1212,6 +1212,39 @@ void IStepDispatcher::handleProcFabIovalidMsg(msg_t * & io_pMsg)
     errlHndl_t err = NULL;
     do
     {
+        //Intentionally fail this message on MPIPL
+        TARGETING::Target* l_pSys = NULL;
+        TARGETING::targetService().getTopLevelTarget(l_pSys);
+        bool l_mpiplMode = l_pSys->getAttr<TARGETING::ATTR_IS_MPIPL_HB>();
+        if(l_mpiplMode)
+        {
+            /*@
+             * @errortype
+             * @reasoncode       ISTEP_INVALID_ON_MPIPL
+             * @severity         ERRORLOG::ERRL_SEV_UNRECOVERABLE
+             * @moduleid         ISTEP_INITSVC_MOD_ID
+             * @userdata1[0:31]  MPIPL State
+             * @userdata1[32:63] N/A
+             * @userdata2[0:31]  N/A
+             * @userdata2[32:63] N/A.
+             * @devdesc          handleProcFabIovalidMsg called during MPIPL,
+                                 which is illegal.
+             */
+            err = new ERRORLOG::ErrlEntry(
+                                          ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                          ISTEP_INITSVC_MOD_ID,
+                                          ISTEP_INVALID_ON_MPIPL,
+                                          TWO_UINT32_TO_UINT64(l_mpiplMode,0x0),
+                                          0x0);
+
+            TRACFCOMP(g_trac_initsvc, "handleProcFabIovalidMsg: Not a valid on MPIPL, PLID = 0x%x",
+                      err->plid());
+
+            io_pMsg->data[0] = err->plid();
+            errlCommit(err, INITSVC_COMP_ID);
+            break;
+        }
+
         // Ensure the libraries needed are loaded
         err = VFS::module_load("libestablish_system_smp.so");
         if (err)
diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index 7b3ac859b..ed86cdfe6 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -454,6 +454,10 @@ void IntrRp::msgHandler()
                     xirr |= CPPR_MASK;  //set all CPPR bits - allow any INTR
                     *xirrAddress = xirr;
 
+                    TRACDCOMP(g_trac_intr,
+                              "EOI issued. XIRR=%x, PIR=%x",
+                              xirr,pir);
+
                     // Now handle any IPC messages
                     if (type == INTERPROC_XISR)
                     {
@@ -1713,7 +1717,7 @@ errlHndl_t IntrRp::blindIssueEOIs(TARGETING::Target * i_proc)
                 pir.threadId = thread;
                 uint64_t xirrAddr = iv_baseAddr +
                   cpuOffsetAddr(pir);
-                uint32_t * xirrPtr =
+                 uint32_t * xirrPtr =
                   reinterpret_cast<uint32_t*>(xirrAddr + XIRR_OFFSET);
                 uint8_t * mfrrPtr = reinterpret_cast<uint8_t*>(
                                                  xirrAddr + MFRR_OFFSET);
@@ -1879,7 +1883,7 @@ errlHndl_t IntrRp::findProcs_Cores(TARGETING::TargetHandleList & o_procs,
                                    DEVICE_FSI_ADDRESS(l_addr));
                 if (err)
                 {
-                    TRACFCOMP(g_trac_intr,"Failed to read CFAM 2838 on %x",
+                    TRACFCOMP(g_trac_intr,"Failed to read CFAM 2839 on %x",
                               TARGETING::get_huid(*proc));
                     break;
                 }
@@ -2011,7 +2015,7 @@ void IntrRp::drainMpIplInterrupts(TARGETING::TargetHandleList & i_cores)
                 pir.threadId = thread;
                 uint64_t xirrAddr = iv_baseAddr +
                   cpuOffsetAddr(pir) + XIRR_RO_OFFSET;
-                uint32_t * xirrPtr =
+                volatile uint32_t * xirrPtr =
                   reinterpret_cast<uint32_t*>(xirrAddr);
                 uint32_t xirr = *xirrPtr & 0x00FFFFFF;
                 TRACDCOMP(g_trac_intr,"   xirrPtr[%p] xirr[%x]\n", xirrPtr, xirr);
@@ -2068,6 +2072,13 @@ errlHndl_t IntrRp::hw_disableIntrMpIpl()
 
     do
     {
+        //extract the node layout for later
+        err = extractHbNodeInfo();
+        if(err)
+        {
+            break;
+        }
+
         //Get the procs/cores
         err = findProcs_Cores(funcProc, procCores);
         if(err)
@@ -2453,6 +2464,74 @@ errlHndl_t  IntrRp::addHbNodeToMpiplSyncArea(uint64_t i_hbNode)
     return err;
 }
 
+errlHndl_t  IntrRp::extractHbNodeInfo(void)
+{
+    errlHndl_t err = NULL;
+    uint64_t hrmorBase = KernelIpc::ipc_data_area.hrmor_base;
+    TARGETING::ATTR_HB_EXISTING_IMAGE_type hb_existing_image = 0;
+    void * node_info_ptr =
+        reinterpret_cast<void *>((iv_masterCpu.nodeId * hrmorBase) +
+                                 VMM_INTERNODE_PRESERVED_MEMORY_ADDR);
+
+    internode_info_t * this_node_info =
+        reinterpret_cast<internode_info_t *>
+        (mm_block_map(node_info_ptr,INTERNODE_INFO_SIZE));
+
+    if(this_node_info)
+    {
+        if(this_node_info->eye_catcher != NODE_INFO_EYE_CATCHER)
+        {
+            TRACFCOMP(g_trac_intr, INFO_MRK
+                      "MPIPL, but INTR node data sync area unintialized."
+                      " Assuming single HB Intance system");
+        }
+        else //multinode
+        {
+            TARGETING::ATTR_HB_EXISTING_IMAGE_type mask = 0x1 <<
+              (MAX_NODES_PER_SYS -1);
+
+            for(uint64_t node = 0; node < MAX_NODES_PER_SYS; ++node)
+            {
+                //If comm area indicates node exists, add to map
+                if(this_node_info->exist[node])
+                {
+                    hb_existing_image |= (mask >> node);
+                }
+            }
+        }
+
+        mm_block_unmap(this_node_info);
+    }
+    else
+    {
+        TRACFCOMP( g_trac_intr, "Failure calling mm_block_map : phys_addr=%p",
+                   node_info_ptr);
+        /*@
+         * @errortype    ERRL_SEV_UNRECOVERABLE
+         * @moduleid     INTR::MOD_INTR_EXTRACTNODEINFO
+         * @reasoncode   INTR::RC_CANNOT_MAP_MEMORY
+         * @userdata1    physical address
+         * @userdata2    Size
+         * @devdesc      Error mapping in memory
+         */
+        err = new ERRORLOG::ErrlEntry(
+                                      ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                      INTR::MOD_INTR_EXTRACTNODEINFO,
+                                      INTR::RC_CANNOT_MAP_MEMORY,
+                                      reinterpret_cast<uint64_t>(node_info_ptr),
+                                      INTERNODE_INFO_SIZE,
+                                      true /*Add HB Software Callout*/);
+
+    }
+
+    TARGETING::Target * sys = NULL;
+    TARGETING::targetService().getTopLevelTarget(sys);
+    sys->setAttr<TARGETING::ATTR_HB_EXISTING_IMAGE>(hb_existing_image);
+    TRACFCOMP( g_trac_intr, "extractHbNodeInfo found map: %x", hb_existing_image);
+
+    return err;
+}
+
 
 //----------------------------------------------------------------------------
 // External interfaces
diff --git a/src/usr/intr/intrrp.H b/src/usr/intr/intrrp.H
index b32cf5738..aff759f58 100644
--- a/src/usr/intr/intrrp.H
+++ b/src/usr/intr/intrrp.H
@@ -562,6 +562,13 @@ namespace INTR
             errlHndl_t  addHbNodeToMpiplSyncArea(uint64_t i_hbNode);
 
             /**
+             * Extract node information stored away and restore into
+             * ATTR for MPIPL
+             * @return error log handle
+             */
+            errlHndl_t  extractHbNodeInfo(void);
+
+            /**
              * Calculate the adress offset for the given cpu
              * @param[in] i_pir PIR value for the presenter
              * @return the offset