Don't Process Interrupts During Shutdown

An issue has come up on OpenPOWER where we would checkstop during
istep 10.2 or 16.1 when completing the interrupt processing due to
the code attempting to access a memory area that got locked during
the shutdown sequence. This change no-ops the complete interrupt
routine if a shutdown has been requested to remedy this issue.

Change-Id: Ie444149884c6a42a0010593fb94b91298b4bc798
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/66049
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>
Reviewed-by: Michael Baiocchi <mbaiocch@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

2 files changed, 42 insertions, 1 deletions

diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index 70747d88f..119b4d6c7 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -297,6 +297,8 @@ errlHndl_t IntrRp::_init()
               iv_masterCpu.groupId, iv_masterCpu.chipId, iv_masterCpu.coreId,
               iv_masterCpu.threadId);
 
+    iv_IntrRpShutdownRequested = false; // Not shutting down
+
     // Do the initialization steps on the master proc chip
     // The other proc chips will be setup at a later point
     TARGETING::Target* procTarget = NULL;
@@ -1924,6 +1926,15 @@ void IntrRp::completeInterruptProcessing(uint64_t& i_intSource, PIR_t& i_pir)
 
     do {
 
+        mutex_lock(&iv_intrRpMutex);
+        // Do not complete interrupt processing if IntrRp has been shut down.
+        // This prevents a race condition from happening where we try to access
+        // a memory region that's been locked as part of the shutdown procedure.
+        if(isIntrRpShutdown())
+        {
+            break;
+        }
+
         //Check if we found a matching proc handler for the interrupt to remove
         //  This is needed so the iNTRRP will honor new interrupts from this
         //  source
@@ -1986,6 +1997,8 @@ void IntrRp::completeInterruptProcessing(uint64_t& i_intSource, PIR_t& i_pir)
 
     } while(0);
 
+    mutex_unlock(&iv_intrRpMutex);
+
     return;
 }
 
@@ -2129,6 +2142,10 @@ void IntrRp::shutDown(uint64_t i_status)
         TRACFCOMP(g_trac_intr, "IntrRp::shutDown() Error masking all interrupt sources.");
     }
 
+    mutex_lock(&iv_intrRpMutex);
+
+    iv_IntrRpShutdownRequested = true;
+
     //Reset PSIHB Interrupt Space
     TRACFCOMP(g_trac_intr, "Reset PSIHB Interrupt Space");
 
@@ -2198,6 +2215,8 @@ void IntrRp::shutDown(uint64_t i_status)
         TRACFCOMP(g_trac_intr, "IntrRp::shutDown() Error disabling Master Interrupt BARs");
     }
 
+    mutex_unlock(&iv_intrRpMutex);
+
 #ifdef CONFIG_ENABLE_P9_IPI
     size_t threads = cpu_thread_count();
     uint64_t en_threads = get_enabled_threads();
@@ -2220,6 +2239,7 @@ void IntrRp::shutDown(uint64_t i_status)
         }
     }
 #endif
+
     TRACFCOMP(g_trac_intr,INFO_MRK"INTR is shutdown");
 }
 
@@ -3652,3 +3672,8 @@ void INTR::IntrRp::printEsbStates() const
         }
     }
 }
+
+bool INTR::IntrRp::isIntrRpShutdown() const
+{
+    return iv_IntrRpShutdownRequested;
+}
diff --git a/src/usr/intr/intrrp.H b/src/usr/intr/intrrp.H
index 8c102e286..2595ca105 100644
--- a/src/usr/intr/intrrp.H
+++ b/src/usr/intr/intrrp.H
@@ -33,6 +33,7 @@
 #include <sys/msg.h>
 #include <sys/misc.h>
 #include <sys/time.h>
+#include <sys/sync.h>
 #include <sys/internode.h>
 #include <intr/interrupt.H>
 #include <map>
@@ -91,6 +92,14 @@ namespace INTR
             ALWAYS_INLINE
             PIR_t intrDestCpuId() const { return iv_masterCpu; }
 
+            /**
+             * @brief Returns a boolean indicating whether the shutDown
+             *        has been initiated for IntrRp
+             * @return true IntrRp is shut down
+             *         false IntrRp is running
+             */
+            bool isIntrRpShutdown() const;
+
         protected:
 
             /**
@@ -99,7 +108,8 @@ namespace INTR
             IntrRp() :
                 iv_msgQ(NULL),
                 iv_baseAddr(0),
-                iv_masterCpu(0) {}
+                iv_masterCpu(0),
+                iv_IntrRpShutdownRequested(false) {}
 
             /**
              * Destructor
@@ -388,12 +398,18 @@ namespace INTR
             typedef std::pair<uint64_t, msg_t*> IPI_Info_t;
             typedef std::map<PIR_t, IPI_Info_t> IPI_Pending_t;
             IPI_Pending_t iv_ipisPending;  //!< Pending IPIs.
+            bool iv_IntrRpShutdownRequested; //!< Whether the shutdown of IntrRp
+                                             // has been requested.
 
             // PE regs
             static const uint32_t cv_PE_IRSN_COMP_SCOM_LIST[]; //IRSN comp regs
             static const uint32_t cv_PE_IRSN_MASK_SCOM_LIST[]; //IRSN mask regs
             static const uint32_t cv_PE_BAR_SCOM_LIST[];       //IRSN enable
 
+            // Mutex to prevent potential race conditions in IntRp shutdown and
+            // complete interrupt processing paths.
+            mutex_t iv_intrRpMutex;
+
         private:        //functions
 
             errlHndl_t _init();