Need to disable interrupt sources before waiting for interrupts(MPIPL)

RTC:41868
Change-Id: I57891102fc0c96c156a9a69467c95645e4848703
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/3018
Tested-by: Jenkins Server
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

6 files changed, 621 insertions, 111 deletions

diff --git a/src/include/usr/intr/intr_reasoncodes.H b/src/include/usr/intr/intr_reasoncodes.H
index b336f8b14..ac0d3ca3f 100644
--- a/src/include/usr/intr/intr_reasoncodes.H
+++ b/src/include/usr/intr/intr_reasoncodes.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2012              */
+/* COPYRIGHT International Business Machines Corp. 2011,2013              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -46,6 +46,7 @@ namespace INTR
         RC_RP_NOT_INITIALIZED         = INTR_COMP_ID | 0x03,
         RC_BAD_VIRTUAL_IO_ADDRESS     = INTR_COMP_ID | 0x04,
         RC_BAD_ISN                    = INTR_COMP_ID | 0x05,
+        RC_PERSISTANT_INTERRUPTS      = INTR_COMP_ID | 0x06,
     };
 };
 
diff --git a/src/include/usr/targeting/common/targetservice.H b/src/include/usr/targeting/common/targetservice.H
index 2082ee149..416aeff5d 100644
--- a/src/include/usr/targeting/common/targetservice.H
+++ b/src/include/usr/targeting/common/targetservice.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2011,2012              */
+/* COPYRIGHT International Business Machines Corp. 2011,2013              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
diff --git a/src/usr/initservice/istepdispatcher/istepdispatcher.C b/src/usr/initservice/istepdispatcher/istepdispatcher.C
index 210a151b1..6f5eeb551 100644
--- a/src/usr/initservice/istepdispatcher/istepdispatcher.C
+++ b/src/usr/initservice/istepdispatcher/istepdispatcher.C
@@ -990,81 +990,19 @@ bool IStepDispatcher::spLess ( void )
 bool IStepDispatcher::checkMpiplMode( ) const
 {
     using namespace TARGETING;
+
     Target* l_pTopLevel = NULL;
     bool l_isMpiplMode = false;
-    TargetService& l_targetService = targetService();
 
+    TargetService& l_targetService = targetService();
     (void)l_targetService.getTopLevelTarget( l_pTopLevel );
 
-
-    if( l_pTopLevel == NULL )
-    {
-        TRACFCOMP( g_trac_initsvc,
-                   "Top level handle was NULL" );
-        l_isMpiplMode = false;
-    }
-    else
+    uint8_t is_mpipl = 0;
+    if(l_pTopLevel &&
+       l_pTopLevel->tryGetAttr<ATTR_IS_MPIPL_HB>(is_mpipl) &&
+       is_mpipl)
     {
-        TARGETING::Target* l_pMasterProcChip = NULL;
-        TARGETING::targetService().masterProcChipTargetHandle(l_pMasterProcChip);
-
-        if( l_pMasterProcChip != NULL )
-        {
-
-            // Setting the MPIPL attribute..
-            // read the MBOX scratch reg 2 and if bit 0 is on then set the
-            // MPIPL attribute to 1 and return l_isMPIPLMode = true.
-            // if bit 0 is not on then set the attribute to 0 and return
-            // l_isMPIPLMode = false
-
-            errlHndl_t l_errl = NULL;
-            size_t  op_size = sizeof( uint64_t );
-            uint64_t data;
-
-            // Scratch register 2 is defined as 0x00050039.. accessing directly
-            // to avoid confusion as the Literals set have 0x00050039 mapped to
-            // MBOX_SCRATCH1 which is confusing.
-            l_errl = DeviceFW::deviceRead(l_pMasterProcChip,
-                                          &(data),
-                                          op_size,
-                                          DEVICE_SCOM_ADDRESS(0x00050039));
-
-            if (l_errl)
-            {
-
-                TRACFCOMP( g_trac_initsvc,
-                           "Read of the Scratch Register failed" );
-
-                // set the MPIPL attribute to 0 (not mpipl)
-                l_pTopLevel->setAttr<ATTR_IS_MPIPL_HB>(0);
-
-                // commit errorlog
-                errlCommit( l_errl,
-                            INITSVC_COMP_ID );
-
-            }
-            else
-            {
-                // If bit 0 is on.. indicates we are in MPIPL
-                if (data & 0x8000000000000000)
-                {
-                    // set the MPIPL attribute to 1 
-                    l_pTopLevel->setAttr<ATTR_IS_MPIPL_HB>(1);
-
-                    // set isMpiplMode to true
-                    l_isMpiplMode = true;
-
-                    TRACFCOMP( g_trac_initsvc,
-                               "IN MPIPL..Updated ATTR_IS_MPIPL_HB to 1" );
-                }
-                else
-                {
-                    // set the MPIPL attribute to 0 (not mpipl)
-                    l_pTopLevel->setAttr<ATTR_IS_MPIPL_HB>(0);
-                }
-            }
-        }
-        
+        l_isMpiplMode = true;
     }
 
     return  l_isMpiplMode;
diff --git a/src/usr/intr/intrrp.C b/src/usr/intr/intrrp.C
index 321a2943d..b374c9f7d 100644
--- a/src/usr/intr/intrrp.C
+++ b/src/usr/intr/intrrp.C
@@ -50,6 +50,27 @@
 using namespace INTR;
 using namespace TARGETING;
 
+const uint32_t IntrRp::cv_PE_IRSN_COMP_SCOM_LIST[] =
+{
+    PE0_IRSN_COMP_SCOM_ADDR,
+    PE1_IRSN_COMP_SCOM_ADDR,
+    PE2_IRSN_COMP_SCOM_ADDR
+};
+
+const uint32_t IntrRp::cv_PE_IRSN_MASK_SCOM_LIST[] =
+{
+    PE0_IRSN_MASK_SCOM_ADDR,
+    PE1_IRSN_MASK_SCOM_ADDR,
+    PE2_IRSN_MASK_SCOM_ADDR
+};
+
+const uint32_t IntrRp::cv_PE_BAR_SCOM_LIST[] =
+{
+    PE0_BAREN_SCOM_ADDR,
+    PE1_BAREN_SCOM_ADDR,
+    PE2_BAREN_SCOM_ADDR
+};
+
 trace_desc_t * g_trac_intr = NULL;
 TRAC_INIT(&g_trac_intr, INTR_TRACE_NAME, KILOBYTE, TRACE::BUFFER_SLOW);
 
@@ -147,7 +168,7 @@ errlHndl_t IntrRp::_init()
     TARGETING::targetService().masterProcChipTargetHandle( procTarget );
 
     uint64_t barValue = 0;
-    procTarget->tryGetAttr<TARGETING::ATTR_INTP_BASE_ADDR>(barValue);
+    barValue = procTarget->getAttr<TARGETING::ATTR_INTP_BASE_ADDR>();
 
     // Mask off node & chip id to get base address
     uint64_t realAddr = barValue & ICPBAR_BASE_ADDRESS_MASK;
@@ -170,6 +191,22 @@ errlHndl_t IntrRp::_init()
 
     if(!err)
     {
+        uint8_t is_mpipl = 0;
+        TARGETING::Target * sys = NULL;
+        TARGETING::targetService().getTopLevelTarget(sys);
+        if(sys &&
+           sys->tryGetAttr<TARGETING::ATTR_IS_MPIPL_HB>(is_mpipl) &&
+           is_mpipl)
+        {
+            TRACFCOMP(g_trac_intr,"Disable interupts for MPIPL");
+            err = hw_disableIntrMpIpl();
+
+            if(err)
+            {
+                errlCommit(err,INTR_COMP_ID);
+                err = NULL;
+            }
+        }
 
         // Set up the interrupt provider registers
         // NOTE: It's only possible to set up the master core at this point.
@@ -186,7 +223,9 @@ errlHndl_t IntrRp::_init()
             // Skip threads that we shouldn't be starting
             if( !(en_threads & (0x8000000000000000>>thread)) )
             {
-                TRACDCOMP(g_trac_intr,"IntrRp::_init: Skipping thread %d : en_threads=%X",thread,en_threads);
+                TRACDCOMP(g_trac_intr,
+                          "IntrRp::_init: Skipping thread %d : en_threads=%X",
+                          thread,en_threads);
                 continue;
             } 
             pir.threadId = thread;
@@ -286,7 +325,9 @@ void IntrRp::msgHandler()
                     // Use the XISR as the type (for now)
                     type = static_cast<ext_intr_t>(xirr & XISR_MASK);
 
-                    TRACFCOMP(g_trac_intr,"External Interrupt recieved, Type=%x",type);
+                    TRACFCOMP(g_trac_intr,
+                              "External Interrupt recieved, Type=%x",
+                              type);
 
                     // Acknowlege msg
                     msg->data[1] = 0;
@@ -510,7 +551,7 @@ errlHndl_t IntrRp::setBAR(TARGETING::Target * i_target,
     errlHndl_t err = NULL;
 
     uint64_t barValue = 0;
-    i_target->tryGetAttr<TARGETING::ATTR_INTP_BASE_ADDR>(barValue);
+    barValue = i_target->getAttr<TARGETING::ATTR_INTP_BASE_ADDR>();
 
     barValue <<= 14;
     barValue |= 1ULL << (63 - ICPBAR_EN);
@@ -545,8 +586,8 @@ errlHndl_t IntrRp::initIRSCReg(TARGETING::Target * i_target)
         uint8_t chip = 0;
         uint8_t node = 0;
 
-        i_target->tryGetAttr<ATTR_FABRIC_NODE_ID>(node);
-        i_target->tryGetAttr<ATTR_FABRIC_CHIP_ID>(chip);
+        node = i_target->getAttr<ATTR_FABRIC_NODE_ID>();
+        chip = i_target->getAttr<ATTR_FABRIC_CHIP_ID>();
 
         size_t scom_len = sizeof(uint64_t);
 
@@ -771,8 +812,8 @@ errlHndl_t IntrRp::registerInterruptISN(msg_q_t i_msgQ,
         {
             uint8_t chip = 0;
             uint8_t node = 0;
-            (*target_itr)->tryGetAttr<ATTR_FABRIC_NODE_ID>(node);
-            (*target_itr)->tryGetAttr<ATTR_FABRIC_CHIP_ID>(chip);
+            node = (*target_itr)->getAttr<ATTR_FABRIC_NODE_ID>();
+            chip = (*target_itr)->getAttr<ATTR_FABRIC_CHIP_ID>();
 
             PIR_t pir(0);
             pir.nodeId = node;
@@ -845,8 +886,8 @@ msg_q_t IntrRp::unregisterInterruptISN(ISNvalue_t i_intr_type)
         {
             uint8_t chip = 0;
             uint8_t node = 0;
-            (*target_itr)->tryGetAttr<ATTR_FABRIC_NODE_ID>(node);
-            (*target_itr)->tryGetAttr<ATTR_FABRIC_CHIP_ID>(chip);
+            node = (*target_itr)->getAttr<ATTR_FABRIC_NODE_ID>();
+            chip = (*target_itr)->getAttr<ATTR_FABRIC_CHIP_ID>();
 
             PIR_t pir(0);
             pir.nodeId = node;
@@ -921,7 +962,7 @@ void IntrRp::initInterruptPresenter(const PIR_t i_pir) const
 
 
 
-void IntrRp::deconfigureInterruptPresenter(const PIR_t i_pir) const
+void IntrRp::disableInterruptPresenter(const PIR_t i_pir) const
 {
     uint64_t baseAddr = iv_baseAddr + cpuOffsetAddr(i_pir);
     uint8_t * cppr =
@@ -931,7 +972,7 @@ void IntrRp::deconfigureInterruptPresenter(const PIR_t i_pir) const
 
     // non- side effect xirr register
     uint32_t * xirrAddr =
-        reinterpret_cast<uint32_t *>(baseAddr) + XIRR_RO_OFFSET;
+        reinterpret_cast<uint32_t *>(baseAddr + XIRR_RO_OFFSET);
 
     uint32_t xirr = *xirrAddr & 0x00FFFFFF;
 
@@ -1090,13 +1131,414 @@ void IntrRp::shutDown()
                 continue;
             } 
             pir.threadId = thread;
-            deconfigureInterruptPresenter(pir);
+            disableInterruptPresenter(pir);
         }
     }
     TRACFCOMP(g_trac_intr,INFO_MRK,"INTR is shutdown");
 }
 
 //----------------------------------------------------------------------------
+
+errlHndl_t IntrRp::hw_disableRouting(TARGETING::Target * i_proc,
+                                     INTR_ROUTING_t i_rx_tx)
+{
+    errlHndl_t err = NULL;
+    do
+    {
+        size_t scom_len = sizeof(uint64_t);
+
+        // PSI
+        PSIHB_ISRN_REG_t reg;
+
+        err = deviceRead
+            (
+             i_proc,
+             &reg,
+             scom_len,
+             DEVICE_SCOM_ADDRESS(PSIHB_ISRN_REG_t::PSIHB_ISRN_REG)
+            );
+
+        if(err)
+        {
+            break;
+        }
+
+        switch(i_rx_tx)
+        {
+            case INTR_UPSTREAM:
+                reg.uie = 0;   //upstream interrupt enable = 0 (disable)
+                break;
+
+            case INTR_DOWNSTREAM:
+                reg.die = 0;  //downstream interrupt enable = 0 (disable)
+                break;
+        }
+
+        scom_len = sizeof(uint64_t);
+        err = deviceWrite
+            (
+             i_proc,
+             &reg,
+             scom_len,
+             DEVICE_SCOM_ADDRESS(PSIHB_ISRN_REG_t::PSIHB_ISRN_REG)
+            );
+
+        if(err)
+        {
+            break;
+        }
+
+        for(size_t i = 0;
+            i < sizeof(cv_PE_BAR_SCOM_LIST)/sizeof(cv_PE_BAR_SCOM_LIST[0]);
+            ++i)
+        {
+            uint64_t reg = 0;
+            scom_len = sizeof(uint64_t);
+            err = deviceRead
+                (
+                 i_proc,
+                 &reg,
+                 scom_len,
+                 DEVICE_SCOM_ADDRESS(cv_PE_BAR_SCOM_LIST[i])
+                );
+
+            if(err)
+            {
+                break;
+            }
+
+            switch(i_rx_tx)
+            {
+                case INTR_UPSTREAM:
+                    // reset bit PE_IRSN_RX
+                    reg &= ~((1ull << (63-PE_IRSN_RX)));
+                    break;
+
+                case INTR_DOWNSTREAM:
+                    // reset bit PE_IRSN_TX
+                    reg &= ~((1ull << (63-PE_IRSN_TX)));
+                    break;
+            }
+
+            scom_len = sizeof(uint64_t);
+            err = deviceWrite
+                (
+                 i_proc,
+                 &reg,
+                 scom_len,
+                 DEVICE_SCOM_ADDRESS(cv_PE_BAR_SCOM_LIST[i])
+                );
+
+            if(err)
+            {
+                break;
+            }
+        }
+        if(err)
+        {
+            break;
+        }
+
+        //NX has no up/down stream enable bit - just one enable bit.
+        //The NX should be cleared as part of an MPIPL so no
+        //interrupts should be pending from this unit.
+        if(i_rx_tx == INTR_UPSTREAM)
+        {
+            uint64_t reg = 0;
+            scom_len = sizeof(uint64_t);
+            err = deviceRead
+                (
+                 i_proc,
+                 &reg,
+                 scom_len,
+                 DEVICE_SCOM_ADDRESS(NX_BUID_SCOM_ADDR)
+                );
+            if(err)
+            {
+                break;
+            }
+
+            // reset bit NX_BUID_ENABLE
+            reg &= ~(1ull << (63-NX_BUID_ENABLE));
+
+            scom_len = sizeof(uint64_t);
+            err = deviceWrite
+                (
+                 i_proc,
+                 &reg,
+                 scom_len,
+                 DEVICE_SCOM_ADDRESS(NX_BUID_SCOM_ADDR)
+                );
+            if(err)
+            {
+                break;
+            }
+        }
+
+    } while(0);
+    return err;
+}
+
+//----------------------------------------------------------------------------
+
+errlHndl_t IntrRp::hw_resetIRSNregs(TARGETING::Target * i_proc)
+{
+    errlHndl_t err = NULL;
+    size_t scom_len = sizeof(uint64_t);
+    do
+    {
+        // PSI
+        PSIHB_ISRN_REG_t reg1; // zeros self
+        reg1.irsn -= 1;  // default all '1's according to scom spec
+        // all other fields = 0
+
+        err = deviceWrite
+            (
+             i_proc,
+             &reg1,
+             scom_len,
+             DEVICE_SCOM_ADDRESS(PSIHB_ISRN_REG_t::PSIHB_ISRN_REG)
+            );
+        if(err)
+        {
+            break;
+        }
+
+        // PE
+        for(size_t i = 0;
+            i < sizeof(cv_PE_BAR_SCOM_LIST)/sizeof(cv_PE_BAR_SCOM_LIST[0]);
+            ++i)
+        {
+            uint64_t reg = 0;
+            scom_len = sizeof(uint64_t);
+            // Note: no default value specified in scom spec - assume 0
+            err = deviceWrite
+                (
+                 i_proc,
+                 &reg,
+                 scom_len,
+                 DEVICE_SCOM_ADDRESS(cv_PE_IRSN_COMP_SCOM_LIST[i])
+                );
+            if(err)
+            {
+                break;
+            }
+
+            scom_len = sizeof(uint64_t);
+            // Note: no default value specified in scom spec - assume 0
+            err = deviceWrite
+                (
+                 i_proc,
+                 &reg,
+                 scom_len,
+                 DEVICE_SCOM_ADDRESS(cv_PE_IRSN_MASK_SCOM_LIST[i])
+                );
+            if(err)
+            {
+                break;
+            }
+        }
+        if(err)
+        {
+            break;
+        }
+
+        // NX [1:19] is BUID [20:32] mask
+        // No default value specified in scom spec. assume 0
+        uint64_t reg = 0;
+        scom_len = sizeof(uint64_t);
+        err = deviceWrite
+            (
+             i_proc,
+             &reg,
+             scom_len,
+             DEVICE_SCOM_ADDRESS(NX_BUID_SCOM_ADDR)
+            );
+        if(err)
+        {
+            break;
+        }
+    } while(0);
+    return err;
+}
+
+
+//----------------------------------------------------------------------------
+
+errlHndl_t IntrRp::hw_disableIntrMpIpl()
+{
+    errlHndl_t err = NULL;
+    do
+    {
+        TARGETING::TargetHandleList procChips;
+        getAllChips(procChips, TYPE_PROC);
+
+        TARGETING::TargetHandleList procCores;
+        getAllChiplets(procCores, TYPE_CORE);
+
+        // Disable upstream intr routing on all processor chips
+        for(TARGETING::TargetHandleList::iterator proc = procChips.begin();
+            proc != procChips.end();
+            ++proc)
+        {
+            // disable upstream intr routing
+            err = hw_disableRouting(*proc,INTR_UPSTREAM);
+            if(err)
+            {
+                break;
+            }
+        }
+        if(err)
+        {
+            break;
+        }
+
+        // Set interrupt presenter to allow all interrupts
+        for(TARGETING::TargetHandleList::iterator 
+            core = procCores.begin();
+            core != procCores.end();
+            ++core)
+        {
+            const TARGETING::Target * proc = getParentChip(*core);
+
+            FABRIC_CHIP_ID_ATTR chip = proc->getAttr<ATTR_FABRIC_CHIP_ID>();
+            FABRIC_NODE_ID_ATTR node = proc->getAttr<ATTR_FABRIC_NODE_ID>();
+            CHIP_UNIT_ATTR coreId = 
+                (*core)->getAttr<TARGETING::ATTR_CHIP_UNIT>();
+
+            PIR_t pir(0);
+            pir.nodeId = node;
+            pir.chipId = chip;
+            pir.coreId = coreId;
+
+            size_t threads = cpu_thread_count();
+            for(size_t thread = 0; thread < threads; ++thread)
+            {
+                pir.threadId = thread;
+                uint64_t cpprAddr=cpuOffsetAddr(pir)+iv_baseAddr+CPPR_OFFSET;
+                uint8_t *cppr = reinterpret_cast<uint8_t*>(cpprAddr);
+                *cppr = 0xff; // allow all interrupts
+            }
+        }
+
+        // Now look for interrupts
+        bool interrupt_found = false;
+        size_t retryCount = 10;
+
+        do
+        {
+            interrupt_found = false;
+            nanosleep(0,1000000);   // 1 ms
+
+            for(TARGETING::TargetHandleList::iterator 
+                core = procCores.begin();
+                core != procCores.end();
+                ++core)
+            {
+                const TARGETING::Target * proc = getParentChip(*core);
+
+                FABRIC_CHIP_ID_ATTR chip = 
+                    proc->getAttr<ATTR_FABRIC_CHIP_ID>();
+                FABRIC_NODE_ID_ATTR node = 
+                    proc->getAttr<ATTR_FABRIC_NODE_ID>();
+                CHIP_UNIT_ATTR coreId = 
+                    (*core)->getAttr<TARGETING::ATTR_CHIP_UNIT>();
+
+                PIR_t pir(0);
+                pir.nodeId = node;
+                pir.chipId = chip;
+                pir.coreId = coreId;
+
+                size_t threads = cpu_thread_count();
+                for(size_t thread = 0; thread < threads; ++thread)
+                {
+                    pir.threadId = thread;
+                    uint64_t xirrAddr = iv_baseAddr +
+                        cpuOffsetAddr(pir) + XIRR_RO_OFFSET;
+                    uint32_t * xirrPtr = 
+                        reinterpret_cast<uint32_t*>(xirrAddr);
+                    uint32_t xirr = *xirrPtr & 0x00FFFFFF;
+                    if(xirr) // pending interrupt
+                    {
+                        // Found pending interrupt!
+                        interrupt_found = true;
+
+                        TRACFCOMP(g_trac_intr,
+                                  ERR_MRK
+                                  "Pending interrupt found on MPIPL."
+                                  " CpuId:0x%x XIRR:0x%x",
+                                  pir.word,
+                                  xirr);
+                        // Signal EOI - read then write xirr value
+                        ++xirrPtr;        // move to RW XIRR reg
+                        volatile uint32_t xirr_rw = *xirrPtr;
+                        *xirrPtr = xirr_rw;
+                        --xirrPtr;      // back to RO XIRR reg
+                    }
+                }
+            }
+        } while(interrupt_found == true && --retryCount != 0);
+
+        if(interrupt_found && (retryCount == 0))
+        {
+            // traces above should identify stuck interrupt
+            INITSERVICE::doShutdown(INTR::RC_PERSISTANT_INTERRUPTS);
+        }
+
+        // Disable all interrupt presenters
+        for(TARGETING::TargetHandleList::iterator 
+            core = procCores.begin();
+            core != procCores.end();
+            ++core)
+        {
+            const TARGETING::Target * proc = getParentChip(*core);
+
+            FABRIC_CHIP_ID_ATTR  chip = proc->getAttr<ATTR_FABRIC_CHIP_ID>();
+            FABRIC_NODE_ID_ATTR node = proc->getAttr<ATTR_FABRIC_NODE_ID>();
+            CHIP_UNIT_ATTR coreId =
+                (*core)->getAttr<TARGETING::ATTR_CHIP_UNIT>();
+
+            PIR_t pir(0);
+            pir.nodeId = node;
+            pir.chipId = chip;
+            pir.coreId = coreId;
+
+            size_t threads = cpu_thread_count();
+            for(size_t thread = 0; thread < threads; ++thread)
+            {
+                pir.threadId = thread;
+                disableInterruptPresenter(pir);
+            }
+        }
+
+        // disable downstream routing and clean up IRSN regs
+        for(TARGETING::TargetHandleList::iterator proc = procChips.begin();
+            proc != procChips.end();
+            ++proc)
+        {
+            // disable downstream routing
+            err = hw_disableRouting(*proc,INTR_DOWNSTREAM);
+            if(err)
+            {
+                break;
+            }
+
+            // reset IRSN values
+            err = hw_resetIRSNregs(*proc);
+            if(err)
+            {
+                break;
+            }
+        }
+        if(err)
+        {
+            break;
+        }
+    } while(0);
+    return err;
+}
+
+//----------------------------------------------------------------------------
 // External interfaces
 //----------------------------------------------------------------------------
 
@@ -1308,4 +1750,3 @@ errlHndl_t INTR::enablePsiIntr(TARGETING::Target * i_target)
     return err;
 }
 
-
diff --git a/src/usr/intr/intrrp.H b/src/usr/intr/intrrp.H
index 884fc27fc..b8c2e9d82 100644
--- a/src/usr/intr/intrrp.H
+++ b/src/usr/intr/intrrp.H
@@ -149,7 +149,35 @@ namespace INTR
 
                 // MASK base ICP address
                 ICPBAR_BASE_ADDRESS_MASK = 0xFFFFFFFFFC000000ULL,
-           };
+
+
+                // The interrupt resource number ctr regs
+                // Used to enable/disable and control interrupt routing
+                NX_BUID_SCOM_ADDR       = 0x0201308E, //INTR CTRL for NX
+                NX_BUID_ENABLE          = 0,          //INTR Enable bit for NX
+
+                PE0_IRSN_COMP_SCOM_ADDR = 0x0201201A, //INTR IRSN compare
+                PE0_IRSN_MASK_SCOM_ADDR = 0x0201201B, //INTR IRSN mask
+                PE0_BAREN_SCOM_ADDR     = 0x02012045, //INTR enable/disable
+
+                PE1_IRSN_COMP_SCOM_ADDR = 0x0201241A, //INTR IRSN compare
+                PE1_IRSN_MASK_SCOM_ADDR = 0x0201241B, //INTR IRSN mask
+                PE1_BAREN_SCOM_ADDR     = 0x02012445, //INTR enable/disable
+
+                PE2_IRSN_COMP_SCOM_ADDR = 0x0201281A, //INTR IRSN compare
+                PE2_IRSN_MASK_SCOM_ADDR = 0x0201281B, //INTR IRSN mask
+                PE2_BAREN_SCOM_ADDR     = 0x02012845, //INTR enable/disable
+
+                // Bit pos in PEx_BAREN_SCOM register
+                PE_IRSN_RX = 3,         // upstream enable
+                PE_IRSN_TX = 4,         // downstream enable
+            };
+
+            enum INTR_ROUTING_t 
+            {
+                INTR_UPSTREAM,
+                INTR_DOWNSTREAM,
+            };
 
             // If the interrupt can't be handled by the current chip there are
             // three link registers used provide targets to forward the
@@ -299,6 +327,10 @@ namespace INTR
             typedef std::map<PIR_t, IPI_Info_t> IPI_Pending_t;
             IPI_Pending_t iv_ipisPending;  //!< Pending IPIs.
 
+            // PE regs
+            static const uint32_t cv_PE_IRSN_COMP_SCOM_LIST[]; //IRSN comp regs
+            static const uint32_t cv_PE_IRSN_MASK_SCOM_LIST[]; //IRSN mask regs
+            static const uint32_t cv_PE_BAR_SCOM_LIST[];       //IRSN enable
 
         private:        //functions
 
@@ -367,10 +399,10 @@ namespace INTR
             void initInterruptPresenter(const PIR_t i_pir) const;
 
             /**
-             * Deconfigure the interrupt presenter registers
+             * Disable the interrupt presenter
              * @param[in] i_pir, the PIR value for the presenter
              */
-            void deconfigureInterruptPresenter(const PIR_t i_pir) const;
+            void disableInterruptPresenter(const PIR_t i_pir) const;
 
             /**
              * Issue IPI to a thread.
@@ -410,6 +442,27 @@ namespace INTR
              */
             errlHndl_t maskXIVR(TARGETING::Target * i_target);
 
+            /*
+             * Disable HW from recieving interrupts
+             * @param i_proc  The processor chip target to disable
+             * @param i_rx_tx [INTR_UPSTREAM | INTR_DOWNSTREAM]
+             * @return an error handle on error
+             */
+            errlHndl_t hw_disableRouting(TARGETING::Target * i_proc,
+                                         INTR_ROUTING_t i_rx_tx);
+
+            /**
+             * Reset ISRN value in hardware regs
+             * @param i_proc  The target processor
+             * @return an error handle on error
+             */
+            errlHndl_t hw_resetIRSNregs(TARGETING::Target * i_proc);
+
+            /**
+             * disableHwMpIpl
+             */
+            errlHndl_t hw_disableIntrMpIpl();
+
             /**
              * Shutdown procedure
              */
diff --git a/src/usr/targeting/targetservicestart.C b/src/usr/targeting/targetservicestart.C
index d010f0e1e..dc252eed5 100644
--- a/src/usr/targeting/targetservicestart.C
+++ b/src/usr/targeting/targetservicestart.C
@@ -1,25 +1,25 @@
-//  IBM_PROLOG_BEGIN_TAG
-//  This is an automatically generated prolog.
-//
-//  $Source: src/usr/targeting/targetservicestart.C $
-//
-//  IBM CONFIDENTIAL
-//
-//  COPYRIGHT International Business Machines Corp. 2012
-//
-//  p1
-//
-//  Object Code Only (OCO) source materials
-//  Licensed Internal Code Source Materials
-//  IBM HostBoot Licensed Internal Code
-//
-//  The source code for this program is not published or other-
-//  wise divested of its trade secrets, irrespective of what has
-//  been deposited with the U.S. Copyright Office.
-//
-//  Origin: 30
-//
-//  IBM_PROLOG_END
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/usr/targeting/targetservicestart.C $                      */
+/*                                                                        */
+/* IBM CONFIDENTIAL                                                       */
+/*                                                                        */
+/* COPYRIGHT International Business Machines Corp. 2012,2013              */
+/*                                                                        */
+/* p1                                                                     */
+/*                                                                        */
+/* Object Code Only (OCO) source materials                                */
+/* Licensed Internal Code Source Materials                                */
+/* IBM HostBoot Licensed Internal Code                                    */
+/*                                                                        */
+/* The source code for this program is not published or otherwise         */
+/* divested of its trade secrets, irrespective of what has been           */
+/* deposited with the U.S. Copyright Office.                              */
+/*                                                                        */
+/* Origin: 30                                                             */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
 
 /**
  *  @file targeting/targetservicestart.C
@@ -45,6 +45,10 @@
 #include <targeting/common/targetservice.H>
 #include <targeting/attrrp.H>
 
+// Others
+#include <errl/errlentry.H>
+#include <devicefw/userif.H>
+
 //******************************************************************************
 // targetService
 //******************************************************************************
@@ -62,6 +66,11 @@ namespace TARGETING
 
 #define TARG_CLASS ""
 
+/*
+ * @brief Initialize any attributes that need to be set early on
+ */
+static void initializeAttributes(TargetService& i_targetService);
+
 /**
  *  @brief Entry point for initialization service to initialize the targeting
  *      code
@@ -84,11 +93,13 @@ static void initTargeting(errlHndl_t& io_pError)
     {
         TargetService& l_targetService = targetService();
         (void)l_targetService.init();
+
+        initializeAttributes(l_targetService);
     }
 
     TARG_EXIT();
 
-    #undef TARG_FN
+#undef TARG_FN
 }
 
 /**
@@ -97,6 +108,72 @@ static void initTargeting(errlHndl_t& io_pError)
  */
 TASK_ENTRY_MACRO(initTargeting);
 
+
+
+/*
+ * @brief Initialize any attributes that need to be set early on
+ */
+static void initializeAttributes(TargetService& i_targetService)
+{
+    #define TARG_FN "initializeAttributes()...)"
+    TARG_ENTER();
+
+    bool l_isMpipl = false;
+    Target* l_pTopLevel = NULL;
+
+    i_targetService.getTopLevelTarget(l_pTopLevel);
+    if(l_pTopLevel)
+    {
+        Target* l_pMasterProcChip = NULL;
+        i_targetService.masterProcChipTargetHandle(l_pMasterProcChip);
+
+        if(l_pMasterProcChip)
+        {
+            errlHndl_t l_errl = NULL;
+            size_t l_size = sizeof(uint64_t);
+            uint64_t l_data;
+
+            // Scratch register 2 is defined as 0x00050039.. accessing
+            // directly to avoid confusion as the Literals set have
+            // 0x00050039 mapped to MBOX_SCRATCH1 which is confusing.
+            l_errl = DeviceFW::deviceRead(l_pMasterProcChip,
+                                          &(l_data),
+                                          l_size,
+                                          DEVICE_SCOM_ADDRESS(0x00050039));
+
+            if(l_errl)
+            {
+                TARG_INF("Read of scratch register failed");
+                errlCommit(l_errl,TARG_COMP_ID);
+            }
+            else
+            {
+                // bit 0 on indicates MPIPL
+                if(l_data & 0x8000000000000000ull)
+                {
+                    l_isMpipl = true;
+                }
+            }
+        }
+
+        if(l_isMpipl)
+        {
+            l_pTopLevel->setAttr<ATTR_IS_MPIPL_HB>(1);
+        }
+        else
+        {
+            l_pTopLevel->setAttr<ATTR_IS_MPIPL_HB>(0);
+        }
+    }
+    else // top level is NULL - never expected
+    {
+        TARG_INF("Top level target is NULL");
+    }
+
+    TARG_EXIT();
+    #undef TARG_FN
+}
+
 #undef TARG_CLASS
 
 #undef TARG_NAMESPACE