Enforce synchronized processor security state

Ensure the secure state is consistent across all processors.

Change-Id: I671253b99d5d87509909681a2cec2380ca6918b0
RTC:167775
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/38764
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Michael Baiocchi <mbaiocch@us.ibm.com>
Reviewed-by: Stephen M. Cprek <smcprek@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

8 files changed, 754 insertions, 62 deletions

diff --git a/src/include/usr/isteps/istep_reasoncodes.H b/src/include/usr/isteps/istep_reasoncodes.H
index 5f9e22718..167b9dc1c 100644
--- a/src/include/usr/isteps/istep_reasoncodes.H
+++ b/src/include/usr/isteps/istep_reasoncodes.H
@@ -95,6 +95,14 @@ namespace ISTEP
         RC_NO_FUNCTIONAL_MEMORY                  = ISTEP_COMP_ID | 0x1E,
         RC_NO_FUSED_CORE_TARGET                  = ISTEP_COMP_ID | 0x1F,
         RC_POWER_MODE_MISMATCH                   = ISTEP_COMP_ID | 0x20,
+        RC_MASTER_PROC_SBE_KEYS_HASH_MISMATCH    = ISTEP_COMP_ID | 0x21,
+        RC_MASTER_PROC_PRIMARY_HASH_READ_FAIL    = ISTEP_COMP_ID | 0x22,
+        RC_MASTER_PROC_BACKUP_HASH_READ_FAIL     = ISTEP_COMP_ID | 0x23,
+        RC_MASTER_PROC_CBS_CONTROL_READ_FAIL     = ISTEP_COMP_ID | 0x24,
+        RC_SLAVE_PROC_PRIMARY_HASH_READ_FAIL     = ISTEP_COMP_ID | 0x25,
+        RC_SLAVE_PROC_BACKUP_HASH_READ_FAIL      = ISTEP_COMP_ID | 0x26,
+        RC_SLAVE_PROC_CBS_CONTROL_READ_FAIL      = ISTEP_COMP_ID | 0x27,
+        RC_PROC_SECURITY_STATE_MISMATCH          = ISTEP_COMP_ID | 0x28,
     };
 
 };
diff --git a/src/include/usr/secureboot/secure_reasoncodes.H b/src/include/usr/secureboot/secure_reasoncodes.H
index 1e7cb797f..57b668a6e 100644
--- a/src/include/usr/secureboot/secure_reasoncodes.H
+++ b/src/include/usr/secureboot/secure_reasoncodes.H
@@ -39,6 +39,7 @@ namespace SECUREBOOT
         MOD_SECURE_ROM_SHA512           = 0x05,
         MOD_SECURE_READ_REG             = 0x06,
         MOD_SECURE_WRITE_REG            = 0x07,
+        MOD_SECURE_SETTINGS_INIT        = 0x08,
    };
 
     enum SECUREReasonCode
@@ -52,6 +53,7 @@ namespace SECUREBOOT
         RC_ROM_VERIFY                   = SECURE_COMP_ID | 0x07,
         RC_ROM_SHA512                   = SECURE_COMP_ID | 0x08,
         RC_SECURE_BAD_TARGET            = SECURE_COMP_ID | 0x09,
+        RC_SECURE_BOOT_DISABLED         = SECURE_COMP_ID | 0x0A,
 
         // Reason codes 0xA0 - 0xEF reserved for trustedboot_reasoncodes.H
     };
diff --git a/src/include/usr/secureboot/service.H b/src/include/usr/secureboot/service.H
index 88d50283b..c8c094fdf 100644
--- a/src/include/usr/secureboot/service.H
+++ b/src/include/usr/secureboot/service.H
@@ -98,14 +98,31 @@ namespace SECUREBOOT
      *      call was successful. If not successful this value is set to
      *      zero. Check the return value for a non null error log to
      *      determine if the call was unsuccessful.
-     *  @param[in] i_targ The target processor to obtain the jumper
+     *  @param[in] i_pProc The target processor to obtain the jumper
      *      state from. Must not be null. Optional parameter that
      *      defaults to master processor.
      *  @return errlHndl_t indicating whether the query was successful.
      *  @retval null if successful otherwise pointer to error log
      */
     errlHndl_t getSecuritySwitch(uint64_t& o_regValue,
-            TARGETING::Target* i_targ
+            TARGETING::Target* i_pProc
+                = TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL);
+
+    /** @brief Get Processor CBS Control register value
+     *  @par Detailed Description:
+     *      Returns the state of the Processor CBS Control register as
+     *      reported by the given processor (via the supplied target
+     *      pointer).
+     *  @param[out] o_regValue The value read from the register if the
+     *      call was successful. If not successful this value is set to
+     *      zero. Check the return value for a non null error log to
+     *      determine if the call was unsuccessful.
+     *  @param[in] i_pProc The target processor to obtain the jumper
+     *      state from. Must not be null. Optional parameter that
+     *      defaults to the master processor sentinel.
+     */
+    errlHndl_t getProcCbsControlRegister(uint64_t& o_regValue,
+            TARGETING::Target* i_pProc
                 = TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL);
 
     /**
@@ -171,7 +188,7 @@ namespace SECUREBOOT
      *      HW can be overridden by some functions.  Use the getEnabled()
      *      API to determine whether security is actually enabled.
      *      Deasserted means the jumper is configured to disble HW security.
-     *  @param[in] i_targ The target processor to obtain the jumper
+     *  @param[in] i_pProc The target processor to obtain the jumper
      *      state from. Must not be null. Optional parameter that
      *      defaults to master processor.
      *
@@ -179,7 +196,7 @@ namespace SECUREBOOT
      *  @retval null if successful otherwise pointer to error log.
      */
     errlHndl_t getJumperState(SecureJumperState& o_state,
-                TARGETING::Target* i_targ
+                TARGETING::Target* i_pProc
                     = TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL);
 
     /**
@@ -240,6 +257,20 @@ namespace SECUREBOOT
     void handleSecurebootFailure(
         errlHndl_t &io_err, bool i_waitForShutdown = true);
 
+    /**
+     *  @brief Adds the values of the Security Registers of the processors in
+     *         the system to an existing error log
+     *
+     *  @param[in/out] io_err  Error Log that the values of the security
+     *                         registers will be added to. Must not be nullptr.
+     *                         NOTE:  The state of the system/processors
+     *                         (ie, SCOM vs FSI) determines which registers can
+     *                         be included.
+     *
+     *  @return N/A
+     */
+    void addSecurityRegistersToErrlog(errlHndl_t & io_err);
+
 }
 
 #endif
diff --git a/src/include/usr/secureboot/settings.H b/src/include/usr/secureboot/settings.H
index 1de694948..417e14d96 100644
--- a/src/include/usr/secureboot/settings.H
+++ b/src/include/usr/secureboot/settings.H
@@ -73,6 +73,16 @@ namespace SECUREBOOT
         StatusRegisterFsi = 0x00002804ull, // used for FSI operation
     };
 
+    inline bool operator|(const ProcCbsControl i_cbs, const uint64_t i_value)
+    {
+        return (static_cast<uint64_t>(i_cbs) | i_value);
+    }
+
+    inline bool operator&(const ProcCbsControl i_cbs, const uint64_t i_value)
+    {
+        return (static_cast<uint64_t>(i_cbs) & i_value);
+    }
+
     /** @class Settings
      *
      *  @brief Caches and parses the hardware settings for Secureboot.
@@ -90,7 +100,14 @@ namespace SECUREBOOT
              *         in Secureboot's service.H for documentation
              */
             errlHndl_t getSecuritySwitch(uint64_t& o_regValue,
-                TARGETING::Target* i_targ
+                TARGETING::Target* i_pProc
+                    = TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL) const;
+
+            /** @brief Get Processor CBS Control regeister value. See wrapper
+             *         in Secureboot's service.H for documentation
+             */
+            errlHndl_t getProcCbsControlRegister(uint64_t& o_regValue,
+                TARGETING::Target* i_pProc
                     = TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL) const;
 
             /**
@@ -115,7 +132,7 @@ namespace SECUREBOOT
              *      for documenation.
              */
             errlHndl_t getJumperState(SecureJumperState& o_state,
-                TARGETING::Target* i_targ
+                TARGETING::Target* i_pProc
                     = TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL) const;
 
         private:
@@ -127,7 +144,7 @@ namespace SECUREBOOT
              *          Reads a register at a given scom address and
              *          provides its result via the supplied register
              *          value reference. Returns an error if unsuccesful.
-             *  @param  [in] i_targ The target processor to obtain the
+             *  @param  [in] i_pProc The target processor to obtain the
              *          jumper state from. Must not be null.
              *  @param  [in] i_scomAddress A uint64_t corresponding to
              *          desired scomAddress to read.
@@ -139,7 +156,7 @@ namespace SECUREBOOT
              *          successful.
              *  @retval null if successful otherwise pointer to error log
              */
-            errlHndl_t readSecurityRegister(TARGETING::Target* i_targ,
+            errlHndl_t readSecurityRegister(TARGETING::Target* i_pProc,
                                            const uint64_t i_scomAddress,
                                             uint64_t& o_regValue) const;
 
diff --git a/src/usr/errl/errlentry.C b/src/usr/errl/errlentry.C
index 398a1a70b..4f9fe4246 100644
--- a/src/usr/errl/errlentry.C
+++ b/src/usr/errl/errlentry.C
@@ -54,7 +54,7 @@
 #include <initservice/initserviceif.H>
 #include <attributeenums.H>
 #include "errlentry_consts.H"
-
+#include <util/misc.H>
 
 // Hostboot Image ID string
 extern char hbi_ImageId;
@@ -707,7 +707,7 @@ void ErrlEntry::commit( compId_t  i_committerComponent )
 
     // Check to make sure targeting is initialized. If so, collect part and
     // serial numbers
-    if( TARGETING::targetService().isInitialized() )
+    if(Util::isTargetingLoaded() && TARGETING::targetService().isInitialized())
     {
         // If this error was a hardware callout, add the serial and part numbers
         // to the log. FSP provides this data so if there is no FSP, get them here.
diff --git a/src/usr/isteps/istep10/call_host_update_redundant_tpm.C b/src/usr/isteps/istep10/call_host_update_redundant_tpm.C
index ffdf52980..59f370e81 100644
--- a/src/usr/isteps/istep10/call_host_update_redundant_tpm.C
+++ b/src/usr/isteps/istep10/call_host_update_redundant_tpm.C
@@ -25,6 +25,7 @@
 #include <errl/errlentry.H>
 #include <errl/errlmanager.H>
 #include <isteps/istep_reasoncodes.H>
+#include <initservice/initserviceif.H>
 
 // targeting support
 #include <targeting/common/target.H>
@@ -49,10 +50,206 @@
 
 // secure boot
 #include <secureboot/service.H>
-
+#include <secureboot/settings.H>
+#include <i2c/eepromif.H>
+#include <sbe/sbeif.H>
+#include "../../secureboot/common/errlud_secure.H"
+#include "../../sbe/sbe_update.H"
 
 namespace ISTEP_10
 {
+
+#ifdef CONFIG_SECUREBOOT
+/**
+ * @brief This structure associates SBE HW keys' hashes with names and SBE sides
+ */
+struct HashNode
+{
+    uint8_t* hash; /** SBE HW keys' hash for the named side */
+    const char* name; /** Name of the side: either primary or backup */
+    uint8_t side; /** A uint8_t value of 0 for primary or 1 for backup */
+    HashNode(uint8_t* i_hash,
+             const char* i_name,
+             uint8_t i_side) : hash(i_hash), name(i_name), side(i_side)
+    {
+    }
+};
+
+/**
+ * @brief This union simplifies bitwise operations for tracking the four
+ * conditions of mismatch when matching security settings of a slave processor
+ * with that of the master processor.
+ */
+union Mismatches
+{
+    uint8_t val;
+    struct {
+        uint8_t reserved : 4; /** unused */
+        uint8_t sabmis : 1; /** Value of 1 indicates the SAB bit didn't match,
+                             *  0 otherwise.
+                             */
+        uint8_t smdmis : 1; /** Value of 1 indicates the SMD bit did not match,
+                             *  0 otherwise.
+                             */
+        uint8_t primis : 1; /** Value of 1 indicates the primary SBE HW key's
+                             *  did not match, 0 otherwise.
+                             */
+        uint8_t bacmis : 1; /** Value of 1 indicates the backup SBE HW key's
+                             *  did not match, 0 otherwise.
+                             */
+    };
+};
+
+/** @brief Handle a processor security error.
+ *  @par Detailed Description:
+ *      Puts all the error handling for mismatched processor security
+ *      settings in one place in order to minimize code footprint. This
+ *      highly-specialized function is not intended for public consumption.
+ *  @param[in] i_pProc The target processor whose secure processor settings are
+ *      exhibiting a problem. Must not be null.
+ *  @param[in] i_rc The reason code of the error to be handled.
+ *  @param[in] i_hashes A vector containing hash/string/side triples, where
+ *      the string indicates the name of the hash, and the side refers to the
+ *      SBE side that the hash was taken from.
+ *  @param[in] i_plid If non zero this plid from a previous error will be linked
+ *      To the error created by this function, ignored otherwise.
+ *  @param[in] i_continue A boolean indicating whether hostboot should continue
+ *      booting (and deconfigure the processor) or stop the IPL.
+ *  @param[in] i_mismatches A bitstring of mismatch bits of type Mismatches
+ *      corresponding to a mismatch of the SAB or SMD register or primary or
+ *      secondary SBE HW Keys' Hash between the supplied target and the master
+ *      processor. Optional parameter is for the RC_PROC_SECURITY_STATE_MISMATCH
+ *      case only and must be left as default (value of 0) for all other cases.
+ */
+void handleProcessorSecurityError(TARGETING::Target* i_pProc,
+                ISTEP::istepReasonCode i_rc,
+                const std::vector<HashNode>& i_hashes,
+                uint16_t i_plid,
+                bool i_continue,
+                Mismatches i_mismatches={0})
+{
+    using namespace ISTEP;
+
+    // stop the caller from passing a null target
+    assert(i_pProc != nullptr, "Bug! Target pointer must not be null");
+
+    // make sure that caller is using the Mismatches parameter at the right time
+    assert( (i_rc!=RC_PROC_SECURITY_STATE_MISMATCH && !i_mismatches.val) ||
+            (i_rc==RC_PROC_SECURITY_STATE_MISMATCH && i_mismatches.val),
+            "Mismatches parameter is for RC_PROC_SECURITY_STATE_MISMATCH only");
+
+    ERRORLOG::errlSeverity_t l_severity = ERRORLOG::ERRL_SEV_UNRECOVERABLE;
+
+    if (i_rc==RC_MASTER_PROC_SBE_KEYS_HASH_MISMATCH ||
+        i_rc==RC_PROC_SECURITY_STATE_MISMATCH)
+    {
+        if (i_rc==RC_PROC_SECURITY_STATE_MISMATCH)
+        {
+            TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                    ERR_MRK"call_host_update_redundant_tpm - handleProcessorSecurityError processor state does match master for processor tgt=0x%X",
+                    TARGETING::get_huid(i_pProc));
+
+            TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                    "SMD is a %smatch", i_mismatches.smdmis? "mis": "");
+            TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                    "SAB is a %smatch", i_mismatches.sabmis? "mis": "");
+            TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                    "Primary SBE hash is a %smatch",
+                    i_mismatches.primis? "mis": "");
+            TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                    "Backup SBE hash is a %smatch",
+                    i_mismatches.bacmis? "mis": "");
+        }
+        else  // master proc sbe keys' hash mismatch
+        {
+            TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                ERR_MRK"call_host_update_redundant_tpm - handleProcessorSecurityError: Master processor sbe keys' hash doesn't match master backup sbe key's hash");
+        }
+
+        // Log as informational if secure boot is disabled
+        if (!SECUREBOOT::enabled())
+        {
+            l_severity = ERRORLOG::ERRL_SEV_INFORMATIONAL;
+        }
+    }
+    else
+    {
+        TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+            "call_host_update_redundant_tpm - handleProcessorSecurityError: Istep error occurred, reason code 0x%X"
+            ,i_rc);
+        TRACDCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+            ERR_MRK"call_host_update_redundant_tpm - handleProcessorSecurityError: %s fail",
+        i_rc==RC_MASTER_PROC_PRIMARY_HASH_READ_FAIL?"Master primary hash read":
+        i_rc==RC_MASTER_PROC_BACKUP_HASH_READ_FAIL?"Master backup hash read":
+        i_rc==RC_MASTER_PROC_CBS_CONTROL_READ_FAIL?"Master CBS control read":
+        i_rc==RC_SLAVE_PROC_PRIMARY_HASH_READ_FAIL?"Slave primary hash read":
+        i_rc==RC_SLAVE_PROC_BACKUP_HASH_READ_FAIL?"Slave backup hash read":
+        i_rc==RC_SLAVE_PROC_CBS_CONTROL_READ_FAIL?"Slave CBS control read":
+        "unknown");
+    }
+
+    auto err = new ERRORLOG::ErrlEntry(l_severity,
+                ISTEP::MOD_UPDATE_REDUNDANT_TPM,
+                i_rc,
+                TARGETING::get_huid(i_pProc),
+                TO_UINT64(i_mismatches.val),
+                true);
+
+    // if a plid was given, link it to the new error.
+    if (i_plid)
+    {
+        err->plid(i_plid);
+    }
+
+    err->collectTrace(ISTEP_COMP_NAME);
+
+    ERRORLOG::ErrlUserDetailsTarget(i_pProc).addToLog(err);
+
+    SECUREBOOT::addSecurityRegistersToErrlog(err);
+
+    sha2_hash_t sysHash = {0};
+    SECUREBOOT::getHwKeyHash(sysHash);
+    SECUREBOOT::UdSystemHwKeyHash(sysHash).addToLog(err);
+
+    // add hashes to log and traces
+    for(auto& hsh : i_hashes)
+    {
+        TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace, ERR_MRK"call_host_update_redundant_tpm - handleProcessorSecurityError - %s hash: ", hsh.name);
+        TRACFBIN(ISTEPS_TRACE::g_trac_isteps_trace,
+                "Data = ",
+                reinterpret_cast<void*>(hsh.hash),
+                SHA512_DIGEST_LENGTH);
+        SECUREBOOT::UdTargetHwKeyHash(
+                i_pProc,
+                hsh.side,
+                hsh.hash).addToLog(err);
+    }
+
+    if (i_continue)
+    {
+         err->addHwCallout(i_pProc,
+                HWAS::SRCI_PRIORITY_LOW,
+                    i_mismatches.val && // for any mismatch
+                        !RC_MASTER_PROC_SBE_KEYS_HASH_MISMATCH?
+                    HWAS::NO_DECONFIG: // don't deconfig the processor
+                    HWAS::DELAYED_DECONFIG,
+                HWAS::GARD_NULL);
+    }
+
+    // save off reason code before committing
+    auto l_reason = err->reasonCode();
+
+    ERRORLOG::errlCommit(err, ISTEP_COMP_ID);
+
+    if (!i_continue)
+    {
+        TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+            "Terminating because future security cannot be guaranteed.");
+        INITSERVICE::doShutdown(l_reason);
+    }
+}
+#endif
+
 void* call_host_update_redundant_tpm (void *io_pArgs)
 {
     TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
@@ -75,24 +272,26 @@ void* call_host_update_redundant_tpm (void *io_pArgs)
 
     // Start of update procedure
     #ifdef CONFIG_SECUREBOOT
-    do {
 
     bool l_force = false;
 
+    TARGETING::TargetHandleList l_procList;
+    getAllChips(l_procList,TARGETING::TYPE_PROC,true);
+
+    // call p9_update_security_ctrl.C HWP
+    do {
+
     if (!SECUREBOOT::enabled() && !l_force)
     {
         break;
     }
 
-    // call p9_update_security_ctrl.C HWP
-    TARGETING::TargetHandleList l_procList;
-    getAllChips(l_procList,TARGETING::TYPE_PROC,true);
-
     TARGETING::TargetHandleList l_tpmList;
     getAllChips(l_tpmList,TARGETING::TYPE_TPM,false);
 
-    // for each processor in list
-    for(auto pProc : l_procList)
+    // loop through the processors
+    auto pProcItr = l_procList.begin();
+    while (pProcItr != l_procList.end())
     {
         bool l_notInMrw = true;
         TARGETING::Target* l_tpm = nullptr;
@@ -103,7 +302,7 @@ void* call_host_update_redundant_tpm (void *io_pArgs)
         // the path of the current processor
         for (auto itpm : l_tpmList)
         {
-            auto l_physPath = pProc->getAttr<TARGETING::ATTR_PHYS_PATH>();
+            auto l_physPath = (*pProcItr)->getAttr<TARGETING::ATTR_PHYS_PATH>();
 
             auto l_tpmInfo = itpm->getAttr<TARGETING::ATTR_TPM_INFO>();
 
@@ -118,11 +317,12 @@ void* call_host_update_redundant_tpm (void *io_pArgs)
         if (l_notInMrw)
         {
             uint8_t l_protectTpm = 1;
-            pProc->setAttr<TARGETING::ATTR_SECUREBOOT_PROTECT_DECONFIGURED_TPM
-                >(l_protectTpm);
+            (*pProcItr)->setAttr<
+                TARGETING::ATTR_SECUREBOOT_PROTECT_DECONFIGURED_TPM
+                                                                >(l_protectTpm);
         }
 
-        const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP> l_fapiTarg(pProc);
+        const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP> l_fapiTarg(*pProcItr);
 
         FAPI_INVOKE_HWP(err, p9_update_security_ctrl, l_fapiTarg);
 
@@ -133,13 +333,13 @@ void* call_host_update_redundant_tpm (void *io_pArgs)
                 "p9_update_security_ctrl failed for processor tgt=0x%X, "
                 "TPM tgt=0x%X. Deconfiguring processor because future "
                 "security cannot be guaranteed.",
-                TARGETING::get_huid(pProc),
+                TARGETING::get_huid(*pProcItr),
                 TARGETING::get_huid(l_tpm));
 
             // save the plid from the error before commiting
             auto plid = err->plid();
 
-            ERRORLOG::ErrlUserDetailsTarget(pProc).addToLog(err);
+            ERRORLOG::ErrlUserDetailsTarget(*pProcItr).addToLog(err);
 
             // commit this error log first before creating the new one
             ERRORLOG::errlCommit(err, ISTEP_COMP_ID);
@@ -159,11 +359,11 @@ void* call_host_update_redundant_tpm (void *io_pArgs)
             err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                 ISTEP::MOD_UPDATE_REDUNDANT_TPM,
                 ISTEP::RC_UPDATE_SECURITY_CTRL_HWP_FAIL,
-                TARGETING::get_huid(pProc),
+                TARGETING::get_huid(*pProcItr),
                 TARGETING::get_huid(l_tpm),
                 true);
 
-            err->addHwCallout(pProc,
+            err->addHwCallout(*pProcItr,
                             HWAS::SRCI_PRIORITY_LOW,
                             HWAS::DELAYED_DECONFIG,
                             HWAS::GARD_NULL);
@@ -173,13 +373,365 @@ void* call_host_update_redundant_tpm (void *io_pArgs)
             // pass on the plid from the previous error log to the new one
             err->plid(plid);
 
-            ERRORLOG::ErrlUserDetailsTarget(pProc).addToLog(err);
+            ERRORLOG::ErrlUserDetailsTarget(*pProcItr).addToLog(err);
 
             ERRORLOG::errlCommit(err, ISTEP_COMP_ID);
 
+            // remove the deconfigured processor from the list so that we can
+            // reuse this list later to enforce processor security state below
+            pProcItr = l_procList.erase(pProcItr);
+
             // we don't break here. we need to continue on to the next
             // processor and run the HWP on that one
         }
+        else
+        {
+            TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+                "p9_update_security_ctrl successful for proc: 0x%X tpm: 0x%X",
+                TARGETING::get_huid(*pProcItr),
+                TARGETING::get_huid(l_tpm));
+            // only move on to the next processor if we didn't erase the
+            // current one, since erasing the current one automatically gives us
+            // the next one
+            ++pProcItr;
+        }
+    }
+
+    } while(0);
+    // end of p9_update_security_ctrl procedure
+
+    // Enforce Synchronized Proc Security State
+    do {
+
+    uint64_t l_mainCbs = 0;
+
+    // a list of hashes we will be using to match primary to backups and
+    // masters to slaves
+    std::vector<HashNode> l_hashes;
+
+    // master processor primary hash
+    SHA512_t l_masterHash = {0};
+
+    // master processor backup hash
+    SHA512_t l_backupHash = {0};
+
+    // slave processor primary hash (reset each time through the loop)
+    SHA512_t l_slaveHashPri = {0};
+
+    // slave processor backup hash (reset each time through the loop)
+    SHA512_t l_slaveHashBac = {0};
+
+    // nodes for the hashes vector only to be added to vector as needed
+    auto l_master = HashNode(l_masterHash, "master primary", SBE::SBE_SEEPROM0);
+    auto l_backup = HashNode(l_backupHash, "master backup", SBE::SBE_SEEPROM1);
+    auto l_slave = HashNode(l_slaveHashPri, "slave primary", SBE::SBE_SEEPROM0);
+    auto l_slaveb = HashNode(l_slaveHashBac, "slave backup", SBE::SBE_SEEPROM1);
+    // obtain the master processor target
+    TARGETING::Target* mProc = nullptr;
+    err = TARGETING::targetService().queryMasterProcChipTargetHandle(mProc);
+    if (err)
+    {
+        // if this happens we are in big trouble
+        auto rc = err->reasonCode();
+        ERRORLOG::errlCommit(err, ISTEP_COMP_ID);
+        TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+            "Terminating because future security cannot be guaranteed.");
+        INITSERVICE::doShutdown(rc);
+        break;
+    }
+
+    // read the CBS control register of the main processor
+    // (has SAB/SMD bits)
+    err = SECUREBOOT::getProcCbsControlRegister(l_mainCbs, mProc);
+    if (err)
+    {
+        auto plid = err->plid();
+        ERRORLOG::errlCommit(err, ISTEP_COMP_ID);
+
+        /*@
+         * @errortype
+         * @reasoncode       ISTEP::RC_MASTER_PROC_CBS_CONTROL_READ_FAIL
+         * @moduleid         ISTEP::MOD_UPDATE_REDUNDANT_TPM
+         * @userdata1        Master Processor Target
+         * @devdesc          Unable to read the master processor CBS control
+         *                   register
+         * @custdesc         Platform security problem detected
+         */
+        handleProcessorSecurityError(mProc,
+                                  ISTEP::RC_MASTER_PROC_CBS_CONTROL_READ_FAIL,
+                                  l_hashes,
+                                  plid,
+                                  false); // stop IPL and deconfig processor
+        break;
+    }
+
+    // read the primary sbe HW keys' hash for the master processor
+    err = SBE::getHwKeyHashFromSbeSeeprom(
+                                     mProc,
+                                     EEPROM::SBE_PRIMARY,
+                                     l_masterHash);
+    if (err)
+    {
+        auto plid = err->plid();
+        ERRORLOG::errlCommit(err, ISTEP_COMP_ID);
+
+        /*@
+         * @errortype
+         * @reasoncode       ISTEP::RC_MASTER_PROC_PRIMARY_HASH_READ_FAIL
+         * @moduleid         ISTEP::MOD_UPDATE_REDUNDANT_TPM
+         * @userdata1        Master Processor Target
+         * @devdesc          Unable to read the master processor primary hash
+         *                   from the SBE
+         * @custdesc         Platform security problem detected
+         */
+        handleProcessorSecurityError(mProc,
+                                  ISTEP::RC_MASTER_PROC_PRIMARY_HASH_READ_FAIL,
+                                  l_hashes,
+                                  plid,
+                                  false); // stop IPL and deconfig processor
+    }
+
+    // read the backup sbe HW keys' hash for the master processor
+    err = SBE::getHwKeyHashFromSbeSeeprom(
+                                      mProc,
+                                      EEPROM::SBE_BACKUP,
+                                      l_backupHash);
+    if (err)
+    {
+        auto plid = err->plid();
+        ERRORLOG::errlCommit(err, ISTEP_COMP_ID);
+
+        /*@
+         * @errortype
+         * @reasoncode       ISTEP::RC_MASTER_PROC_BACKUP_HASH_READ_FAIL
+         * @moduleid         ISTEP::MOD_UPDATE_REDUNDANT_TPM
+         * @userdata1        Processor Target
+         * @devdesc          Unable to read the master processor backup hash
+         *                   from the SBE
+         * @custdesc         Platform security problem detected
+         */
+        handleProcessorSecurityError(mProc,
+                                  ISTEP::RC_MASTER_PROC_BACKUP_HASH_READ_FAIL,
+                                  l_hashes,
+                                  plid,
+                                  false); // stop IPL and deconfig processor
+        break;
+    }
+
+    // make sure the master processor primary and backup SBE HW keys' hashes
+    // match each other
+    if (memcmp(l_masterHash,l_backupHash, SHA512_DIGEST_LENGTH)!=0)
+    {
+        // add only the hashes relevant to the error to hashes vector
+        l_hashes.push_back(l_master);
+        l_hashes.push_back(l_backup);
+
+       bool l_continue = !SECUREBOOT::enabled();
+        /*@
+         * @errortype
+         * @reasoncode       ISTEP::RC_MASTER_PROC_SBE_KEYS_HASH_MISMATCH
+         * @moduleid         ISTEP::MOD_UPDATE_REDUNDANT_TPM
+         * @severity         ERRL_SEV_UNRECOVERABLE
+         * @userdata1        Master Processor Target
+         * @devdesc          The primary SBE HW Keys' hash does not match the
+         *                   the backup SBE HW Keys' hash, so we cannot
+         *                   guarrantee platform security for the system
+         * @custdesc         Platform security problem detected
+         */
+        handleProcessorSecurityError(mProc,
+                                  ISTEP::RC_MASTER_PROC_SBE_KEYS_HASH_MISMATCH,
+                                  l_hashes,
+                                  0,
+                                  l_continue); // stop IPL if secureboot enabled
+
+        break;
+    }
+    TRACFCOMP(ISTEPS_TRACE::g_trac_isteps_trace,
+        "Master Primary SBE HW keys' hash successfully matches backup.");
+
+    for(auto pProc : l_procList)
+    {
+        uint64_t l_procCbs = 0;
+
+        if (mProc == pProc)
+        {
+            // skip the master processor
+            continue;
+        }
+
+        // start with empty slave hashes each time through the loop
+        memset(l_slaveHashPri,0,SHA512_DIGEST_LENGTH);
+        memset(l_slaveHashBac,0,SHA512_DIGEST_LENGTH);
+
+        // read the CBS control register of the current processor
+        err = SECUREBOOT::getProcCbsControlRegister(l_procCbs, pProc);
+        if (err)
+        {
+            auto plid = err->plid();
+            ERRORLOG::errlCommit(err, ISTEP_COMP_ID);
+
+            /*@
+             * @errortype
+             * @reasoncode       ISTEP::RC_SLAVE_PROC_CBS_CONTROL_READ_FAIL
+             * @moduleid         ISTEP::MOD_UPDATE_REDUNDANT_TPM
+             * @userdata1        Slave Processor Target
+             * @devdesc          Unable to read the slave processor CBS control
+             *                   register
+             *
+             * @custdesc         Platform security problem detected
+             */
+            handleProcessorSecurityError(pProc,
+                                  ISTEP::RC_SLAVE_PROC_CBS_CONTROL_READ_FAIL,
+                                  l_hashes,
+                                  plid,
+                                  true); // deconfigure proc and move on
+            continue;
+        }
+
+        // read the primary sbe HW keys' hash for the current processor
+        err = SBE::getHwKeyHashFromSbeSeeprom(
+                                         pProc,
+                                         EEPROM::SBE_PRIMARY,
+                                         l_slaveHashPri);
+        if (err)
+        {
+            auto plid = err->plid();
+            ERRORLOG::errlCommit(err, ISTEP_COMP_ID);
+
+            /*@
+             * @errortype
+             * @reasoncode       ISTEP::RC_SLAVE_PROC_PRIMARY_HASH_READ_FAIL
+             * @moduleid         ISTEP::MOD_UPDATE_REDUNDANT_TPM
+             * @userdata1        Slave Processor Target
+             * @devdesc          Unable to read the slave processor primary
+             *                   hash from the SBE
+             * @custdesc         Platform security problem detected
+             */
+            handleProcessorSecurityError(pProc,
+                                  ISTEP::RC_SLAVE_PROC_PRIMARY_HASH_READ_FAIL,
+                                  l_hashes,
+                                  plid,
+                                  true); // deconfigure proc and move on
+            continue;
+        }
+
+        // read the backup sbe HW keys' hash for the current processor
+        err = SBE::getHwKeyHashFromSbeSeeprom(
+                                         pProc,
+                                         EEPROM::SBE_BACKUP,
+                                         l_slaveHashBac);
+        if (err)
+        {
+            auto plid = err->plid();
+            ERRORLOG::errlCommit(err, ISTEP_COMP_ID);
+
+            /*@
+             * @errortype
+             * @reasoncode       ISTEP::RC_SLAVE_PROC_BACKUP_HASH_READ_FAIL
+             * @moduleid         ISTEP::MOD_UPDATE_REDUNDANT_TPM
+             * @userdata1        Slave Processor Target
+             * @devdesc          Unable to read the slave processor backup
+             *                   hash from the SBE
+             * @custdesc         Platform security problem detected
+             */
+            handleProcessorSecurityError(pProc,
+                                  ISTEP::RC_SLAVE_PROC_BACKUP_HASH_READ_FAIL,
+                                  l_hashes,
+                                  plid,
+                                  true); // deconfigure proc and move on
+
+            continue;
+        }
+
+        // If the current processor has a key or SAB mismatch
+        // then throw a terminate error. For SMD mismatch we throw
+        // an informational error
+        Mismatches l_mismatches = {0};
+
+        // SAB bit mismatch
+        l_mismatches.sabmis =
+                      (SECUREBOOT::ProcCbsControl::SabBit & l_mainCbs) !=
+                       (SECUREBOOT::ProcCbsControl::SabBit & l_procCbs);
+
+        // Jumper state mismatch
+        l_mismatches.smdmis =
+                (SECUREBOOT::ProcCbsControl::JumperStateBit & l_mainCbs) !=
+                 (SECUREBOOT::ProcCbsControl::JumperStateBit & l_procCbs);
+
+        // primary sbe hash mismatch
+        l_mismatches.primis = memcmp(l_slaveHashPri,
+                             l_masterHash,
+                             SHA512_DIGEST_LENGTH) != 0;
+
+        // backup sbe hash mismatch
+        l_mismatches.bacmis = memcmp(l_slaveHashBac,
+                             l_masterHash,
+                             SHA512_DIGEST_LENGTH) != 0;
+
+        // only provide the relevant hashes for error handling cases
+        if(l_mismatches.primis || l_mismatches.bacmis)
+        {
+            l_hashes.push_back(l_master);
+        }
+        if(l_mismatches.primis)
+        {
+            l_hashes.push_back(l_slave);
+        }
+        if(l_mismatches.bacmis)
+        {
+            l_hashes.push_back(l_slaveb);
+        }
+
+        // if there was any mismatch
+        if (l_mismatches.val)
+        {
+            auto l_continue = true;
+            // do not continue booting if there is a SAB mismatch under any
+            // circumstance
+            if (l_mismatches.sabmis)
+            {
+                l_continue = false;
+            }
+            // In secure mode, do not continue booting when SBE HW keys' hashes
+            // are mismatched
+            if ((l_mismatches.primis || l_mismatches.bacmis)
+                 && SECUREBOOT::enabled())
+            {
+                l_continue = false;
+            }
+
+            /*@
+             * @errortype
+             * @reasoncode       ISTEP::RC_PROC_SECURITY_STATE_MISMATCH
+             * @moduleid         ISTEP::MOD_UPDATE_REDUNDANT_TPM
+             * @userdata1        Processor Target
+             * @userdata2[60:60] SAB bit mismatch
+             * @userdata2[61:61] Jumper (SMD) bit mismatch
+             * @userdata2[62:62] Primary SBE hash mismatch
+             * @userdata2[63:63] Backup SBE hash mismatch
+             * @devdesc          Mismatch processor state was detected for this
+             *                   processor, so we cannot guarrantee platform
+             *                   security for the system
+             * @custdesc         Platform security problem detected
+             */
+            handleProcessorSecurityError(pProc,
+                                  ISTEP::RC_PROC_SECURITY_STATE_MISMATCH,
+                                  l_hashes,
+                                  0,
+                                  l_continue,
+                                  l_mismatches);
+
+            // In non-secure mode, look for other inconsistencies and log the
+            // issues
+            if (l_continue)
+            {
+                continue;
+            }
+            // In secure mode,  stop checking for proc security state mismatches
+            // as soon as a mismatch has been found
+            break;
+        }
+
     }
 
     } while(0);
diff --git a/src/usr/secureboot/base/service.C b/src/usr/secureboot/base/service.C
index 673ec709e..7513f7dc6 100644
--- a/src/usr/secureboot/base/service.C
+++ b/src/usr/secureboot/base/service.C
@@ -85,20 +85,6 @@ struct SecureRegisterValues
  */
 errlHndl_t getAllSecurityRegisters(std::vector<SecureRegisterValues> & o_regs);
 
-/**
- *  @brief Adds the values of the Security Registers of the processors in the
- *         system to an existing error log
- *
- *  @param[in/out] io_err  Error Log that the values of the security registers
- *                         will be added to
- *                         NOTE:  The state of the system/processors (ie, SCOM
- *                         vs FSI) determines which registers can be included
- *
- *  @return N/A
- */
-void addSecurityRegistersToErrlog(errlHndl_t & io_err);
-
-
 void* initializeBase(void* unused)
 {
     errlHndl_t l_errl = NULL;
@@ -140,14 +126,23 @@ bool enabled()
 }
 #endif
 
-errlHndl_t getSecuritySwitch(uint64_t& o_regValue, TARGETING::Target* i_targ)
+errlHndl_t getSecuritySwitch(uint64_t& o_regValue, TARGETING::Target* i_pProc)
+{
+    return Singleton<Settings>::instance().getSecuritySwitch(o_regValue,
+                                                                       i_pProc);
+}
+
+errlHndl_t getProcCbsControlRegister(uint64_t& o_regValue,
+    TARGETING::Target* i_pProc)
 {
-    return Singleton<Settings>::instance().getSecuritySwitch(o_regValue,i_targ);
+    return Singleton<Settings>::instance().getProcCbsControlRegister(o_regValue,
+        i_pProc);
 }
 
-errlHndl_t getJumperState(SecureJumperState& o_state, TARGETING::Target* i_targ)
+errlHndl_t getJumperState(SecureJumperState& o_state,
+                                                    TARGETING::Target* i_pProc)
 {
-    return Singleton<Settings>::instance().getJumperState(o_state, i_targ);
+    return Singleton<Settings>::instance().getJumperState(o_state, i_pProc);
 }
 
 errlHndl_t clearSecuritySwitchBits(
diff --git a/src/usr/secureboot/base/settings.C b/src/usr/secureboot/base/settings.C
index 9e1cb2ee0..0c4046d59 100644
--- a/src/usr/secureboot/base/settings.C
+++ b/src/usr/secureboot/base/settings.C
@@ -31,6 +31,9 @@
 #include <targeting/common/target.H>
 #include <initservice/initserviceif.H>
 #include <secureboot/settings.H>
+#include <config.h>
+#include <errl/errludlogregister.H>
+#include <console/consoleif.H>
 
 // SECUREBOOT : General driver traces
 #include "../common/securetrace.H"
@@ -65,6 +68,77 @@ namespace SECUREBOOT
                             static_cast<uint64_t>(ProcSecurity::SabBit)));
 
         SB_INF("getEnabled() state:%i",iv_enabled);
+
+        // send informational log if secure boot is disabled
+        #ifdef CONFIG_SECUREBOOT
+        if (!iv_enabled)
+        {
+            #ifdef CONFIG_CONSOLE
+            CONSOLE::displayf(SECURE_COMP_NAME, "Booting in non-secure mode.");
+            #endif
+            /*@
+             * @errortype
+             * @reasoncode       SECUREBOOT::RC_SECURE_BOOT_DISABLED
+             * @moduleid         SECUREBOOT::MOD_SECURE_SETTINGS_INIT
+             * @severity         ERRL_SEV_INFORMATIONAL
+             * @userdata1        Security switch register value
+             * @devdesc          Secureboot has been disabled.
+             * @custdesc         Platform security informational message
+             */
+            auto err = new ERRORLOG::ErrlEntry(
+                ERRORLOG::ERRL_SEV_INFORMATIONAL,
+                SECUREBOOT::MOD_SECURE_SETTINGS_INIT,
+                SECUREBOOT::RC_SECURE_BOOT_DISABLED,
+                l_regValue,
+                0,
+                false);
+
+            err->collectTrace(SECURE_COMP_NAME);
+
+            // we can't call getAllSecurityRegisters from here because it
+            // will deadlock when it circles back to getSecuritySwitch - the
+            // call to retreive the singleton for Settings class will hang.
+            // So, we just log the security switch and cbs control registers
+            ERRORLOG::ErrlUserDetailsLogRegister l_logReg(
+                MASTER_PROCESSOR_CHIP_TARGET_SENTINEL,
+                &l_regValue,
+                sizeof(l_regValue),
+                DEVICE_SCOM_ADDRESS(
+                    static_cast<uint64_t>(ProcSecurity::SwitchRegister)
+                ));
+            l_logReg.addToLog(err);
+
+            uint64_t l_cbsReg = 0;
+            auto l_cbsErrl = getProcCbsControlRegister(
+                l_cbsReg,
+                MASTER_PROCESSOR_CHIP_TARGET_SENTINEL);
+
+            if (l_cbsErrl)
+            {
+                // link the CBS control register erorr plid to the original err
+                err->plid(l_cbsErrl->plid());
+
+                // commit the CBS control register error
+                ERRORLOG::errlCommit(l_cbsErrl, SECURE_COMP_ID);
+
+                // we're already in the error path so we just keep going
+                // without the register
+            }
+            else
+            {
+                ERRORLOG::ErrlUserDetailsLogRegister l_logCbsReg(
+                    MASTER_PROCESSOR_CHIP_TARGET_SENTINEL,
+                    &l_cbsReg,
+                    sizeof(l_cbsReg),
+                    DEVICE_SCOM_ADDRESS(
+                        static_cast<uint64_t>(ProcCbsControl::StatusRegister)
+                    ));
+                l_logCbsReg.addToLog(err);
+            }
+
+            ERRORLOG::errlCommit(err, SECURE_COMP_ID);
+        }
+        #endif
     }
 
     bool Settings::getEnabled() const
@@ -73,7 +147,7 @@ namespace SECUREBOOT
     }
 
     errlHndl_t Settings::getJumperState(SecureJumperState& o_state,
-                                        Target* i_targ) const
+                                        Target* i_pProc) const
     {
         uint64_t l_regValue = 0;
         o_state = SecureJumperState::SECURITY_DEASSERTED;
@@ -82,14 +156,10 @@ namespace SECUREBOOT
 
         do
         {
-            // the supplied target input parameter is validated in one place
-            // inside the readSecurityRegister function
-            l_errl = readSecurityRegister(i_targ,
-                    static_cast<uint64_t>(ProcCbsControl::StatusRegister),
-                    l_regValue);
+            l_errl = getProcCbsControlRegister(l_regValue, i_pProc);
 
             SB_DBG("getJumperState() err:%i reg:%.16llX huid:%.8X",
-                !!l_errl, l_regValue, get_huid(i_targ));
+                !!l_errl, l_regValue, get_huid(i_pProc));
 
             if (l_errl)
             {
@@ -105,21 +175,31 @@ namespace SECUREBOOT
                      SecureJumperState::SECURITY_ASSERTED;
 
             SB_INF("getJumperState() state:%i huid:%.8X", o_state,
-                                                            get_huid(i_targ));
+                                                            get_huid(i_pProc));
 
         } while(0);
 
         return l_errl;
     }
 
+    errlHndl_t Settings::getProcCbsControlRegister(uint64_t& o_regValue,
+                                                   Target* i_pProc) const
+    {
+        // the supplied target input parameter is validated in one place
+        // inside the readSecurityRegister function
+        return readSecurityRegister(i_pProc,
+                    static_cast<uint64_t>(ProcCbsControl::StatusRegister),
+                    o_regValue);
+    }
+
     errlHndl_t Settings::getSecuritySwitch(uint64_t& o_regValue,
-                                           Target* i_targ) const
+                                           Target* i_pProc) const
     {
-        auto l_errl = readSecurityRegister(i_targ,
+        auto l_errl = readSecurityRegister(i_pProc,
                     static_cast<uint64_t>(ProcSecurity::SwitchRegister),
                     o_regValue);
         SB_INF("getSecuritySwitch() err:%i reg:%.16llX huid:%.8X",
-            !!l_errl, o_regValue, get_huid(i_targ));
+            !!l_errl, o_regValue, get_huid(i_pProc));
 
         return l_errl;
     }
@@ -259,7 +339,7 @@ namespace SECUREBOOT
         return pError;
     }
 
-    errlHndl_t Settings::readSecurityRegister(Target* i_targ,
+    errlHndl_t Settings::readSecurityRegister(Target* i_pProc,
                                             const uint64_t i_scomAddress,
                                             uint64_t& o_regValue) const
     {
@@ -271,8 +351,8 @@ namespace SECUREBOOT
 
         // make sure we are not passed a null target pointer or the wrong
         // target type (must be a processor target) or the sentinel
-        if ( i_targ != MASTER_PROCESSOR_CHIP_TARGET_SENTINEL &&
-            (i_targ == nullptr || i_targ->getAttr<ATTR_TYPE>() != TYPE_PROC)
+        if ( i_pProc != MASTER_PROCESSOR_CHIP_TARGET_SENTINEL &&
+            (i_pProc == nullptr || i_pProc->getAttr<ATTR_TYPE>() != TYPE_PROC)
            )
         {
             /*@
@@ -289,15 +369,22 @@ namespace SECUREBOOT
             l_errl = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
                                          SECUREBOOT::MOD_SECURE_READ_REG,
                                          SECUREBOOT::RC_SECURE_BAD_TARGET,
-                                         reinterpret_cast<uint64_t>(i_targ),
-                                         TO_UINT64(get_huid(i_targ)),
+                                         reinterpret_cast<uint64_t>(i_pProc),
+                                         TO_UINT64(get_huid(i_pProc)),
                                          true /* Add HB Software Callout */ );
             l_errl->collectTrace(SECURE_COMP_NAME, ERROR_TRACE_SIZE);
             break;
         }
 
+        // Make sure the processor is SCOMable
+        if (i_pProc != MASTER_PROCESSOR_CHIP_TARGET_SENTINEL)
+        {
+            assert(i_pProc->getAttr<ATTR_SCOM_SWITCHES>().useXscom,
+                "Bug! Processor security register read too early.");
+        }
+
         // Read security switch setting from processor.
-        l_errl = deviceRead(i_targ,
+        l_errl = deviceRead(i_pProc,
                             &o_regValue, size,
                             DEVICE_SCOM_ADDRESS(i_scomAddress));