Slot check after firmware update

Verify every NVDIMM is running from slot 1
and is at the latest firmware version.
Create a predictive error for those that are
not running latest code level.

Change-Id: I46766d267f4a1293739a49c989ef5ab7dde585b5
CQ:SW478561
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/85656
Reviewed-by: Corey V Swenson <cswenson@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Reviewed-by: TSUNG K YEUNG <tyeung@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M Crowell <dcrowell@us.ibm.com>

4 files changed, 146 insertions, 11 deletions

diff --git a/src/usr/isteps/nvdimm/nvdimm.C b/src/usr/isteps/nvdimm/nvdimm.C
index 2d63854b9..65e2c7e85 100644
--- a/src/usr/isteps/nvdimm/nvdimm.C
+++ b/src/usr/isteps/nvdimm/nvdimm.C
@@ -2063,8 +2063,7 @@ void nvdimm_init(Target *i_nvdimm)
         }
 
         // Check if the firmware slot is 0
-        l_err = nvdimmReadReg ( i_nvdimm, FW_SLOT_INFO, l_data);
-
+        l_err = nvdimmGetRunningSlot(i_nvdimm, l_data);
         if (l_err)
         {
             nvdimmSetStatusFlag(i_nvdimm, NSTD_ERR);
@@ -2073,7 +2072,7 @@ void nvdimm_init(Target *i_nvdimm)
             break;
         }
 
-        if (!(l_data & RUNNING_FW_SLOT))
+        if (l_data == 0)
         {
             nvdimmSetStatusFlag(i_nvdimm, NSTD_ERR_VAL_SR);
             TRACFCOMP(g_trac_nvdimm, ERR_MRK"nvdimm_init() nvdimm[%X], running on fw slot 0",
@@ -2083,7 +2082,7 @@ void nvdimm_init(Target *i_nvdimm)
              *@reasoncode       NVDIMM_INVALID_FW_SLOT
              *@severity         ERRORLOG_SEV_PREDICTIVE
              *@moduleid         NVDIMM_CHECK_FW_SLOT
-             *@userdata1[0:31]  Related ops (0xff = NA)
+             *@userdata1[0:31]  Slot running
              *@userdata1[32:63] Target Huid
              *@userdata2        <UNUSED>
              *@devdesc          Encountered error when checking the firmware slot running
@@ -4650,5 +4649,29 @@ void send_ATTR_NVDIMM_ARMED( Target* i_nvdimm,
 #endif //__HOSTBOOT_RUNTIME
 }
 
+/**
+ * @brief Grab the current slot that NVDIMM code is running
+ */
+errlHndl_t nvdimmGetRunningSlot(TARGETING::Target *i_nvdimm, uint8_t & o_slot)
+{
+    errlHndl_t l_err = nullptr;
+    uint8_t l_data = 0;
+    o_slot = 0;  //default to slot 0
+
+    // Check if the firmware slot is 0
+    l_err = nvdimmReadReg ( i_nvdimm, FW_SLOT_INFO, l_data);
+    if (l_err)
+    {
+        nvdimmSetStatusFlag(i_nvdimm, NSTD_ERR);
+        TRACFCOMP(g_trac_nvdimm, ERR_MRK"nvdimmGetRunningSlot() nvdimm[%X], failed to read slot info",
+                  get_huid(i_nvdimm));
+    }
+    else
+    {
+        // Bits 7-4 = RUNNING_FW_SLOT - slot number of running firmware
+        o_slot = (l_data & RUNNING_FW_SLOT) >> 4;
+    }
+    return l_err;
+}
 
 } // end NVDIMM namespace
diff --git a/src/usr/isteps/nvdimm/nvdimm.H b/src/usr/isteps/nvdimm/nvdimm.H
index 8645d6353..683763087 100644
--- a/src/usr/isteps/nvdimm/nvdimm.H
+++ b/src/usr/isteps/nvdimm/nvdimm.H
@@ -733,6 +733,19 @@ errlHndl_t nvdimm_getTPM(TARGETING::Target*& o_tpm);
  */
 errlHndl_t nvdimmValidImage(TARGETING::Target *i_nvdimm, bool &o_imgValid);
 
+
+/**
+ * @brief This function grabs the current slot NVDIMM code is running
+ *        Slot 0 is the failure slot, Slot 1 is the updateable slot
+ *
+ * @param[in] i_nvdimm - nvdimm target with NV controller
+ * @param[out] o_slot -  0 or 1
+ *
+ * @return errlHndl_t - Null if successful, otherwise a pointer to
+ *      the error log.
+ */
+errlHndl_t nvdimmGetRunningSlot(TARGETING::Target *i_nvdimm, uint8_t & o_slot);
+
 } //End NVDIMM namespace
 
 
diff --git a/src/usr/isteps/nvdimm/nvdimm_update.C b/src/usr/isteps/nvdimm/nvdimm_update.C
index 359ad2e3f..dc5f7e693 100644
--- a/src/usr/isteps/nvdimm/nvdimm_update.C
+++ b/src/usr/isteps/nvdimm/nvdimm_update.C
@@ -186,6 +186,7 @@ uint16_t NvdimmLidImage::getVersion()
     return o_version;
 }
 
+
 const uint8_t * NvdimmLidImage::getHeaderAndSmartSignature(uint16_t & o_size)
 {
     o_size = 0;
@@ -356,7 +357,6 @@ errlHndl_t NvdimmInstalledImage::getVersion(uint16_t & o_version,
     return l_err;
 }
 
-
 errlHndl_t NvdimmInstalledImage::updateImage(NvdimmLidImage * i_lidImage)
 {
     errlHndl_t l_err = nullptr;
@@ -1906,6 +1906,9 @@ bool NvdimmsUpdate::runUpdateUsingLid(NvdimmLidImage * i_lidImage,
     errlHndl_t l_err = nullptr;
     for (auto pInstalledImage : i_list)
     {
+        TARGETING::Target * l_nvdimm = pInstalledImage->getNvdimmTarget();
+        uint64_t l_nvdimm_huid = TARGETING::get_huid(l_nvdimm);
+
         INITSERVICE::sendProgressCode();
         bool updateNeeded = false;
         l_err = isUpdateNeeded(updateNeeded, i_lidImage, pInstalledImage);
@@ -1923,9 +1926,6 @@ bool NvdimmsUpdate::runUpdateUsingLid(NvdimmLidImage * i_lidImage,
         else if (updateNeeded)
         {
             // shared trace variables
-            const TARGETING::Target * l_nvdimm = pInstalledImage->getNvdimmTarget();
-            uint64_t l_nvdimm_huid = TARGETING::get_huid(l_nvdimm);
-
             uint32_t l_installed_type = INVALID_TYPE;
             l_err = pInstalledImage->getType(l_installed_type);
             if (l_err)
@@ -1938,6 +1938,7 @@ bool NvdimmsUpdate::runUpdateUsingLid(NvdimmLidImage * i_lidImage,
                     ERRL_GETRC_SAFE(l_err), ERRL_GETPLID_SAFE(l_err));
                 commitPredictiveNvdimmError(l_err);
                 l_err = nullptr;
+                continue;
             }
 
             uint16_t l_oldVersion = INVALID_VERSION;
@@ -2002,6 +2003,7 @@ bool NvdimmsUpdate::runUpdateUsingLid(NvdimmLidImage * i_lidImage,
                 commitPredictiveNvdimmError(l_err);
                 l_err = nullptr;
                 o_no_error_found = false;
+                continue;
             }
             else
             {
@@ -2049,6 +2051,97 @@ bool NvdimmsUpdate::runUpdateUsingLid(NvdimmLidImage * i_lidImage,
                 ERRORLOG::errlCommit(l_err, NVDIMM_COMP_ID);
             }
         } // end of updateNeeded
+
+        /////////////////////////////////////////////////////////////////
+        // Should not exit the nvdimm update stage until each nvdimm
+        // is running at the lid's code level
+        // (or a predictive error was logged for that nvdimm)
+        /////////////////////////////////////////////////////////////////
+
+        // Check NVDIMM is at the latest level and it is running from slot 1
+        uint16_t l_curVersion = INVALID_VERSION;
+        l_err = pInstalledImage->getVersion(l_curVersion, true);
+        if (l_err)
+        {
+            TRACFCOMP(g_trac_nvdimm_upd,
+                ERR_MRK"NvdimmsUpdate::runUpdateUsingLid() - "
+                "Failed to find current level of NVDIMM %.8X. "
+                "RC=0x%X, PLID=0x%.8X", l_nvdimm_huid,
+                ERRL_GETRC_SAFE(l_err), ERRL_GETPLID_SAFE(l_err));
+            commitPredictiveNvdimmError(l_err);
+            l_err = nullptr;
+            o_no_error_found = false;
+            continue;
+        }
+        uint8_t l_slot_running = 0;
+        l_err = nvdimmGetRunningSlot(l_nvdimm, l_slot_running);
+        if (l_err)
+        {
+            TRACFCOMP(g_trac_nvdimm_upd,
+                ERR_MRK"NvdimmsUpdate::runUpdateUsingLid() - "
+                "Failed to find running slot of NVDIMM %.8X. "
+                "RC=0x%X, PLID=0x%.8X", l_nvdimm_huid,
+                ERRL_GETRC_SAFE(l_err), ERRL_GETPLID_SAFE(l_err));
+            commitPredictiveNvdimmError(l_err);
+            l_err = nullptr;
+            o_no_error_found = false;
+            continue;
+        }
+
+        if ((l_slot_running == 0) || (l_curVersion != i_lidImage->getVersion()))
+        {
+            // Not running latest code on this NVDIMM
+            TRACFCOMP(g_trac_nvdimm_upd,
+                ERR_MRK"NvdimmsUpdate::runUpdateUsingLid() - "
+                "NVDIMM %.8X running from slot %d with code level "
+                "0x%04X (lid level: 0x%04X)",
+                l_nvdimm_huid, l_slot_running, l_curVersion,
+                i_lidImage->getVersion());
+            /*@
+             *@errortype
+             *@reasoncode       NVDIMM_NOT_RUNNING_LATEST_LEVEL
+             *@severity         ERRORLOG_SEV_PREDICTIVE
+             *@moduleid         NVDIMM_RUN_UPDATE_USING_LID
+             *@userdata1        NVDIMM Target Huid
+             *@userdata2[0:15]  NVDIMM slot
+             *@userdata2[16:31] slot1 version
+             *@userdata2[32:47] latest version from lid
+             *@devdesc          Encountered error after update while checking
+             *                  if NVDIMM is running latest code level
+             *@custdesc         NVDIMM not running latest firmware level
+             */
+            l_err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_PREDICTIVE,
+                                          NVDIMM_RUN_UPDATE_USING_LID,
+                                          NVDIMM_NOT_RUNNING_LATEST_LEVEL,
+                                          l_nvdimm_huid,
+                                          FOUR_UINT16_TO_UINT64(
+                                              l_slot_running,
+                                              l_curVersion,
+                                              i_lidImage->getVersion(),
+                                              0x0000),
+                                          ERRORLOG::ErrlEntry::NO_SW_CALLOUT );
+
+            l_err->collectTrace( NVDIMM_COMP_NAME );
+
+            // Add callout of nvdimm with no deconfig/gard
+            l_err->addHwCallout( l_nvdimm,
+                                 HWAS::SRCI_PRIORITY_LOW,
+                                 HWAS::NO_DECONFIG,
+                                 HWAS::GARD_NULL);
+
+            // Maybe vendor log will tell why it isn't running latest code level
+            nvdimmAddVendorLog(l_nvdimm, l_err);
+            commitPredictiveNvdimmError(l_err);
+            l_err = nullptr;
+            o_no_error_found = false;
+        }
+        else
+        {
+            TRACFCOMP(g_trac_nvdimm_upd,
+                "NvdimmsUpdate::runUpdateUsingLid() - "
+                "NVDIMM %.8X running from slot %d with latest level 0x%04X",
+                l_nvdimm_huid, l_slot_running, l_curVersion);
+        }
     }
     return o_no_error_found;
 }
@@ -2195,7 +2288,6 @@ bool NvdimmsUpdate::runUpdate(void)
 
             for(auto& lid : info.lidIds)
             {
-
                 TRACFCOMP(g_trac_nvdimm,"LID ID=0x%08X, size=%d, vAddr=%p",
                     lid.id, lid.size, lid.vAddr);
 
@@ -2306,7 +2398,7 @@ errlHndl_t NvdimmsUpdate::isUpdateNeeded(bool & o_update_needed,
     uint32_t curType = INVALID_TYPE;
 
     do {
-        const TARGETING::Target * l_dimm = i_cur_image->getNvdimmTarget();
+        TARGETING::Target * l_dimm = i_cur_image->getNvdimmTarget();
 
         // check Types match (same manufacturer and product)
         lidType = i_lid_image->getType();
diff --git a/src/usr/isteps/nvdimm/nvdimm_update.H b/src/usr/isteps/nvdimm/nvdimm_update.H
index 191d683f1..1e48f7ef8 100644
--- a/src/usr/isteps/nvdimm/nvdimm_update.H
+++ b/src/usr/isteps/nvdimm/nvdimm_update.H
@@ -220,10 +220,17 @@ class NvdimmInstalledImage
                           const bool i_force_recollect = false);
 
     /**
+     * @brief Read the current slot that is running
+     * @param o_slot - 0 or 1
+     * @return error if read operation fails
+     */
+    errlHndl_t getRunningSlot(uint8_t & o_slot);
+
+    /**
      * @brief Accessor to grab the current NVDIMM target
      * @return NVDIMM target
      */
-    const TARGETING::Target * getNvdimmTarget(void)
+    TARGETING::Target * getNvdimmTarget(void)
     {
         return iv_dimm;
     }