Added support for checking on the health status of an NVDIMM

* The check for the health status piggy backs off the current
  NVDIMM operation interface.
* Added the method doNvDimmCheckHealthStatus to perform the
  health check status of the individual NVDIMMs.
* Added wrapper methods nvDimmCheckHealthStatusOnSystem and
  nvdimmCheckHealthStatus that call the doNvDimmCheckHealthStatus
* Added an interface in the runtime commands to call make
  the check health status call.

Change-Id: Iefa1fcf5cb6a13c496fd776cdc34ade58ae0612b
CQ:SW469962
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/80589
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Corey V Swenson <cswenson@us.ibm.com>
Reviewed-by: Matt Derksen <mderkse1@us.ibm.com>
Reviewed-by: Daniel M Crowell <dcrowell@us.ibm.com>

6 files changed, 519 insertions, 86 deletions

diff --git a/src/include/runtime/interface.h b/src/include/runtime/interface.h
index 7d66f0ff6..9a0cb478e 100644
--- a/src/include/runtime/interface.h
+++ b/src/include/runtime/interface.h
@@ -579,6 +579,8 @@ typedef struct hostInterfaces
     //       error out.
     enum NVDIMM_Op_t: uint16_t
     {
+        /// The following operations pertain to arming/disarming
+        /// the NVDIMM
         // Disarm the NV logic such that the next save attempt is a NOOP
         HBRT_FW_NVDIMM_DISARM             = 0x0001,
         // Disable encryption on the NVDIMM and clear saved values from FW
@@ -589,6 +591,12 @@ typedef struct hostInterfaces
         HBRT_FW_NVDIMM_ENABLE_ENCRYPTION  = 0x0008,
         // Arm the NV logic
         HBRT_FW_NVDIMM_ARM                = 0x0010,
+
+        /// The following operation pertains to the Health of the NVDIMM
+        /// This operation can be performed with the arming/disarming
+        /// operation, these operation types are orthogonal to each other
+        // Manufacturing energy source(ES) health check request
+        HBRT_FW_MNFG_ES_HEALTH_CHECK      = 0x0020,
     };
 
     // NVDIMM (PHYP -> HBRT) message to request NVDIMM operation(s)
diff --git a/src/include/usr/isteps/nvdimm/nvdimm.H b/src/include/usr/isteps/nvdimm/nvdimm.H
index c07df0420..c1d6b334c 100644
--- a/src/include/usr/isteps/nvdimm/nvdimm.H
+++ b/src/include/usr/isteps/nvdimm/nvdimm.H
@@ -193,6 +193,39 @@ bool nvdimmArm(TARGETING::TargetHandleList &i_nvdimmTargetList);
  */
 bool nvdimmDisarm(TARGETING::TargetHandleList &i_nvdimmTargetList);
 
+/**
+ * @brief Check the health status of the individual NVDIMMs supplied in list
+ *
+ * @details The BPM will trigger the health check when power is applied at the
+ *          beginning of the IPL, with results ready to check about 20 mins
+ *          later. It is the caller's responsibility to ensure enough time has
+ *          passed to make this call.
+ *          Excerpt from the Jedec Standard, Byte Addressable Energy Backed
+ *          Interface of the interested flags (bits 0 .. 2).
+ *          ES_CMD_STATUS0
+ *              Bit 0 : Health Check in Progress
+ *              Bit 1 : Health Check Succeeded
+ *              Bit 2 : Health Check Failed
+ *
+ * @param[in] i_nvdimmTargetList - list of NVDIMMs to check the health of
+ *
+ * @return false if one or more NVDIMMs fail health check, else true
+ */
+bool nvDimmCheckHealthStatus(TARGETING::TargetHandleList &i_nvdimmTargetList);
+
+/**
+ * @brief A wrapper around the call to nvDimmCheckHealthStatus
+ *
+ * @details This will aggregate all the NVDIMMs of the system and pass
+ *          them to the call nvDimmCheckHealthStatus
+ *
+ * @see nvDimmCheckHealthStatus for more details
+ *
+ * @return false if one or more NVDIMMs fail health check, else true
+ */
+bool nvDimmCheckHealthStatusOnSystem();
+
+
 #endif
 
 /**
@@ -241,7 +274,6 @@ errlHndl_t notifyNvdimmProtectionChange(TARGETING::Target* i_target,
  * @param i_target     nvdimm target
  */
 void nvdimm_init(TARGETING::Target *i_nvdimm);
-
 }
 
 #endif  // NVDIMM_EXT_H__
diff --git a/src/include/usr/isteps/nvdimm/nvdimmreasoncodes.H b/src/include/usr/isteps/nvdimm/nvdimmreasoncodes.H
index 938361e54..8ed928a2d 100644
--- a/src/include/usr/isteps/nvdimm/nvdimmreasoncodes.H
+++ b/src/include/usr/isteps/nvdimm/nvdimmreasoncodes.H
@@ -98,6 +98,7 @@ enum nvdimmModuleId
     SET_ATTR_NVDIMM_ENCRYPTION_KEYS_FW      = 0x30,
     SEND_ATTR_NVDIMM_ARMED                  = 0x31,
     NVDIMM_FACTORY_RESET                    = 0x32,
+    NVDIMM_HEALTH_CHECK                     = 0x33,
 };
 
 /**
@@ -161,6 +162,10 @@ enum nvdimmReasonCode
     NVDIMM_ENCRYPTION_MAX_DARN_ERRORS       = NVDIMM_COMP_ID | 0x37,   // Darn random key gen reached max errors
     NVDIMM_ENCRYPTION_BAD_RANDOM_DATA       = NVDIMM_COMP_ID | 0x38,   // Generated key data not valid
     NVDIMM_CANNOT_MAKE_ATTRIBUTE            = NVDIMM_COMP_ID | 0x39,   // Cannot make Attribute
+    NVDIMM_HEALTH_CHECK_IN_PROGRESS_FAILURE = NVDIMM_COMP_ID | 0x3A,   // !< pertains to ES_CMD_STATUS0[0]; the health check in progress flag
+    NVDIMM_HEALTH_CHECK_REPORTED_FAILURE    = NVDIMM_COMP_ID | 0x3B,   // !< pertains to ES_CMD_STATUS0[2]; the health check reported a failure flag
+    NVDIMM_LIFETIME_MIN_REQ_NOT_MET         = NVDIMM_COMP_ID | 0x3C,   // !< pertains to ES_LIFETIME; BPM does not meet minimum requirement for a new BPM
+    NVDIMM_HEALTH_CHECK_NEVER_INITIATED     = NVDIMM_COMP_ID | 0x3D,   // !< A health check was never initiated at start of IPL
 };
 
 enum UserDetailsTypes
diff --git a/src/usr/isteps/nvdimm/runtime/nvdimm_rt.C b/src/usr/isteps/nvdimm/runtime/nvdimm_rt.C
index 97f6efedf..d3a9d41a4 100644
--- a/src/usr/isteps/nvdimm/runtime/nvdimm_rt.C
+++ b/src/usr/isteps/nvdimm/runtime/nvdimm_rt.C
@@ -27,6 +27,9 @@
  *
  *  @brief NVDIMM functions only needed for runtime
  */
+
+/// BPM - Backup Power Module
+
 #include <trace/interface.H>
 #include <errl/errlentry.H>
 #include <errl/errlmanager.H>
@@ -46,6 +49,7 @@
 #define TRACUCOMP(args...)
 
 using namespace TARGETING;
+using namespace ERRORLOG;
 
 namespace NVDIMM
 {
@@ -470,5 +474,322 @@ errlHndl_t nvdimm_getRandom(uint8_t* o_genData)
     return l_err;
 }
 
+/*
+ * @brief Check the health status of the individual NVDIMMs supplied in list
+ *
+ * @param[in] i_nvdimmTargetList - list of NVDIMMs to check the health of
+ *
+ * @return false if one or more NVDIMMs fail health check, else true
+ */
+bool nvDimmCheckHealthStatus(TargetHandleList &i_nvdimmTargetList)
+{
+    TRACFCOMP(g_trac_nvdimm, ENTER_MRK"nvDimmCheckHealthStatus(): "
+              "Target list size(%d)", i_nvdimmTargetList.size());
+
+    // The minimum lifetime value
+    const uint8_t LIFETIME_MINIMUM_REQUIREMENT = 0x62;   // > 97%
+
+    // The health check status flags for the different states of a health check
+    const uint8_t HEALTH_CHECK_IN_PROGRESS_FLAG = 0x01;  // bit 0
+    const uint8_t HEALTH_CHECK_SUCCEEDED_FLAG   = 0x02;  // bit 1
+    const uint8_t HEALTH_CHECK_FAILED_FLAG      = 0x04;  // bit 2
+
+    // Handle to catch any errors
+    errlHndl_t l_err(nullptr);
+
+    // The health check status from a health check call
+    uint8_t l_healthCheck(0);
+
+    // Status of the accumulation of all calls related to the health check.
+    // If any one call is bad/fails, then this will be false, else it stays true
+    bool l_didHealthCheckPass(true);
+
+    // Iterate thru the NVDIMMs checking the health status of each one.
+    // Going with the assumption that the caller waited the allotted time,
+    // roughly 20 to 30 minutes, after the start of an IPL.
+    // Success case:
+    //   * Health check initiated at start of the IPL, caller waited the
+    //     allotted time (20 to 30 mins) before doing a health check, health
+    //     check returned success and the lifetime meets the minimum threshold
+    //     for a new BPM.
+    // Error cases are:
+    //   * Health check is in progress, will assume BPM is hung
+    //   * Health check failed
+    //   * Health check succeeded but lifetime does not meet a certain threshold
+    //   * If none of the above apply (success case and other error cases),
+    //     then assume the health check was never initiated at the start of the
+    //     IPL
+    //   For each of these error cases do a predictive callout
+    for (auto const l_nvdimm : i_nvdimmTargetList)
+    {
+        // Retrieve the Health Check status from the BPM
+        TRACFCOMP(g_trac_nvdimm, INFO_MRK"nvDimmCheckHealthStatus(): "
+                  "Reading NVDIMM(0x%.8X) health check data, "
+                  "register ES_CMD_STATUS0(0x%.2X)",
+                   get_huid(l_nvdimm), ES_CMD_STATUS0);
+
+        l_err = nvdimmReadReg(l_nvdimm, ES_CMD_STATUS0, l_healthCheck);
+
+        if (l_err)
+        {
+            TRACFCOMP(g_trac_nvdimm, ERR_MRK"nvDimmCheckHealthStatus(): "
+                      "NVDIMM(0x%X) failed to read the health check "
+                      "data, register ES_CMD_STATUS0(0x%.2X)",
+                      get_huid(l_nvdimm), ES_CMD_STATUS0);
+
+            l_err->setSev(ERRORLOG::ERRL_SEV_PREDICTIVE);
+            l_err->collectTrace(NVDIMM_COMP_NAME);
+            errlCommit(l_err, NVDIMM_COMP_ID);
+
+            // Let the caller know something went amiss
+            l_didHealthCheckPass = false;
+
+            // Proceed to next NVDIMM, better luck next time
+            continue;
+        }
+
+        // Trace out the returned data for inspection
+        TRACFCOMP(g_trac_nvdimm, INFO_MRK"nvDimmCheckHealthStatus(): "
+                  "NVDIMM(0x%X) returned value(0x%.2X) from health check "
+                  "data, register ES_CMD_STATUS0(0x%.2X)",
+                  get_huid(l_nvdimm), l_healthCheck, ES_CMD_STATUS0)
+
+        if (l_healthCheck & HEALTH_CHECK_IN_PROGRESS_FLAG)
+        {
+            TRACFCOMP( g_trac_nvdimm, ERR_MRK"nvDimmCheckHealthStatus(): "
+                       "Assuming caller waited the allotted time before "
+                       "doing a health check on NVDIMM(0x%.8X), the BPM "
+                       "is hung doing the health check.",
+                       get_huid(l_nvdimm) );
+
+            /*@
+             * @errortype
+             * @severity    ERRL_SEV_PREDICTIVE
+             * @moduleid    NVDIMM_HEALTH_CHECK
+             * @reasoncode  NVDIMM_HEALTH_CHECK_IN_PROGRESS_FAILURE
+             * @userdata1   HUID of NVDIMM target
+             * @userdata2   Health check status
+             * @devdesc     Assuming caller waited the allotted time before
+             *              doing a health check, then the BPM is hung doing
+             *              the health check.
+             * @custdesc    NVDIMM Health Check failed.
+             */
+            l_err = new ErrlEntry( ERRL_SEV_PREDICTIVE,
+                                   NVDIMM_HEALTH_CHECK,
+                                   NVDIMM_HEALTH_CHECK_IN_PROGRESS_FAILURE,
+                                   get_huid(l_nvdimm),
+                                   l_healthCheck,
+                                   ErrlEntry::ADD_SW_CALLOUT );
+            l_err->collectTrace(NVDIMM_COMP_NAME);
+
+            // Add a BPM callout
+            l_err->addPartCallout( l_nvdimm,
+                                   HWAS::BPM_PART_TYPE,
+                                   HWAS::SRCI_PRIORITY_HIGH);
+            // Collect the error
+            errlCommit(l_err, NVDIMM_COMP_ID);
+
+            // Let the caller know something went amiss
+            l_didHealthCheckPass = false;
+        }
+        else if (l_healthCheck & HEALTH_CHECK_FAILED_FLAG)
+        {
+            TRACFCOMP( g_trac_nvdimm, ERR_MRK"nvDimmCheckHealthStatus(): "
+                       "Assuming caller waited the allotted time before "
+                       "doing a health check on NVDIMM(0x%.8X), the BPM "
+                       "reported a failure.",
+                       get_huid(l_nvdimm) );
+
+            /*@
+             * @errortype
+             * @severity    ERRL_SEV_PREDICTIVE
+             * @moduleid    NVDIMM_HEALTH_CHECK
+             * @reasoncode  NVDIMM_HEALTH_CHECK_REPORTED_FAILURE
+             * @userdata1   HUID of NVDIMM target
+             * @userdata2   Health check status
+             * @devdesc     NVDIMM Health Check failed
+             * @devdesc     Assuming caller waited the allotted time before
+             *              doing a health check, the BPM reported a failure
+             *              while doing a health check.
+             * @custdesc    NVDIMM Health Check failed.
+             */
+            l_err = new ErrlEntry( ERRL_SEV_PREDICTIVE,
+                                   NVDIMM_HEALTH_CHECK,
+                                   NVDIMM_HEALTH_CHECK_REPORTED_FAILURE,
+                                   get_huid(l_nvdimm),
+                                   l_healthCheck,
+                                   ErrlEntry::ADD_SW_CALLOUT );
+            l_err->collectTrace(NVDIMM_COMP_NAME);
+
+            // Add a BPM callout
+            l_err->addPartCallout( l_nvdimm,
+                                   HWAS::BPM_PART_TYPE,
+                                   HWAS::SRCI_PRIORITY_HIGH);
+            // Collect the error
+            errlCommit(l_err, NVDIMM_COMP_ID);
+
+            // Let the caller know something went amiss
+            l_didHealthCheckPass = false;
+        }
+        else if (l_healthCheck & HEALTH_CHECK_SUCCEEDED_FLAG)
+        {
+            TRACFCOMP(g_trac_nvdimm, INFO_MRK"nvDimmCheckHealthStatus(): "
+                      "Reading NVDIMM(0x%.8X) es lifetime data, "
+                      "register ES_LIFETIME(0x%.2X)",
+                       get_huid(l_nvdimm), ES_LIFETIME);
+
+            // The lifetime percentage
+            uint8_t l_lifetimePercentage(0);
+
+            // Retrieve the Lifetime Percentage from the BPM
+            l_err = nvdimmReadReg(l_nvdimm, ES_LIFETIME, l_lifetimePercentage);
+
+            if (l_err)
+            {
+                TRACFCOMP( g_trac_nvdimm, ERR_MRK"nvDimmCheckHealthStatus(): "
+                           "NVDIMM(0x%.8X) failed to read the "
+                           "ES_LIFETIME(0x%.2X) data",
+                           get_huid(l_nvdimm),
+                           ES_LIFETIME );
+
+                l_err->setSev(ERRORLOG::ERRL_SEV_PREDICTIVE);
+                l_err->collectTrace(NVDIMM_COMP_NAME);
+                errlCommit(l_err, NVDIMM_COMP_ID);
+
+                // Let the caller know something went amiss
+                l_didHealthCheckPass = false;
+            }
+            else if (l_lifetimePercentage < LIFETIME_MINIMUM_REQUIREMENT)
+            {
+                TRACFCOMP( g_trac_nvdimm, ERR_MRK"nvDimmCheckHealthStatus(): "
+                           "Health check on NVDIMM(0x%.8X) succeeded but the "
+                           "BPM's lifetime(%d) does not meet the minimum "
+                           "requirement(%d) needed to qualify as a new BPM.",
+                            get_huid(l_nvdimm),
+                            l_lifetimePercentage,
+                            LIFETIME_MINIMUM_REQUIREMENT );
+
+                /*@
+                 * @errortype
+                 * @severity         ERRL_SEV_PREDICTIVE
+                 * @moduleid         NVDIMM_HEALTH_CHECK
+                 * @reasoncode       NVDIMM_LIFETIME_MIN_REQ_NOT_MET
+                 * @userdata1[00:31] HUID of NVDIMM target
+                 * @userdata1[32:63] Health check status
+                 * @userdata2[00:31] Retrieved lifetime percentage
+                 * @userdata2[32:63] lifetime minimum requirement
+                 * @devdesc          Health check succeeded but the BPM's
+                 *                   lifetime does not meet the minimum
+                 *                   requirement needed to qualify as a
+                 *                   new BPM.
+                 * @custdesc    NVDIMM Health Check failed
+                 */
+                l_err = new ErrlEntry( ERRL_SEV_PREDICTIVE,
+                                       NVDIMM_HEALTH_CHECK,
+                                       NVDIMM_LIFETIME_MIN_REQ_NOT_MET,
+                                       TWO_UINT32_TO_UINT64(
+                                           get_huid(l_nvdimm),
+                                           l_healthCheck),
+                                       TWO_UINT32_TO_UINT64(
+                                           l_lifetimePercentage,
+                                           LIFETIME_MINIMUM_REQUIREMENT),
+                                       ErrlEntry::ADD_SW_CALLOUT );
+                l_err->collectTrace(NVDIMM_COMP_NAME);
+
+                // Add a BPM callout
+                l_err->addPartCallout( l_nvdimm,
+                                       HWAS::BPM_PART_TYPE,
+                                       HWAS::SRCI_PRIORITY_HIGH);
+                // Collect the error
+                errlCommit(l_err, NVDIMM_COMP_ID);
+
+                // Let the caller know something went amiss
+                l_didHealthCheckPass = false;
+            } // end else if (l_lifetimePercentage ...
+            else
+            {
+                TRACFCOMP( g_trac_nvdimm, ERR_MRK"nvdimmCheckHealthStatus(): "
+                           "Success: Health check on NVDIMM(0x%.8X) succeeded "
+                           "and the BPM's lifetime(%d) meet's the minimum "
+                           "requirement(%d) needed to qualify as a new BPM.",
+                            get_huid(l_nvdimm),
+                            l_lifetimePercentage,
+                            LIFETIME_MINIMUM_REQUIREMENT );
+            }
+        }  // end else if (l_healthCheck & HEALTH_CHECK_SUCCEEDED_FLAG)
+        else  // Assume the health check was never initiated at
+              // the start of the IPL.
+        {
+            TRACFCOMP( g_trac_nvdimm, ERR_MRK"nvDimmCheckHealthStatus(): "
+                       "The health check on NVDIMM(0x%.8X) shows no status (in "
+                       "progress, fail or succeed) so assuming it was never "
+                       "initiated at the start of the IPL.",
+                       get_huid(l_nvdimm) );
+
+            /*@
+             * @errortype
+             * @severity    ERRL_SEV_PREDICTIVE
+             * @moduleid    NVDIMM_HEALTH_CHECK
+             * @reasoncode  NVDIMM_HEALTH_CHECK_NEVER_INITIATED
+             * @userdata1   HUID of NVDIMM target
+             * @userdata2   Health check status
+             * @devdesc     The health check shows no status (in progress, fail
+             *              or succeed) so assuming it was never initiated
+             *              at the start of the IPL.
+             * @custdesc    NVDIMM Health Check failed.
+             */
+            l_err = new ErrlEntry( ERRL_SEV_PREDICTIVE,
+                                   NVDIMM_HEALTH_CHECK,
+                                   NVDIMM_HEALTH_CHECK_NEVER_INITIATED,
+                                   get_huid(l_nvdimm),
+                                   l_healthCheck,
+                                   ErrlEntry::ADD_SW_CALLOUT );
+            l_err->collectTrace(NVDIMM_COMP_NAME);
+
+            // Add a BPM callout
+            l_err->addPartCallout( l_nvdimm,
+                                   HWAS::BPM_PART_TYPE,
+                                   HWAS::SRCI_PRIORITY_HIGH);
+            // Collect the error
+            errlCommit(l_err, NVDIMM_COMP_ID);
+
+            // Let the caller know something went amiss
+            l_didHealthCheckPass = false;
+        }
+    }  // end for (auto const l_nvdimm : i_nvdimmTargetList)
+
+    // Should not have any uncommitted errors
+    assert(l_err == NULL, "nvDimmCheckHealthStatus() - unexpected uncommitted"
+                          "error found" );
+
+    TRACFCOMP(g_trac_nvdimm, EXIT_MRK"nvDimmCheckHealthStatus(): "
+              "Returning %s", l_didHealthCheckPass == true ? "true" : "false" );
+
+    return l_didHealthCheckPass;
+}  // end nvDimmCheckHealthStatus
+
+/**
+ * @brief A wrapper around the call to nvDimmCheckHealthStatus
+ *
+ * @see nvDimmCheckHealthStatus for more details
+ *
+ * @return false if one or more NVDIMMs fail health check, else true
+ */
+bool nvDimmCheckHealthStatusOnSystem()
+{
+    TRACFCOMP(g_trac_nvdimm, ENTER_MRK"nvDimmCheckHealthStatusOnSystem()");
+
+    // Get the list of NVDIMM Targets from the system
+    TargetHandleList l_nvDimmTargetList;
+    nvdimm_getNvdimmList(l_nvDimmTargetList);
+
+    // Return status of doing a check health status
+    bool l_didHealthCheckPass = nvDimmCheckHealthStatus(l_nvDimmTargetList);
+
+    TRACFCOMP(g_trac_nvdimm, EXIT_MRK"nvDimmCheckHealthStatusOnSystem(): "
+              "Returning %s", l_didHealthCheckPass == true ? "true" : "false" );
+
+    return l_didHealthCheckPass;
+}  // end nvDimmCheckHealthStatusOnSystem
 
 } // end NVDIMM namespace
diff --git a/src/usr/util/runtime/rt_cmds.C b/src/usr/util/runtime/rt_cmds.C
index 032559644..c669aae4f 100644
--- a/src/usr/util/runtime/rt_cmds.C
+++ b/src/usr/util/runtime/rt_cmds.C
@@ -1178,6 +1178,27 @@ void cmd_nvdimm_protection_msg( char* &o_output, uint32_t i_huid,
         errlCommit(l_err, UTIL_COMP_ID);
     }
 }
+
+void cmd_nvdimmCheckHealthStatus( char* &o_output)
+{
+    o_output = new char[500];
+    if (NVDIMM::nvDimmCheckHealthStatusOnSystem())
+    {
+        sprintf( o_output, "cmd_doNvDimmCheckHealthStatus: "
+                           "health status check passed.");
+
+    }
+    else
+    {
+        sprintf( o_output, "cmd_doNvDimmCheckHealthStatus: "
+                           "health status check failed. Inspect HBRT traces "
+                           "for further details.");
+
+    }
+
+    return;
+}  // end cmd_nvdimmCheckHealthStatus
+
 #endif
 
 /**
@@ -1514,6 +1535,18 @@ int hbrtCommand( int argc,
             sprintf(*l_output, "ERROR: nvdimm_protection <huid> <0 or 1>");
         }
     }
+    else if( !strcmp( argv[0], "nvdimm_check_status" ) )
+    {
+        if (argc == 1)
+        {
+            cmd_nvdimmCheckHealthStatus( *l_output );
+        }
+        else
+        {
+            *l_output = new char[100];
+            sprintf(*l_output, "Usage: nvdimm_check_status");
+        }
+    }
 #endif
     else
     {
@@ -1554,6 +1587,8 @@ int hbrtCommand( int argc,
 #ifdef CONFIG_NVDIMM
         sprintf( l_tmpstr, "nvdimm_protection <huid> <0 or 1>\n");
         strcat( *l_output, l_tmpstr );
+        sprintf( l_tmpstr, "nvdimm_check_status\n");
+        strcat( *l_output, l_tmpstr );
 #endif
 
     }
diff --git a/src/usr/util/runtime/rt_fwnotify.C b/src/usr/util/runtime/rt_fwnotify.C
index 5b02584c6..252876524 100644
--- a/src/usr/util/runtime/rt_fwnotify.C
+++ b/src/usr/util/runtime/rt_fwnotify.C
@@ -396,7 +396,7 @@ void set_ATTR_NVDIMM_ENCRYPTION_ENABLE(
          *@errortype
          *@reasoncode       RC_CANNOT_MAKE_ATTRIBUTE
          *@severity         ERRORLOG_SEV_PREDICTIVE
-         *@moduleid         SET_ATTR_NVDIMM_ENCRYPTION_ENABLE 
+         *@moduleid         SET_ATTR_NVDIMM_ENCRYPTION_ENABLE
          *@devdesc          Couldn't create an Attribute to send the data
          *                  to the FSP
          *@custdesc         NVDIMM encryption error
@@ -427,16 +427,18 @@ void set_ATTR_NVDIMM_ENCRYPTION_ENABLE(
 
 /**
  *  @brief Perform an NVDIMM operation
- *  @param[in] nvDimmOp - A struct that contains the operation(s) to perform
- *                        and a flag indicating whether to perform operation
- *                        on all processors or a given single processor.
- *  @Note The operations below are in the order of which they should be
- *        performed.  If a new operation is added, make sure it inserted in the
+ *  @param[in] i_nvDimmOp - A struct that contains the operation(s) to perform
+ *                          and a flag indicating whether to perform operation
+ *                          on all processors or a given single processor.
+ *
+ *  @Note The arming/disarming operations below are in the order of which they
+ *        should be performed.  If a new sequence is added to the
+ *        arming/disarming sequence, make sure it is inserted in the
  *        correct order.
  *        The current order is: disarm -> disable encryption -> remove keys ->
  *                              enable encryption -> arm
  **/
-void doNvDimmOperation(const hostInterfaces::nvdimm_operation_t& nvDimmOp)
+void doNvDimmOperation(const hostInterfaces::nvdimm_operation_t& i_nvDimmOp)
 {
 #ifndef CONFIG_NVDIMM
     TRACFCOMP(g_trac_runtime, ENTER_MRK"doNvDimmOperation: not an "
@@ -445,20 +447,21 @@ void doNvDimmOperation(const hostInterfaces::nvdimm_operation_t& nvDimmOp)
 #else
     TRACFCOMP(g_trac_runtime, ENTER_MRK"doNvDimmOperation: Operation(s) "
               "0x%0X, processor ID  0x%0X",
-              nvDimmOp.opType,
-              nvDimmOp.procId);
+              i_nvDimmOp.opType,
+              i_nvDimmOp.procId);
 
     // Error log handle for capturing any errors
     errlHndl_t       l_err{nullptr};
     // List of NVDIMM Targets to execute NVDIMM operation on
     TargetHandleList l_nvDimmTargetList;
 
+    // Perform the operations requested
     do
     {
         /// Populate the NVDIMM target list
         // If requesting to perform operation on all NVDIMMs, then
         // retrieve all NVDIMMs from system
-        if (HBRT_NVDIMM_OPERATION_APPLY_TO_ALL_NVDIMMS == nvDimmOp.procId)
+        if (HBRT_NVDIMM_OPERATION_APPLY_TO_ALL_NVDIMMS == i_nvDimmOp.procId)
         {
             nvdimm_getNvdimmList(l_nvDimmTargetList);
         }
@@ -468,13 +471,13 @@ void doNvDimmOperation(const hostInterfaces::nvdimm_operation_t& nvDimmOp)
             /// Get the NVDIMMs associated with procId
             // Convert the procId to a real boy, uh, I mean target
             TARGETING::TargetHandle_t l_procTarget;
-            l_err = RT_TARG::getHbTarget(nvDimmOp.procId, l_procTarget);
+            l_err = RT_TARG::getHbTarget(i_nvDimmOp.procId, l_procTarget);
             if (l_err)
             {
                 TRACFCOMP(g_trac_runtime, "doNvDimmOperation: Error getting "
                                           "HB Target from processor ID 0x%0X, "
                                           "exiting ...",
-                                          nvDimmOp.procId);
+                                          i_nvDimmOp.procId);
 
                 break;
             }
@@ -491,113 +494,142 @@ void doNvDimmOperation(const hostInterfaces::nvdimm_operation_t& nvDimmOp)
             break;
         }
 
-        /// Perform the operation(s) requested
-        // Disarm the NV logic
-        if (nvDimmOp.opType & hostInterfaces::HBRT_FW_NVDIMM_DISARM)
+        // Perform the arming/disarming operations.  If anyone fails in the
+        // sequence, no point in calling the next, if there is a next operation.
+        do
         {
-            // Make call to disarm
-            if (!nvdimmDisarm(l_nvDimmTargetList))
+            // Disarm the NV logic
+            if (i_nvDimmOp.opType & hostInterfaces::HBRT_FW_NVDIMM_DISARM)
             {
-                TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
-                                          "Call to disarm failed, exiting ...");
-                break;
-            }
-            else
-            {
-                TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
-                                          "Call to disarm succeeded");
+                // Make call to disarm
+                if (!nvdimmDisarm(l_nvDimmTargetList))
+                {
+                    TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
+                              "Call to disarm failed. Will not perform any "
+                              "more arming/disarming calls, if they exist");
+                    break;
+                }
+                else
+                {
+                    TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
+                                              "Call to disarm succeeded");
+                }
             }
-        }
 
-        // Disable encryption on the NVDIMM and clear saved values from FW
-        if (nvDimmOp.opType & hostInterfaces::HBRT_FW_NVDIMM_DISABLE_ENCRYPTION)
-        {
-            // Make call to disable encryption
-            if (!nvdimm_crypto_erase(l_nvDimmTargetList))
+            // Disable encryption on the NVDIMM and clear saved values from FW
+            if (i_nvDimmOp.opType &
+                              hostInterfaces::HBRT_FW_NVDIMM_DISABLE_ENCRYPTION)
             {
-                TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
-                              "Call to disable encryption failed, exiting ...");
+                // Make call to disable encryption
+                if (!nvdimm_crypto_erase(l_nvDimmTargetList))
+                {
+                    TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
+                              "Call to disable encryption failed.  Will not "
+                              "perform any more arming/disarming calls, if "
+                              "they exist");
 
-                // Clear the encryption enable attribute
-                set_ATTR_NVDIMM_ENCRYPTION_ENABLE(0);
+                    // Clear the encryption enable attribute
+                    set_ATTR_NVDIMM_ENCRYPTION_ENABLE(0);
 
-                break;
-            }
-            else
-            {
-                TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
+                    break;
+                }
+                else
+                {
+                    TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
                                        "Call to disable encryption succeeded.");
-            }
-
-            // Clear the encryption enable attribute
-            set_ATTR_NVDIMM_ENCRYPTION_ENABLE(0);
-        }
+                }
 
-        // Remove keys
-        if (nvDimmOp.opType & hostInterfaces::HBRT_FW_NVDIMM_REMOVE_KEYS)
-        {
-            // Make call to remove keys
-            if (!nvdimm_remove_keys())
-            {
-                TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
-                                     "Call to remove keys failed, exiting ...");
-                break;
+                // Clear the encryption enable attribute
+                set_ATTR_NVDIMM_ENCRYPTION_ENABLE(0);
             }
-            else
+
+            // Remove keys
+            if (i_nvDimmOp.opType & hostInterfaces::HBRT_FW_NVDIMM_REMOVE_KEYS)
             {
-                TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
-                                          "Call to remove keys succeeded.");
+                // Make call to remove keys
+                if (!nvdimm_remove_keys())
+                {
+                    TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
+                              "Call to remove keys failed.  Will not perform "
+                              "any more arming/disarming calls, if they exist");
+                    break;
+                }
+                else
+                {
+                    TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
+                                              "Call to remove keys succeeded.");
+                }
             }
-        }
 
-        // Enable encryption on the NVDIMM
-        if (nvDimmOp.opType & hostInterfaces::HBRT_FW_NVDIMM_ENABLE_ENCRYPTION)
-        {
-            // Set the encryption enable attribute
-            set_ATTR_NVDIMM_ENCRYPTION_ENABLE(1);
-
-            // Make call to generate keys before enabling encryption
-            if(!nvdimm_gen_keys())
+            // Enable encryption on the NVDIMM
+            if (i_nvDimmOp.opType &
+                               hostInterfaces::HBRT_FW_NVDIMM_ENABLE_ENCRYPTION)
             {
-                TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
-                                          "Call to generate keys failed, unable"
-                                          " to enable encryption, exiting ...");
-                break;
+                // Set the encryption enable attribute
+                set_ATTR_NVDIMM_ENCRYPTION_ENABLE(1);
+
+                // Make call to generate keys before enabling encryption
+                if(!nvdimm_gen_keys())
+                {
+                    TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
+                              "Call to generate keys failed, unable to enable "
+                              "encryption. Will not perform any more "
+                              "arming/disarming calls, if they exist");
+                    break;
+                }
+                else
+                {
+                    // Make call to enable encryption
+                    if (!nvdimm_encrypt_enable(l_nvDimmTargetList))
+                    {
+                        TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
+                                  "Call to enable encryption failed. "
+                                  "Will not perform any more arming/disarming "
+                                  "calls, if they exist");
+                        break;
+                    }
+                    else
+                    {
+                        TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
+                                        "Call to enable encryption succeeded.");
+                    }
+                } // end if(!nvdimm_gen_keys()) ... else ...
             }
-            else
+
+            // Arm the NV logic
+            if (i_nvDimmOp.opType & hostInterfaces::HBRT_FW_NVDIMM_ARM)
             {
-                // Make call to enable encryption
-                if (!nvdimm_encrypt_enable(l_nvDimmTargetList))
+                // Make call to arm
+                if (!nvdimmArm(l_nvDimmTargetList))
                 {
                     TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
-                               "Call to enable encryption failed, exiting ...");
+                                              "Call to arm failed.");
                     break;
                 }
                 else
                 {
                     TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
-                                        "Call to enable encryption succeeded.");
+                                              "Call to arm succeeded.");
                 }
-            } // end if(!nvdimm_gen_keys()) ... else ...
-        }
+            }  // end if (nvDimmOp.opType & hostInterfaces::HBRT_FW_NVDIMM_ARM)
+        }  while (0); // end Perform the arming/disarming operations.
 
-        // Arm the NV logic
-        if (nvDimmOp.opType & hostInterfaces::HBRT_FW_NVDIMM_ARM)
+        // Perform the health check operation
+        if (i_nvDimmOp.opType & hostInterfaces::HBRT_FW_MNFG_ES_HEALTH_CHECK)
         {
-            // Make call to arm
-            if (!nvdimmArm(l_nvDimmTargetList))
+            if (!nvDimmCheckHealthStatus(l_nvDimmTargetList))
             {
                 TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
-                                          "Call to arm failed, exiting ...");
+                                          "Call to do a health check failed.");
                 break;
             }
             else
             {
                 TRACFCOMP(g_trac_runtime, "doNvDimmOperation: "
-                                          "Call to arm succeeded.");
+                                        "Call to do a health check succeeded.");
             }
-        }  // end if (nvDimmOp.opType & hostInterfaces::HBRT_FW_NVDIMM_ARM)
-    } while(0);
+        }
+    } while(0); // end Perform the operations requested
 
     if (l_err)
     {