ERRL: Hardware Callouts

Support for HW and Procedure Callouts, including GARD and
deconfigure information.

 . Add errl functions to get the information;
 . Add errlog userdetail code to package callout data;
 . Add to errlog commit function to find callout user-detail
    sections and call a HWAS:common function to process
    the callout.
 . Remove gardLevel from Targeting hwasState field

Change-Id: I01c7c3721e6fd843a0ba2ad4df8399f6db68b590
RTC: 41276
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/1266
Tested-by: Jenkins Server
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

14 files changed, 680 insertions, 169 deletions

diff --git a/src/usr/errl/errlentry.C b/src/usr/errl/errlentry.C
index 5f0fa4fbf..992f0a0c1 100644
--- a/src/usr/errl/errlentry.C
+++ b/src/usr/errl/errlentry.C
@@ -37,8 +37,11 @@
 #include <errl/errlentry.H>
 #include <errl/errlmanager.H>
 #include <errl/errludbacktrace.H>
+#include <errl/errludcallout.H>
 #include <trace/interface.H>
 #include <arch/ppc.H>
+#include <hwas/common/hwasCallout.H>
+#include <targeting/common/targetservice.H>
 
 namespace ERRORLOG
 {
@@ -245,6 +248,48 @@ void ErrlEntry::removeBackTrace()
     iv_pBackTrace = NULL;
 }
 
+
+////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////
+void ErrlEntry::addHwCallout(const TARGETING::Target *i_target,
+                        const HWAS::callOutPriority i_priority,
+                        const HWAS::DeconfigEnum i_deconfigState,
+                        const HWAS::GARD_ErrorType i_gardErrorType)
+{
+    TRACFCOMP(g_trac_errl, ENTER_MRK"addHwCallout(%p, 0x%x)",
+                i_target, i_priority);
+
+    if (i_target == TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL)
+    {
+        ErrlUserDetailsCallout(
+                &HWAS::TARGET_IS_SENTINEL, sizeof(HWAS::TARGET_IS_SENTINEL),
+                i_priority, i_deconfigState, i_gardErrorType).addToLog(this);
+    }
+    else
+    {   // we got a non MASTER_SENTINEL target, therefore the targeting
+        // module is loaded, therefore we can make this call.
+        TARGETING::EntityPath ep;
+        ep = i_target->getAttr<TARGETING::ATTR_PHYS_PATH>();
+
+        ErrlUserDetailsCallout(&ep, sizeof(ep),
+                i_priority, i_deconfigState, i_gardErrorType).addToLog(this);
+    }
+} // addHwCallout
+
+
+////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////
+void ErrlEntry::addProcedureCallout(const HWAS::epubProcedureID i_procedure,
+                            const HWAS::callOutPriority i_priority)
+{
+    TRACDCOMP( g_trac_errl, ENTER_MRK"addProcedureCallout(0x%x, 0x%x)",
+                i_procedure, i_priority);
+
+    ErrlUserDetailsCallout(i_procedure, i_priority).addToLog(this);
+
+} // addProcedureCallout
+
+
 ////////////////////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////////
 errlSeverity_t ErrlEntry::sev() const
@@ -253,7 +298,6 @@ errlSeverity_t ErrlEntry::sev() const
 }
 
 
-
 ////////////////////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////////
 void ErrlEntry::setSev(const errlSeverity_t i_sev)
@@ -288,7 +332,6 @@ epubSubSystem_t ErrlEntry::subSys() const
 }
 
 
-
 ////////////////////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////////
 void ErrlEntry::setSubSys(const epubSubSystem_t i_subSys)
@@ -297,6 +340,7 @@ void ErrlEntry::setSubSys(const epubSubSystem_t i_subSys)
     return;
 }
 
+
 ///////////////////////////////////////////////////////////////////////////////
 // for use by ErrlManager
 void ErrlEntry::commit( compId_t  i_committerComponent )
@@ -308,6 +352,41 @@ void ErrlEntry::commit( compId_t  i_committerComponent )
     // User header contains the component ID of the committer.
     iv_User.setComponentId( i_committerComponent );
 
+    // see if HWAS has been loaded and has set the processCallout function
+    // TODO RTC 46680
+    // If the PNOR resource provider commits an error then this function will
+    // call HWAS to process the callouts/deconfigure/GARD requests in the
+    // error log, if the HWAS function is not paged into memory then VMM will
+    // attempt to page it in which will invoke the PNOR resource provider,
+    // this will deadlock if the PNOR resource provider is waiting for the
+    // error log commit to complete before processing further requests. There
+    // may be other similar deadlock scenarios involving 'base' functions
+    // logging errors. This can be solved with a separate errl-manager task
+    // that processes error logs
+    HWAS::processCalloutFn pFn;
+    pFn = ERRORLOG::theErrlManager::instance().getHwasProcessCalloutFn();
+    if (pFn != NULL)
+    {
+        // look thru the errlog for any Callout UserDetail sections
+        for(std::vector<ErrlUD*>::iterator it = iv_SectionVector.begin();
+                it != iv_SectionVector.end();
+                it++ )
+        {
+            // if this is a CALLOUT
+            if ((HBERRL_COMP_ID     == (*it)->iv_header.iv_compId) &&
+                (1                  == (*it)->iv_header.iv_ver) &&
+                (HBERRL_UDT_CALLOUT == (*it)->iv_header.iv_sst))
+            {
+                // call HWAS to have this processed
+                (*pFn)(plid(),(*it)->iv_pData, (*it)->iv_Size);
+            }
+        } // for each SectionVector
+    } // if HWAS module loaded
+    else
+    {
+        TRACFCOMP(g_trac_errl, INFO_MRK"hwas processCalloutFn not set!");
+    }
+
     // Add the captured backtrace to the error log
     if (iv_pBackTrace)
     {
@@ -354,7 +433,7 @@ uint64_t ErrlEntry::flatten( void * o_pBuffer,  uint64_t i_bufsize )
         if ( i_bufsize < l_flatCount )
         {
             // buffer is not big enough; return zero
-            TRACFCOMP( ERRORLOG::g_trac_errl, ERR_MRK"Invalid buffer size");
+            TRACFCOMP( g_trac_errl, ERR_MRK"Invalid buffer size");
             l_flatCount = 0;
             break;
         }
diff --git a/src/usr/errl/errlmanager.C b/src/usr/errl/errlmanager.C
index 45a1fd3ca..c615aeeba 100644
--- a/src/usr/errl/errlmanager.C
+++ b/src/usr/errl/errlmanager.C
@@ -1,25 +1,26 @@
-//  IBM_PROLOG_BEGIN_TAG
-//  This is an automatically generated prolog.
-//
-//  $Source: src/usr/errl/errlmanager.C $
-//
-//  IBM CONFIDENTIAL
-//
-//  COPYRIGHT International Business Machines Corp. 2011
-//
-//  p1
-//
-//  Object Code Only (OCO) source materials
-//  Licensed Internal Code Source Materials
-//  IBM HostBoot Licensed Internal Code
-//
-//  The source code for this program is not published or other-
-//  wise divested of its trade secrets, irrespective of what has
-//  been deposited with the U.S. Copyright Office.
-//
-//  Origin: 30
-//
-//  IBM_PROLOG_END
+/*  IBM_PROLOG_BEGIN_TAG
+ *  This is an automatically generated prolog.
+ *
+ *  $Source: src/usr/errl/errlmanager.C $
+ *
+ *  IBM CONFIDENTIAL
+ *
+ *  COPYRIGHT International Business Machines Corp. 2011-2012
+ *
+ *  p1
+ *
+ *  Object Code Only (OCO) source materials
+ *  Licensed Internal Code Source Materials
+ *  IBM HostBoot Licensed Internal Code
+ *
+ *  The source code for this program is not published or other-
+ *  wise divested of its trade secrets, irrespective of what has
+ *  been deposited with the U.S. Copyright Office.
+ *
+ *  Origin: 30
+ *
+ *  IBM_PROLOG_END_TAG
+ */
 /**
  *  @file errlmanager.C
  *
@@ -81,7 +82,10 @@ ErrlManager::ErrlManager()
     // PNOR will be reinitialized every time hostboot runs
     iv_currLogId = 0;
 
-    mutex_init(&iv_mutex);
+    iv_hwasProcessCalloutFn = NULL;
+
+    mutex_init(&iv_commitMutex);
+    mutex_init(&iv_hwasMutex);
 
     // Scaffolding.
     // For now, put error logs in a 64KB buffer in L3 RAM
@@ -132,7 +136,7 @@ void ErrlManager::commitErrLog(errlHndl_t& io_err, compId_t i_committerComp )
         TRACFCOMP(g_trac_errl, "commitErrLog() called by %.4X for plid=0x%X, Reasoncode=%.4X", i_committerComp, io_err->plid(), io_err->reasonCode() );
 
         // lock sem
-        mutex_lock(&iv_mutex);
+        mutex_lock(&iv_commitMutex);
 
         // Ask the ErrlEntry to assign commit component, commit time, etc.
         io_err->commit( i_committerComp  );
@@ -174,7 +178,7 @@ void ErrlManager::commitErrLog(errlHndl_t& io_err, compId_t i_committerComp )
 
 
         // unlock sem
-        mutex_unlock(&iv_mutex);
+        mutex_unlock(&iv_commitMutex);
 
         delete io_err;
         io_err = NULL;
@@ -194,6 +198,26 @@ uint32_t ErrlManager::getUniqueErrId()
 
 ///////////////////////////////////////////////////////////////////////////////
 ///////////////////////////////////////////////////////////////////////////////
+void ErrlManager::setHwasProcessCalloutFn(HWAS::processCalloutFn i_fn)
+{
+    mutex_lock(&(ERRORLOG::theErrlManager::instance().iv_hwasMutex));
+    ERRORLOG::theErrlManager::instance().iv_hwasProcessCalloutFn = i_fn;
+    mutex_unlock(&(ERRORLOG::theErrlManager::instance().iv_hwasMutex));
+}
+
+///////////////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
+HWAS::processCalloutFn ErrlManager::getHwasProcessCalloutFn() const
+{
+    HWAS::processCalloutFn l_fp = NULL;
+    mutex_lock(&iv_hwasMutex);
+    l_fp = iv_hwasProcessCalloutFn;
+    mutex_unlock(&iv_hwasMutex);
+    return l_fp;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
 // Global function (not a method on an object) to commit the error log.
 void errlCommit(errlHndl_t& io_err, compId_t i_committerComp )
 {
diff --git a/src/usr/errl/errludcallout.C b/src/usr/errl/errludcallout.C
new file mode 100644
index 000000000..aa6b61204
--- /dev/null
+++ b/src/usr/errl/errludcallout.C
@@ -0,0 +1,102 @@
+/*  IBM_PROLOG_BEGIN_TAG
+ *  This is an automatically generated prolog.
+ *
+ *  $Source: src/usr/errl/errludcallout.C $
+ *
+ *  IBM CONFIDENTIAL
+ *
+ *  COPYRIGHT International Business Machines Corp. 2012
+ *
+ *  p1
+ *
+ *  Object Code Only (OCO) source materials
+ *  Licensed Internal Code Source Materials
+ *  IBM HostBoot Licensed Internal Code
+ *
+ *  The source code for this program is not published or other-
+ *  wise divested of its trade secrets, irrespective of what has
+ *  been deposited with the U.S. Copyright Office.
+ *
+ *  Origin: 30
+ *
+ *  IBM_PROLOG_END_TAG
+ */
+/**
+ *  @file errludcallout.C
+ *
+ *  @brief Implementation of ErrlUserDetailsCallout
+ */
+#include <sys/task.h>
+#include <errl/errludcallout.H>
+
+#include <targeting/common/targetservice.H>
+#include <targeting/common/util.H>
+#include <targeting/common/trace.H>
+
+namespace ERRORLOG
+{
+
+extern TARGETING::TARG_TD_t g_trac_errl;
+
+//------------------------------------------------------------------------------
+// Hardware callout
+ErrlUserDetailsCallout::ErrlUserDetailsCallout(
+        const void *i_pTargetData,
+        uint32_t i_targetDataLength,
+        const HWAS::callOutPriority i_priority,
+        const HWAS::DeconfigEnum i_deconfigState,
+        const HWAS::GARD_ErrorType i_gardErrorType)
+{
+    TRACDCOMP(g_trac_errl, "HWCallout entry");
+
+    // Set up ErrlUserDetails instance variables
+    iv_CompId = HBERRL_COMP_ID;
+    iv_Version = 1;
+    iv_SubSection = HBERRL_UDT_CALLOUT;
+
+    //iv_merge = false; // use the default of false
+
+    uint32_t pDataLength = sizeof(HWAS::callout_ud_t) + i_targetDataLength;
+    HWAS::callout_ud_t *pData;
+    pData = reinterpret_cast<HWAS::callout_ud_t *>
+                (reallocUsrBuf(pDataLength));
+    pData->type = HWAS::HW_CALLOUT;
+    pData->priority = i_priority;
+    pData->cpuid = task_getcpuid();
+    pData->deconfigState = i_deconfigState;
+    pData->gardErrorType = i_gardErrorType;
+    memcpy(pData + 1, i_pTargetData, i_targetDataLength);
+
+    TRACDCOMP(g_trac_errl, "HWCallout exit; pDataLength %d", pDataLength);
+
+} // Hardware callout
+
+
+//------------------------------------------------------------------------------
+// Procedure callout
+ErrlUserDetailsCallout::ErrlUserDetailsCallout(
+        const HWAS::epubProcedureID i_procedure,
+        const HWAS::callOutPriority i_priority)
+{
+    TRACDCOMP(g_trac_errl, "Procedure Callout");
+
+    // Set up ErrlUserDetails instance variables
+    iv_CompId = HBERRL_COMP_ID;
+    iv_Version = 1;
+    iv_SubSection = HBERRL_UDT_CALLOUT;
+
+    //iv_merge = false; // use the default of false
+
+    HWAS::callout_ud_t *pData;
+    pData = reinterpret_cast<HWAS::callout_ud_t *>
+                (reallocUsrBuf(sizeof(HWAS::callout_ud_t)));
+
+    pData->type = HWAS::PROCEDURE_CALLOUT;
+    pData->procedure = i_procedure;
+    pData->priority = i_priority;
+
+    TRACDCOMP(g_trac_errl, "Procedure Callout exit");
+
+} // Procedure callout
+
+}
diff --git a/src/usr/errl/makefile b/src/usr/errl/makefile
index 72f9ab76f..7ce89dc70 100644
--- a/src/usr/errl/makefile
+++ b/src/usr/errl/makefile
@@ -1,11 +1,11 @@
-#  IBM_PROLOG_BEGIN_TAG
+#   IBM_PROLOG_BEGIN_TAG
 #  This is an automatically generated prolog.
 #
 #  $Source: src/usr/errl/makefile $
 #
 #  IBM CONFIDENTIAL
 #
-#  COPYRIGHT International Business Machines Corp. 2011
+#  COPYRIGHT International Business Machines Corp. 2011-2012
 #
 #  p1
 #
@@ -19,13 +19,14 @@
 #
 #  Origin: 30
 #
-#  IBM_PROLOG_END
+#  IBM_PROLOG_END_TAG
 ROOTPATH = ../../..
 MODULE = errl
 
 OBJS = errlentry.o errlmanager.o errlsctn.o errlsctnhdr.o errlprvt.o errluh.o \
        errlud.o errlsrc.o errluserdetails.o backtrace.o errludtarget.o \
-       errludstring.o errludbacktrace.o errludattribute.o errludlogregister.o
+       errludstring.o errludbacktrace.o errludattribute.o errludlogregister.o \
+       errludcallout.o
 
 SUBDIRS = test.d  parser.d
 
diff --git a/src/usr/errl/test/errltest.H b/src/usr/errl/test/errltest.H
index 2abca0d09..31d128b75 100644
--- a/src/usr/errl/test/errltest.H
+++ b/src/usr/errl/test/errltest.H
@@ -1,25 +1,26 @@
-//  IBM_PROLOG_BEGIN_TAG
-//  This is an automatically generated prolog.
-//
-//  $Source: src/usr/errl/test/errltest.H $
-//
-//  IBM CONFIDENTIAL
-//
-//  COPYRIGHT International Business Machines Corp. 2011
-//
-//  p1
-//
-//  Object Code Only (OCO) source materials
-//  Licensed Internal Code Source Materials
-//  IBM HostBoot Licensed Internal Code
-//
-//  The source code for this program is not published or other-
-//  wise divested of its trade secrets, irrespective of what has
-//  been deposited with the U.S. Copyright Office.
-//
-//  Origin: 30
-//
-//  IBM_PROLOG_END
+/*  IBM_PROLOG_BEGIN_TAG
+ *  This is an automatically generated prolog.
+ *
+ *  $Source: src/usr/errl/test/errltest.H $
+ *
+ *  IBM CONFIDENTIAL
+ *
+ *  COPYRIGHT International Business Machines Corp. 2011-2012
+ *
+ *  p1
+ *
+ *  Object Code Only (OCO) source materials
+ *  Licensed Internal Code Source Materials
+ *  IBM HostBoot Licensed Internal Code
+ *
+ *  The source code for this program is not published or other-
+ *  wise divested of its trade secrets, irrespective of what has
+ *  been deposited with the U.S. Copyright Office.
+ *
+ *  Origin: 30
+ *
+ *  IBM_PROLOG_END_TAG
+ */
 #ifndef __ERRLTEST_H
 #define __ERRLTEST_H
 
@@ -36,6 +37,12 @@
 #include <trace/trace.H>
 #include <hbotcompid.H>
 
+#include <errl/errludtarget.H>
+#include <targeting/common/target.H>
+#include <targeting/common/iterators/rangefilter.H>
+#include <targeting/common/predicates/predicates.H>
+#include <hwas/common/hwasCallout.H>
+
 
 #define TEST_SEVERITY       ERRORLOG::ERRL_SEV_INFORMATIONAL
 
@@ -331,6 +338,79 @@ public:
 
         }
     }
+
+    /**
+     * @brief Test callouts
+     */
+    void testErrl3(void)
+    {
+        TS_TRACE( "test testErrl3");
+        do
+        {
+            // Create an error log
+            errlHndl_t errl = new ERRORLOG::ErrlEntry(
+                                    ERRORLOG::ERRL_SEV_UNRECOVERABLE,
+                                    HBERRL_TEST_MOD_ID,
+                                    HBERRL_TEST_REASON_CODE);
+
+            // Make sure log is created
+            if (errl == NULL)
+            {
+                TS_FAIL("testErrl3: createErrlLog() returns NULL pointer!");
+                break;
+            }
+
+            // test the different callout types
+
+            // find a proc target
+            TARGETING::PredicateCTM procChipFilter(
+                    TARGETING::CLASS_CHIP, TARGETING::TYPE_PROC);
+            TARGETING::TargetRangeFilter pProc(
+                    TARGETING::targetService().begin(),
+                    TARGETING::targetService().end(),
+                    &procChipFilter);
+
+            // find a membuf target
+            TARGETING::PredicateCTM membufChipFilter(
+                    TARGETING::CLASS_CHIP,TARGETING::TYPE_MEMBUF);
+            TARGETING::TargetRangeFilter pMembuf(
+                    TARGETING::targetService().begin(),
+                    TARGETING::targetService().end(),
+                    &membufChipFilter);
+
+            TS_TRACE( "test callout pProc %p", *pProc);
+            ERRORLOG::ErrlUserDetailsTarget(*pProc).addToLog(errl);
+
+            errl->addHwCallout(*pProc,
+                    HWAS::SRCI_PRIORITY_LOW,
+                    HWAS::DECONFIG,
+                    HWAS::GARD_Fatal);
+
+            errl->addHwCallout(*pMembuf,
+                    HWAS::SRCI_PRIORITY_MED,
+                    HWAS::DECONFIG,
+                    HWAS::GARD_NULL);
+
+            errl->addHwCallout(TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL,
+                    HWAS::SRCI_PRIORITY_LOW,
+                    HWAS::DECONFIG,
+                    HWAS::GARD_PoreError);
+
+            errl->addHwCallout(TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL,
+                    HWAS::SRCI_PRIORITY_MED,
+                    HWAS::DECONFIG,
+                    HWAS::GARD_NULL);
+
+            errl->addProcedureCallout(
+                    HWAS::EPUB_PRC_MEMORY_UE,
+                    HWAS::SRCI_PRIORITY_HIGH);
+
+            errlCommit(errl, CXXTEST_COMP_ID);
+            TS_TRACE( "test callout done");
+
+        }
+        while(0);
+    }
 };
 
 #endif
diff --git a/src/usr/errl/test/errluserdetailtest.H b/src/usr/errl/test/errluserdetailtest.H
index 2adeefd86..34401ae35 100644
--- a/src/usr/errl/test/errluserdetailtest.H
+++ b/src/usr/errl/test/errluserdetailtest.H
@@ -39,6 +39,7 @@
 #include <errl/errludbacktrace.H>
 #include <errl/errludtarget.H>
 #include <errl/errludlogregister.H>
+#include <errl/errludcallout.H>
 #include <errludattribute.H>
 #include <targeting/common/targetservice.H>
 #include <targeting/common/iterators/rangefilter.H>
@@ -445,6 +446,74 @@ public:
         errlCommit(errl, CXXTEST_COMP_ID);
         TS_TRACE( "testLogRegister errorlog user detail data - complete");
     } // testLogRegister
+
+    /**
+     * @test testCallout - Capture a callout
+     */
+    void testCallout(void)
+    {
+        errlHndl_t errl = NULL;
+
+        TS_TRACE( "testCallout errorlog user detail data");
+        /*@
+        * @errortype
+        * @severity ERRORLOG_SEV_INFORMATIONAL
+        * @moduleid HBERRL_USERDATA_TEST_MOD_ID
+        * @reasoncode HBERRL_TEST_CALLOUT_UD
+        * @userdata1 Test data 1
+        * @userdata2 Test data 2
+        * @devdesc User Details unit test - create callout user detail data
+        */
+        errl = new ERRORLOG::ErrlEntry(
+                ERRORLOG::ERRL_SEV_INFORMATIONAL,
+                HBERRL_USERDATA_TEST_MOD_ID,
+                HBERRL_TEST_CALLOUT_UD,
+                0x0001002300450067,                   //  user1
+                0x008900AB00CD00EF );                 //  user2
+
+        using namespace TARGETING;
+        ErrlUserDetailsString("Callout test").addToLog(errl);
+
+        // first test with Master
+        ErrlUserDetailsCallout(
+                    &HWAS::TARGET_IS_SENTINEL,
+                    sizeof(HWAS::TARGET_IS_SENTINEL),
+                    HWAS::SRCI_PRIORITY_LOW,
+                    HWAS::DECONFIG,
+                    HWAS::GARD_PoreError).addToLog(errl);
+
+        ErrlUserDetailsCallout(
+                    &HWAS::TARGET_IS_SENTINEL,
+                    sizeof(HWAS::TARGET_IS_SENTINEL),
+                    HWAS::SRCI_PRIORITY_MED,
+                    HWAS::DECONFIG,
+                    HWAS::GARD_NULL).addToLog(errl);
+
+        // find a proc target
+        PredicateCTM procChipFilter(CLASS_CHIP,TYPE_PROC);
+        TargetRangeFilter pProc(
+            targetService().begin(), targetService().end(),
+            &procChipFilter);
+        TARGETING::EntityPath ep;
+        ep = pProc->getAttr<TARGETING::ATTR_PHYS_PATH>();
+
+        ErrlUserDetailsCallout(
+                    &ep,
+                    sizeof(ep),
+                    HWAS::SRCI_PRIORITY_LOW,
+                    HWAS::DECONFIG,
+                    HWAS::GARD_Fatal).addToLog(errl);
+
+        ErrlUserDetailsCallout(
+                    HWAS::EPUB_PRC_MEMORY_UE,
+                    HWAS::SRCI_PRIORITY_HIGH).addToLog(errl);
+
+        // commit the errorlog
+        errlCommit(errl, CXXTEST_COMP_ID);
+        TS_TRACE( "testCallout done");
+
+    } // testCallout
+
 };
 
 #endif
diff --git a/src/usr/hwas/common/hwas.C b/src/usr/hwas/common/hwas.C
index 9080f64c0..3d3ab10ca 100644
--- a/src/usr/hwas/common/hwas.C
+++ b/src/usr/hwas/common/hwas.C
@@ -88,7 +88,6 @@ errlHndl_t discoverTargets()
         hwasState.present               = false;
         hwasState.functional            = false;
         hwasState.changedSinceLastIPL   = false;
-        hwasState.gardLevel             = 0;
         target->setAttr<ATTR_HWAS_STATE>(hwasState);
     }
 
diff --git a/src/usr/hwas/common/hwasCallout.C b/src/usr/hwas/common/hwasCallout.C
new file mode 100644
index 000000000..6d90c3b6d
--- /dev/null
+++ b/src/usr/hwas/common/hwasCallout.C
@@ -0,0 +1,175 @@
+/*  IBM_PROLOG_BEGIN_TAG
+ *  This is an automatically generated prolog.
+ *
+ *  $Source: src/usr/hwas/common/hwasCallout.C $
+ *
+ *  IBM CONFIDENTIAL
+ *
+ *  COPYRIGHT International Business Machines Corp. 2012
+ *
+ *  p1
+ *
+ *  Object Code Only (OCO) source materials
+ *  Licensed Internal Code Source Materials
+ *  IBM HostBoot Licensed Internal Code
+ *
+ *  The source code for this program is not published or other-
+ *  wise divested of its trade secrets, irrespective of what has
+ *  been deposited with the U.S. Copyright Office.
+ *
+ *  Origin: 30
+ *
+ *  IBM_PROLOG_END_TAG
+ */
+/**
+ *  @file hwasCallout.C
+ *
+ *  HardWare Availability Service Callout functions.
+ *
+ */
+
+
+/******************************************************************************/
+// Includes
+/******************************************************************************/
+#include <stdint.h>
+
+#include <hwas/common/hwasCallout.H>
+#include <hwas/common/hwasCommon.H>
+#include <hwas/common/hwas_reasoncodes.H>
+#include <hwas/deconfigGard.H>
+#include <targeting/common/attributes.H>
+#include <errl/errlmanager.H>
+
+namespace HWAS
+{
+
+class RegisterHWASFunctions
+{
+    public:
+    RegisterHWASFunctions()
+    {
+        // HWAS is awake - register our processCallout function
+        HWAS_DBG("module load: calling errlog::setHwasProcessCalloutFn");
+        ERRORLOG::ErrlManager::setHwasProcessCalloutFn((processCalloutFn)(&processCallout));
+    }
+};
+// this causes the function to get run at module load.
+RegisterHWASFunctions registerHWASFunctions;
+
+void processCallout(const uint32_t i_errlPlid,
+        uint8_t *i_pData,
+        uint64_t i_Size)
+{
+    HWAS_INF("processCallout entry. plid 0x%x data %p %lld",
+            i_errlPlid, i_pData, i_Size);
+
+    callout_ud_t *pCalloutUD = (callout_ud_t *)i_pData;
+    errlHndl_t errl = NULL;
+    switch (pCalloutUD->type)
+    {
+        case (HW_CALLOUT):
+        {
+            TARGETING::Target *pTarget;
+            // data after the pCalloutUD structure is either a token
+            //  indicating it's the MASTER_PROCESSOR_CHIP_TARGET_SENTINEL
+            //  or it's the EntityPath - getAttr<TARGETING::ATTR_PHYS_PATH>()
+            if (*((uint8_t *)(pCalloutUD + 1)) == TARGET_IS_SENTINEL)
+            {
+                pTarget = TARGETING::MASTER_PROCESSOR_CHIP_TARGET_SENTINEL;
+                // TODO RTC: 45780
+                HWAS_INF("TARGET_SENTINEL - can't handle; exiting");
+                break; // HWAS can't handle this right now... 
+            }
+            else
+            {
+                // convert the EntityPath to a Target pointer
+                TARGETING::EntityPath ep;
+                memcpy(&ep, (pCalloutUD + 1), sizeof(ep));
+                pTarget = TARGETING::targetService().toTarget(ep);
+            }
+
+            if (pTarget == NULL)
+            {   // should only happen if we have a corrupt errlog or targeting.
+                HWAS_ERR("HW callout; pTarget was NULL!!!");
+
+                /*@
+                 * @errortype
+                 * @moduleid     MOD_PROCESS_CALLOUT
+                 * @reasoncode   INVALID_TARGET
+                 * @devdesc      Invalid Target encountered into processCallout
+                 */
+                errl = new ERRORLOG::ErrlEntry(
+                    ERRORLOG::ERRL_SEV_INFORMATIONAL,
+                    MOD_PROCESS_CALLOUT,
+                    RC_INVALID_TARGET);
+                errlCommit(errl, HWAS_COMP_ID);
+                break;
+            }
+
+            HWAS_INF("HW callout; pTarget %p", pTarget);
+            const DeconfigEnum deconfigState = pCalloutUD->deconfigState;
+            const GARD_ErrorType gardErrorType = pCalloutUD->gardErrorType;
+            //const callOutPriority priority = pCalloutUD->priority;
+            switch (gardErrorType)
+            {
+                case (GARD_NULL):
+                {   // means no GARD operations
+                    break;
+                }
+                default:
+                {
+                    // call HWAS common function
+                    errl = HWAS::theDeconfigGard().createGardRecord(*pTarget,
+                            i_errlPlid,
+                            GARD_Fatal);
+                    break;
+                }
+            } // switch gardErrorType
+            switch (deconfigState)
+            {
+                case (NO_DECONFIG):
+                {
+                    break;
+                }
+                case (DECONFIG):
+                {
+                    // call HWAS common function
+                    errl = HWAS::theDeconfigGard().deconfigureTarget(*pTarget,
+                                i_errlPlid);
+                    break;
+                }
+                case (DELAYED_DECONFIG):
+                {
+                    // call HWAS common function
+                    // TODO RTC: 45781
+                    //errl = HWAS::theDeconfigGard().registerDelayedDeconfigure(*pTarget,
+                                //i_errlPlid);
+                    break;
+                }
+            } // switch deconfigState
+            break;
+        }
+        case (PROCEDURE_CALLOUT):
+        {
+            HWAS_INF("Procedure callout; proc 0x%x priority 0x%x",
+                pCalloutUD->type, pCalloutUD->type);
+            //const HWAS::epubProcedureID procedure = pCalloutUD->procedure;
+            //const callOutPriority priority = pCalloutUD->priority;
+
+            // TODO RTC: 35108
+            // call HWAS common function
+            //errl = HWAS::processCallout(procedure, priority, i_errlPlid);
+            break;
+        }
+        default:
+        {
+            HWAS_ERR("bad data in Callout UD %x", pCalloutUD->type);
+            break;
+        }
+    } // switch
+
+    HWAS_INF("processCallout exit errl %p", errl);
+} // processCallout
+
+}; // end namespace
diff --git a/src/usr/hwas/deconfigGard.C b/src/usr/hwas/deconfigGard.C
index 549c5e09d..6bb65cf39 100644
--- a/src/usr/hwas/deconfigGard.C
+++ b/src/usr/hwas/deconfigGard.C
@@ -206,11 +206,11 @@ errlHndl_t DeconfigGard::deconfigureTarget(TARGETING::Target & i_target,
 //******************************************************************************
 errlHndl_t DeconfigGard::createGardRecord(const TARGETING::Target & i_target,
                                           const uint32_t i_errlPlid,
-                                          const GardSeverity i_severity)
+                                          const GARD_ErrorType i_errorType)
 {
     TRAC_ERR("Usr Request: Create GARD Record");
     mutex_lock(&iv_mutex);
-    errlHndl_t l_pErr = _createGardRecord(i_target, i_errlPlid, i_severity);
+    errlHndl_t l_pErr = _createGardRecord(i_target, i_errlPlid, i_errorType);
     mutex_unlock(&iv_mutex);
     return l_pErr;
 }
@@ -468,7 +468,7 @@ void DeconfigGard::_getDeconfigureRecords(
 //******************************************************************************
 errlHndl_t DeconfigGard::_createGardRecord(const TARGETING::Target & i_target,
                                            const uint32_t i_errlPlid,
-                                           const GardSeverity i_severity)
+                                           const GARD_ErrorType i_errorType)
 {
     errlHndl_t l_pErr = NULL;
 
@@ -539,7 +539,7 @@ errlHndl_t DeconfigGard::_createGardRecord(const TARGETING::Target & i_target,
                 l_pRecord->iv_targetId = l_id;
                 // TODO Setup iv_cardMruSn or iv_chipMruEcid    
                 l_pRecord->iv_errlogPlid = i_errlPlid;
-                l_pRecord->iv_severity = i_severity;
+                l_pRecord->iv_errorType = i_errorType;
                 l_pRecord->iv_padding[0] = 0;
                 l_pRecord->iv_padding[1] = 0;
                 l_pRecord->iv_padding[2] = 0;
diff --git a/src/usr/hwas/makefile b/src/usr/hwas/makefile
index 57958474a..e17115296 100644
--- a/src/usr/hwas/makefile
+++ b/src/usr/hwas/makefile
@@ -30,11 +30,11 @@ VPATH += ${ROOTPATH}/src/usr/hwas/common
 
 MODULE = hwas
 
-# files in this directory
-OBJS =  hwas.o deconfigGard.o
+# files in hwas/common directory
+OBJS =  hwas.o hwasCallout.o
 
-# files in hwas/plat
-OBJS +=  hwasPlat.o hostbootIstep.o
+# files in this directory
+OBJS +=  hwasPlat.o hostbootIstep.o deconfigGard.o
 
 SUBDIRS = test.d
 
diff --git a/src/usr/hwas/test/hwas1test.H b/src/usr/hwas/test/hwas1test.H
index 894db8a1a..168aeba15 100644
--- a/src/usr/hwas/test/hwas1test.H
+++ b/src/usr/hwas/test/hwas1test.H
@@ -1,25 +1,26 @@
-//  IBM_PROLOG_BEGIN_TAG
-//  This is an automatically generated prolog.
-//
-//  $Source: src/usr/targeting/test/targetingtest.H $
-//
-//  IBM CONFIDENTIAL
-//
-//  COPYRIGHT International Business Machines Corp. 2011
-//
-//  p1
-//
-//  Object Code Only (OCO) source materials
-//  Licensed Internal Code Source Materials
-//  IBM HostBoot Licensed Internal Code
-//
-//  The source code for this program is not published or other-
-//  wise divested of its trade secrets, irrespective of what has
-//  been deposited with the U.S. Copyright Office.
-//
-//  Origin: 30
-//
-//  IBM_PROLOG_END
+/*  IBM_PROLOG_BEGIN_TAG
+ *  This is an automatically generated prolog.
+ *
+ *  $Source: src/usr/hwas/test/hwas1test.H $
+ *
+ *  IBM CONFIDENTIAL
+ *
+ *  COPYRIGHT International Business Machines Corp. 2011-2012
+ *
+ *  p1
+ *
+ *  Object Code Only (OCO) source materials
+ *  Licensed Internal Code Source Materials
+ *  IBM HostBoot Licensed Internal Code
+ *
+ *  The source code for this program is not published or other-
+ *  wise divested of its trade secrets, irrespective of what has
+ *  been deposited with the U.S. Copyright Office.
+ *
+ *  Origin: 30
+ *
+ *  IBM_PROLOG_END_TAG
+ */
 #ifndef __TESTTARGETING_H
 #define __TESTTARGETING_H
 
@@ -80,7 +81,6 @@ public:
             l_hwasState.present               =   true;
             l_hwasState.functional            =   true;
             l_hwasState.changedSinceLastIPL   =   true;
-            l_hwasState.gardLevel             =   3;
 
             //  Now write the modified value back to Targeting.
             l_pTarget->setAttr<ATTR_HWAS_STATE>( l_hwasState );
@@ -110,13 +110,6 @@ public:
                         l_pTarget->getAttr<ATTR_HWAS_STATE>().changedSinceLastIPL );
             }
 
-            if ( l_pTarget->getAttr<ATTR_HWAS_STATE>().gardLevel != 3 )
-            {
-                TS_FAIL( " gardLevel = 0x%x, should be 3",
-                        l_pTarget->getAttr<ATTR_HWAS_STATE>().gardLevel );
-            }
-
-
             //
             //  Now write the original value back.
             //
@@ -150,13 +143,6 @@ public:
                 TS_FAIL( " changedSinceLastIPL = 0x%x, not restored",
                         l_pTarget->getAttr<ATTR_HWAS_STATE>().changedSinceLastIPL );
             }
-
-            if ( l_pTarget->getAttr<ATTR_HWAS_STATE>().gardLevel
-                    != l_orgHwasState.gardLevel )
-            {
-                TS_FAIL( " gardLevel = 0x%x, not restored",
-                        l_pTarget->getAttr<ATTR_HWAS_STATE>().gardLevel );
-            }
         }
 
         TS_TRACE( "testHWASReadWrite exit" );
diff --git a/src/usr/hwas/test/hwasGardTest.H b/src/usr/hwas/test/hwasGardTest.H
index 9338678e9..f1d10c032 100644
--- a/src/usr/hwas/test/hwasGardTest.H
+++ b/src/usr/hwas/test/hwasGardTest.H
@@ -1,25 +1,26 @@
-//  IBM_PROLOG_BEGIN_TAG
-//  This is an automatically generated prolog.
-//
-//  $Source: src/usr/hwas/test/hwasGardTest.H $
-//
-//  IBM CONFIDENTIAL
-//
-//  COPYRIGHT International Business Machines Corp. 2011
-//
-//  p1
-//
-//  Object Code Only (OCO) source materials
-//  Licensed Internal Code Source Materials
-//  IBM HostBoot Licensed Internal Code
-//
-//  The source code for this program is not published or other-
-//  wise divested of its trade secrets, irrespective of what has
-//  been deposited with the U.S. Copyright Office.
-//
-//  Origin: 30
-//
-//  IBM_PROLOG_END
+/*  IBM_PROLOG_BEGIN_TAG
+ *  This is an automatically generated prolog.
+ *
+ *  $Source: src/usr/hwas/test/hwasGardTest.H $
+ *
+ *  IBM CONFIDENTIAL
+ *
+ *  COPYRIGHT International Business Machines Corp. 2011-2012
+ *
+ *  p1
+ *
+ *  Object Code Only (OCO) source materials
+ *  Licensed Internal Code Source Materials
+ *  IBM HostBoot Licensed Internal Code
+ *
+ *  The source code for this program is not published or other-
+ *  wise divested of its trade secrets, irrespective of what has
+ *  been deposited with the U.S. Copyright Office.
+ *
+ *  Origin: 30
+ *
+ *  IBM_PROLOG_END_TAG
+ */
 #ifndef _HWASGARDTEST_H
 #define _HWASGARDTEST_H
 
@@ -445,7 +446,7 @@ public:
             // valid PLID
             l_pErr = HWAS::theDeconfigGard().
                 createGardRecord(*l_pTarget, 0,
-                                 HWAS::DeconfigGard::GARD_SEVERITY_PREDICTIVE);
+                                 HWAS::GARD_Predictive);
         
             if (l_pErr)
             {
@@ -469,11 +470,11 @@ public:
                 break;
             }
             
-            if (l_records[0].iv_severity !=
-                HWAS::DeconfigGard::GARD_SEVERITY_PREDICTIVE)
+            if (l_records[0].iv_errorType !=
+                HWAS::GARD_Predictive)
             {
-                TS_FAIL("testGard4: Record severity %d, expected predictive",
-                        l_records[0].iv_severity);
+                TS_FAIL("testGard4: Record errorType %d, expected predictive",
+                        l_records[0].iv_errorType);
                 break;
             }
             
@@ -552,7 +553,7 @@ public:
             // valid PLID
             l_pErr = HWAS::theDeconfigGard().
                 createGardRecord(*l_pTarget, 0,
-                                 HWAS::DeconfigGard::GARD_SEVERITY_PREDICTIVE);
+                                 HWAS::GARD_Predictive);
         
             if (l_pErr)
             {
@@ -563,7 +564,7 @@ public:
             // Create another GARD Record for the chip
             l_pErr = HWAS::theDeconfigGard().
                 createGardRecord(*l_pTarget, 0,
-                                 HWAS::DeconfigGard::GARD_SEVERITY_FATAL);
+                                 HWAS::GARD_Fatal);
         
             if (l_pErr)
             {
@@ -662,7 +663,7 @@ public:
             // valid PLID
             l_pErr = HWAS::theDeconfigGard().
                 createGardRecord(*l_pTarget, 0,
-                                 HWAS::DeconfigGard::GARD_SEVERITY_PREDICTIVE);
+                                 HWAS::GARD_Predictive);
         
             if (l_pErr)
             {
@@ -673,7 +674,7 @@ public:
             // Create another GARD Record for the chip
             l_pErr = HWAS::theDeconfigGard().
                 createGardRecord(*l_pTarget, 0,
-                                 HWAS::DeconfigGard::GARD_SEVERITY_FATAL);
+                                 HWAS::GARD_Fatal);
         
             if (l_pErr)
             {
@@ -801,7 +802,7 @@ public:
             // valid PLID
             l_pErr = HWAS::theDeconfigGard().
                 createGardRecord(*l_pTarget, 0,
-                                 HWAS::DeconfigGard::GARD_SEVERITY_PREDICTIVE);
+                                 HWAS::GARD_Predictive);
         
             if (l_pErr)
             {
@@ -812,7 +813,7 @@ public:
             // Create another GARD Record for the chip
             l_pErr = HWAS::theDeconfigGard().
                 createGardRecord(*l_pTarget, 0,
-                                 HWAS::DeconfigGard::GARD_SEVERITY_FATAL);
+                                 HWAS::GARD_Fatal);
         
             if (l_pErr)
             {
diff --git a/src/usr/hwpf/plat/fapiPlatHwpInvoker.C b/src/usr/hwpf/plat/fapiPlatHwpInvoker.C
index 22e954e2d..c35a90eec 100644
--- a/src/usr/hwpf/plat/fapiPlatHwpInvoker.C
+++ b/src/usr/hwpf/plat/fapiPlatHwpInvoker.C
@@ -1,25 +1,26 @@
-//  IBM_PROLOG_BEGIN_TAG
-//  This is an automatically generated prolog.
-//
-//  $Source: src/usr/hwpf/plat/fapiPlatHwpInvoker.C $
-//
-//  IBM CONFIDENTIAL
-//
-//  COPYRIGHT International Business Machines Corp. 2011
-//
-//  p1
-//
-//  Object Code Only (OCO) source materials
-//  Licensed Internal Code Source Materials
-//  IBM HostBoot Licensed Internal Code
-//
-//  The source code for this program is not published or other-
-//  wise divested of its trade secrets, irrespective of what has
-//  been deposited with the U.S. Copyright Office.
-//
-//  Origin: 30
-//
-//  IBM_PROLOG_END
+/*  IBM_PROLOG_BEGIN_TAG
+ *  This is an automatically generated prolog.
+ *
+ *  $Source: src/usr/hwpf/plat/fapiPlatHwpInvoker.C $
+ *
+ *  IBM CONFIDENTIAL
+ *
+ *  COPYRIGHT International Business Machines Corp. 2011-2012
+ *
+ *  p1
+ *
+ *  Object Code Only (OCO) source materials
+ *  Licensed Internal Code Source Materials
+ *  IBM HostBoot Licensed Internal Code
+ *
+ *  The source code for this program is not published or other-
+ *  wise divested of its trade secrets, irrespective of what has
+ *  been deposited with the U.S. Copyright Office.
+ *
+ *  Origin: 30
+ *
+ *  IBM_PROLOG_END_TAG
+ */
 /**
  *  @file fapiPlatHwpInvoker.C
  *
@@ -85,6 +86,7 @@ void processEICallouts(const ErrorInfo & i_errInfo,
          l_itr != i_errInfo.iv_callouts.end(); ++l_itr)
     {
         // TODO Add callout to error log
+        // RTC 45800
         FAPI_ERR("processEICallouts: Adding target callout to errlog (TODO). Type: 0x%x. Pri: 0x%x",
                  (*l_itr)->iv_target.getType(), (*l_itr)->iv_priority);
     }
@@ -103,6 +105,7 @@ void processEIDeconfigs(const ErrorInfo & i_errInfo,
          l_itr != i_errInfo.iv_deconfigs.end(); ++l_itr)
     {
         // TODO Deconfigure target
+        // RTC 45800
         FAPI_ERR("processEIDeconfigs: Deconfiguring target (TODO). Type: 0x%x",
                  (*l_itr)->iv_target.getType());
     }
@@ -110,10 +113,10 @@ void processEIDeconfigs(const ErrorInfo & i_errInfo,
 
 
 //******************************************************************************
-// processEiGards
+// processEIGards
 // Processes any Gard requests in the ReturnCode Error Information
 //******************************************************************************
-void processEiGards(const ErrorInfo & i_errInfo,
+void processEIGards(const ErrorInfo & i_errInfo,
                        errlHndl_t io_pError)
 {
     // Iterate through gard requests, creating a GARD record for each target
@@ -121,6 +124,7 @@ void processEiGards(const ErrorInfo & i_errInfo,
          l_itr != i_errInfo.iv_gards.end(); ++l_itr)
     {
         // TODO Create GARD record for target
+        // RTC 45800
         FAPI_ERR("processEIGards: Garding target (TODO). Type: 0x%x",
                  (*l_itr)->iv_target.getType());
     }
@@ -177,7 +181,7 @@ errlHndl_t fapiRcToErrl(ReturnCode & io_rc)
                 processEIFfdcs(*l_pErrorInfo, l_pError);
                 processEICallouts(*l_pErrorInfo, l_pError);
                 processEIDeconfigs(*l_pErrorInfo, l_pError);
-                processEiGards(*l_pErrorInfo, l_pError);
+                processEIGards(*l_pErrorInfo, l_pError);
             }
             else
             {
diff --git a/src/usr/targeting/common/xmltohb/attribute_types.xml b/src/usr/targeting/common/xmltohb/attribute_types.xml
index c05071a7a..747334984 100644
--- a/src/usr/targeting/common/xmltohb/attribute_types.xml
+++ b/src/usr/targeting/common/xmltohb/attribute_types.xml
@@ -796,10 +796,9 @@
     <id>HWAS_STATE</id>
     <description>HardWare Availability Service State Attribute.
         Keeps track of Target values poweredOn, present, functional,
-        changedSinceLastIPL, and gardLevel</description>
+        and changedSinceLastIPL</description>
     <complexType>
-        <description>struct - so far contains 4 booleans and a 4-bit GARD level
-            field</description>
+        <description>struct - so far contains 5 booleans</description>
         <field>
             <name>poweredOn</name>
             <description>boolean: Target is powered on, or Not.
@@ -833,14 +832,6 @@
             <default>0</default>
         </field>
         <field>
-            <name>gardLevel</name>
-            <description>GARD Level.  This is a 4-bit value.
-                comes up as 0  </description>
-            <type>uint8_t</type>
-            <bits>4</bits>
-            <default>0</default>
-        </field>
-        <field>
             <name>dumpfunctional</name>
             <description>FSP Only, used by DUMP applet to indicate targets dump capability.
                 Comes up as  0 which indicates the target is not dump capable</description>