IBSCOM error path support

This commit updates IBSCOM to have error path support.
It also updates the good-path test cases since there is
limited good path support in simics.  Full enablement
will be done later.

Change-Id: I5f9d66165db119473f606303a1026c8c71988785
RTC: 34743
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/3972
Tested-by: Jenkins Server
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Reviewed-by: Douglas R. Gilbert <dgilbert@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

3 files changed, 507 insertions, 94 deletions

diff --git a/src/usr/ibscom/ibscom.C b/src/usr/ibscom/ibscom.C
index f379289ca..a6d17a2a6 100644
--- a/src/usr/ibscom/ibscom.C
+++ b/src/usr/ibscom/ibscom.C
@@ -33,12 +33,17 @@
 #include <trace/interface.H>
 #include <errl/errlentry.H>
 #include <errl/errlmanager.H>
+#include <errl/errludlogregister.H>
 #include <targeting/common/targetservice.H>
 #include <ibscom/ibscomreasoncodes.H>
 #include "ibscom.H"
 #include <assert.h>
 #include <limits.h>
 
+// Easy macro replace for unit testing
+//#define TRACUCOMP(args...)  TRACFCOMP(args)
+#define TRACUCOMP(args...)
+
 // Trace definition
 trace_desc_t* g_trac_ibscom = NULL;
 TRAC_INIT(&g_trac_ibscom, "IBSCOM", KILOBYTE);
@@ -49,16 +54,12 @@ using namespace TARGETING;
 namespace IBSCOM
 {
 
-
 // Register XSCcom access functions to DD framework
 DEVICE_REGISTER_ROUTE(DeviceFW::WILDCARD,
                       DeviceFW::IBSCOM,
                       TYPE_MEMBUF,
                       ibscomPerformOp);
 
-
-
-
 /**
  * @brief Internal routine that verifies the validity of input parameters
  * for an inband scom access.
@@ -84,6 +85,30 @@ errlHndl_t ibscomOpSanityCheck(const DeviceFW::OperationType i_opType,
 
     do
     {
+        // Verify address is somewhat valid (not over 32-bits long)
+        if(0 != (i_addr & 0xFFFFFFFF00000000))
+        {
+            TRACFCOMP(g_trac_ibscom, ERR_MRK"ibscomOpSanityCheck: Impossible address.  i_addr=0x%.16X",
+                      i_buflen);
+            /*@
+             * @errortype
+             * @moduleid     IBSCOM_SANITY_CHECK
+             * @reasoncode   IBSCOM_INVALID_ADDRESS
+             * @userdata1    Inband Scom  address
+             * @userdata2    <none>
+             * @devdesc      The provided address is over 32 bits long
+             *               which makes it invalid.
+             */
+            l_err = new ErrlEntry(ERRL_SEV_UNRECOVERABLE,
+                                  IBSCOM_SANITY_CHECK,
+                                  IBSCOM_INVALID_ADDRESS,
+                                  i_addr,
+                                  0);
+            l_err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
+                                       HWAS::SRCI_PRIORITY_HIGH);
+            break;
+        }
+
         // Verify data buffer
         if ( (i_buflen < IBSCOM_BUFFER_SIZE) ||
              (i_buffer == NULL) )
@@ -105,6 +130,8 @@ errlHndl_t ibscomOpSanityCheck(const DeviceFW::OperationType i_opType,
                                   IBSCOM_INVALID_DATA_BUFFER,
                                   i_buflen,
                                   i_addr);
+            l_err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
+                                       HWAS::SRCI_PRIORITY_HIGH);
             break;
         }
 
@@ -128,6 +155,8 @@ errlHndl_t ibscomOpSanityCheck(const DeviceFW::OperationType i_opType,
                                   IBSCOM_INVALID_OP_TYPE,
                                   i_opType,
                                   i_addr);
+            l_err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
+                                       HWAS::SRCI_PRIORITY_HIGH);
             break;
         }
 
@@ -214,6 +243,8 @@ errlHndl_t getTargetVirtualAddress(Target* i_target,
                                                      get_huid(i_target),
                                                      0),
                                 0);
+                l_err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
+                                           HWAS::SRCI_PRIORITY_HIGH);
                 break;
             }
             parentMCS = *(mcs_list.begin());
@@ -221,7 +252,7 @@ errlHndl_t getTargetVirtualAddress(Target* i_target,
             l_IBScomBaseAddr =
               parentMCS->getAttr<ATTR_IBSCOM_MCS_BASE_ADDR>();
 
-            TRACFCOMP(g_trac_ibscom, INFO_MRK
+            TRACUCOMP(g_trac_ibscom, INFO_MRK
                       "getTargetVirtualAddress: From Attribute query l_IBScomBaseAddr=0x%llX, i_target=0x%.8x",
                       l_IBScomBaseAddr,
                       i_target->getAttr<ATTR_HUID>());
@@ -235,6 +266,20 @@ errlHndl_t getTargetVirtualAddress(Target* i_target,
                             THIRTYTWO_GB));
 
             // Save the virtual address attribute.
+
+            // Leaving the comments as a discussion point...
+            // This issue is tracked under RTC: 35315
+            // Technically there is a race condition here. The mutex is
+            // a per-hardware thread mutex, not a mutex for the whole XSCOM
+            // logic. So there is possibility that this same thread is running
+            // on another thread at the exact same time. We can use atomic
+            // update instructions here.
+            // Comment for Nick: This is a good candidate for having a way
+            // to return a reference to the attribute instead of requiring
+            // to call setAttr. We currently have no way to SMP-safely update
+            // this attribute, where as if we had a reference to it we could use
+            // the atomic update functions (_sync_bool_compare_and_swap in
+            // this case.
             i_target->setAttr<ATTR_IBSCOM_VIRTUAL_ADDR>
               (reinterpret_cast<uint64_t>(o_virtAddr));
         }
@@ -252,25 +297,26 @@ errlHndl_t getTargetVirtualAddress(Target* i_target,
 
 ///////////////////////////////////////////////////////////////////////////////
 ///////////////////////////////////////////////////////////////////////////////
-errlHndl_t ibscomPerformOp(DeviceFW::OperationType i_opType,
+errlHndl_t doIBScom(DeviceFW::OperationType i_opType,
                           Target* i_target,
                           void* io_buffer,
                           size_t& io_buflen,
-                          int64_t i_accessType,
-                          va_list i_args)
+                          uint64_t i_addr,
+                          bool i_errDataPath)
 {
     errlHndl_t l_err = NULL;
-    uint64_t l_addr = va_arg(i_args,uint64_t);
+    mutex_t* l_mutex = NULL;
+    bool need_unlock = false;
 
     do
     {
-        TRACDCOMP(g_trac_ibscom, INFO_MRK
-                  ">>ibscomPerformOp: Perform op to SCOM Address 0x%X",
-                  l_addr);
+        TRACUCOMP(g_trac_ibscom, INFO_MRK
+                  ">>doIBScom: Perform op to SCOM Address 0x%X",
+                  i_addr);
 
         // inband scom operation sanity check
         l_err = ibscomOpSanityCheck(i_opType, i_target, io_buffer,
-                                    io_buflen, l_addr);
+                                    io_buflen, i_addr);
         if (l_err)
         {
             break;
@@ -287,44 +333,344 @@ errlHndl_t ibscomPerformOp(DeviceFW::OperationType i_opType,
             break;
         }
 
+        TRACDCOMP(g_trac_ibscom,
+                  "doIBScom: Base Virt Addr: 0x%.16X, Read addr: 0x%.16X",
+                  l_virtAddr, &(l_virtAddr[i_addr]));
+
+
         // The dereferencing should handle Cache inhibited internally
         // Use local variable and memcpy to avoid unaligned memory access
         uint64_t l_data = 0;
+        bool rw_error = false;
+
+        //Device already locked when re-entering function for error collection..
+        if(!i_errDataPath)
+        {
+            l_mutex = i_target->getHbMutexAttr<TARGETING::ATTR_IBSCOM_MUTEX>();
+            mutex_lock(l_mutex);
+            need_unlock = true;
+        }
 
         if (i_opType == DeviceFW::READ)
         {
-            //TODO: Check that address isn't greater than allocated 32GB range
-            // RTC: 47212
-            l_data = l_virtAddr[l_addr];
+            //This is the actual MMIO Read
+            l_data = l_virtAddr[i_addr];
+            eieio();
 
-            memcpy(io_buffer, &l_data, sizeof(l_data));
-            TRACDCOMP(g_trac_ibscom,
-                      "ibscomPerformOp: Read data: %.16llx",
-                      l_data);
+            TRACUCOMP(g_trac_ibscom,
+                      "doIBScom: Read address: 0x%.8X data: %.16X",
+                      i_addr, l_data);
 
+            // Check for error or done
+            if(l_data == MMIO_IBSCOM_UE_DETECTED)
+            {
+                if(i_errDataPath)
+                {
+                    TRACFCOMP(g_trac_ibscom, ERR_MRK"doIBScom:  SUE Occurred during IBSCOM Error path");
+                    /*@
+                     * @errortype
+                     * @moduleid     IBSCOM_DO_IBSCOM
+                     * @reasoncode   IBSCOM_SUE_IN_ERR_PATH
+                     * @userdata1[0:31]   HUID of Centaur Target
+                     * @userdata1[32:64]  SCOM Address
+                     * @userdata2    Not Used
+                     * @devdesc      SUE Encountered when collecting IBSCOM
+                     *               Error Data
+                     */
+                    l_err =
+                      new ErrlEntry(ERRL_SEV_UNRECOVERABLE,
+                                    IBSCOM_DO_IBSCOM,
+                                    IBSCOM_SUE_IN_ERR_PATH,
+                                    TWO_UINT32_TO_UINT64(
+                                                         get_huid(i_target),
+                                                         i_addr),
+                                    0);
+                    //This error should NEVER get returned to caller, so it's a
+                    //FW bug if it actually gets comitted.
+                    l_err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
+                                               HWAS::SRCI_PRIORITY_HIGH);
+                    break;
+                }
+                else
+                {
+                    TRACFCOMP(g_trac_ibscom,
+                              "doIBScom: Error code found in read data");
+                    rw_error = true;
+                }
+            }
+            else
+            {
+                //Copy data to user buffer
+                memcpy(io_buffer, &l_data, sizeof(l_data));
+            }
         }
-        else
+        else //i_opType == DeviceFW::WRITE
         {
-            TRACDCOMP(g_trac_ibscom,
-                      "ibscomPerformOp: Write data: %.16llx",
-                      l_data);
             memcpy(&l_data, io_buffer, sizeof(l_data));
-            l_virtAddr[l_addr] = l_data;
+            TRACUCOMP(g_trac_ibscom,
+                      "doIBScom: Write addr: 0x%.8X data: %.16X",
+                      i_addr, l_data);
+            //This is the actual MMIO Write
+            l_virtAddr[i_addr] = l_data;
+            eieio();
+
+            TRACDCOMP(g_trac_ibscom,
+                      "doIBScom: Read MBSIBWRSTAT to check for error");
+            //Read MBSIBWRSTAT to check for errors
+            //If an error occured on last write, reading MBSIBWRSTAT will
+            //trigger a SUE.
+            const uint32_t MBSIBWRSTAT = 0x201141D;
+            uint64_t statData = 0;
+            size_t readSize = sizeof(uint64_t);
+            l_err = doIBScom(DeviceFW::READ,
+                                  i_target,
+                                  &statData,
+                                  readSize,
+                                  MBSIBWRSTAT,
+                                  true);
+            if(l_err != NULL)
+            {
+                if( IBSCOM_SUE_IN_ERR_PATH == l_err->reasonCode() )
+                {
+                    TRACFCOMP(g_trac_ibscom, ERR_MRK
+                              "doIBScom: SUE on write detected");
+                    delete l_err;
+                    l_err = NULL;
+                    rw_error = true;
+                }
+                else
+                {
+                    TRACFCOMP(g_trac_ibscom, ERR_MRK"doIBScom: Unexpected error when checking for SUE");
+                    break;
+                }
+            }
+
         }
 
-        // Check for error or done
-        //TODO - check for errors  RTC: 47212
-        //assume successful for now
-        io_buflen = sizeof(uint64_t);
+        if(rw_error)
+        {
+            bool busDown = false;
+            TRACUCOMP(g_trac_ibscom,
+                      "doIBScom: Get Error data, read MBSIBERR0");
+            const uint32_t MBSIBERR0 = 0x201141B;
+            const uint64_t HOST_ERROR_VALID = 0x0000000080000000;
+            const uint64_t PIB_ERROR_STATUS_MASK = 0x0000000070000000;
+            const uint64_t PIB_ERROR_SHIFT = 28;
+            uint64_t errData = 0;
+            size_t readSize = sizeof(uint64_t);
+
+            //Use FSISCOM as workaround for DD1.x centaur chips (HW246298)
+            if(i_target->getAttr<TARGETING::ATTR_EC>() < 0x20)
+            {
+                //Need to explicitly use FSI SCOM in DD1X chips
+                l_err = deviceOp( DeviceFW::READ,
+                                     i_target,
+                                     &errData,
+                                     readSize,
+                                     DEVICE_FSISCOM_ADDRESS(MBSIBERR0) );
+                if(l_err)
+                {
+                    TRACFCOMP(g_trac_ibscom, ERR_MRK
+                              "doIBScom: Error reading MBSIBERR0 over FSI");
+                    //Save away the IBSCOM address
+                    ERRORLOG::ErrlUserDetailsLogRegister
+                      l_logReg(i_target);
+                    //Really just want to save the addres, so stick in some
+                    //obvious dummy data
+                    uint64_t dummyData = 0x00000000DEADBEEF;
+                    l_logReg.addDataBuffer(&dummyData, sizeof(dummyData),
+                                           DEVICE_IBSCOM_ADDRESS(i_addr));
+                    l_logReg.addToLog(l_err);
+                    break;
+                }
+                TRACUCOMP(g_trac_ibscom,
+                          "doIBScom: MBSIBERR0(0x%.16x) = 0x%.16X",
+                          MBSIBERR0, errData);
+
+                //attempt to clear the error register so future accesses
+                //will work
+                uint64_t zeroData = 0x0;
+                readSize = sizeof(uint64_t);
+                l_err = deviceOp( DeviceFW::WRITE,
+                                     i_target,
+                                     &zeroData,
+                                     readSize,
+                                     DEVICE_FSISCOM_ADDRESS(MBSIBERR0) );
+                if(l_err )
+                {
+                    errlCommit(l_err,IBSCOM_COMP_ID);
+                    l_err = NULL;
+                }
+
+
+                uint64_t pib_code = (errData && PIB_ERROR_STATUS_MASK)
+                  >> PIB_ERROR_SHIFT;
+                if((errData & HOST_ERROR_VALID) &&//bit 32
+                   // technically between 0x001 && 0x7, but only 3
+                   // bits are valid.
+                   (pib_code > 0x000))
+                {
+                    //TODO RTC: 35064 - This will be the same as the FSI SCOM
+                    //error handling.
+                    //Look at data to decide if bus is down
+                    //Make smart decisions based on PIB error code.
+
+                    //Assume caller provided bad address for now.
+                    TRACFCOMP(g_trac_ibscom, "doIBScom: MBSIBERR0 bit 0 set, caller most likely used a bad address (0x%.8x)",
+                              i_addr);
+
+                    /*@
+                     * @errortype
+                     * @moduleid     IBSCOM_DO_IBSCOM
+                     * @reasoncode   IBSCOM_INVALID_ADDRESS
+                     * @userdata1[0:31]   HUID of Centaur Target
+                     * @userdata1[32:64]  SCOM Address
+                     * @userdata2    Contents of MBSIBERR0 register
+                     * @devdesc      Operation rejected by HW due to bad
+                     *               address.
+                     */
+                    l_err =
+                      new ErrlEntry(ERRL_SEV_UNRECOVERABLE,
+                                    IBSCOM_DO_IBSCOM,
+                                    IBSCOM_INVALID_ADDRESS,
+                                    TWO_UINT32_TO_UINT64(
+                                                         get_huid(i_target),
+                                                         i_addr),
+                                    errData);
+                    l_err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
+                                               HWAS::SRCI_PRIORITY_HIGH);
+                    break;
+                }
+                else
+                {
+                    //Bus is down
+                    busDown = true;
+                }
 
-        TRACDCOMP(g_trac_ibscom,
-                  "ibscomPerformOp: OpType 0x%.16llX, SCOM Address 0x%llX, Virtual Address 0x%llX",
+
+            }
+            else  // >= DD20
+            {
+                //TODO RTC: 68984: Validate error path on DD2.0 Centaurs
+                l_err = doIBScom(DeviceFW::READ,
+                                    i_target,
+                                    &errData,
+                                    readSize,
+                                    MBSIBERR0,
+                                    true);
+                if(l_err != NULL)
+                {
+                    if( IBSCOM_SUE_IN_ERR_PATH == l_err->reasonCode() )
+                    {
+                        TRACFCOMP(g_trac_ibscom, ERR_MRK
+                                  "doIBScom: SUE on write detected");
+                        delete l_err;
+                        l_err = NULL;
+                        busDown = true;
+                    }
+                    else
+                    {
+                        TRACFCOMP(g_trac_ibscom, ERR_MRK"doIBScom: Unexpected error when checking for SUE");
+                        break;
+                    }
+                }
+                else //MBSIBERR0 returned valid data
+                {
+                    //TODO RTC: 35064 - This will be the same as the FSI SCOM
+                    //error handling.
+                    //Look at data to decide if bus is down
+                    //Make smart decisions based on PIB error code.
+
+                    //For now, assume bus is Down
+                    busDown = true;
+
+                }
+            } // >= DD20
+
+            if(busDown)
+            {
+                //TODO RTC: 69115 - call PRD to do FIR analysis, return PRD
+                //error instead.
+                /*@
+                 * @errortype
+                 * @moduleid     IBSCOM_DO_IBSCOM
+                 * @reasoncode   IBSCOM_BUS_FAILURE
+                 * @userdata1[0:31]   HUID of Centaur Target
+                 * @userdata1[32:64]  SCOM Address
+                 * @userdata2    Contents of MBSIBERR0 register
+                 * @devdesc      Bus failure when attempting to perform
+                 *               IBSCOM operation.  IBSCOM disabled.
+                 */
+                l_err =
+                  new ErrlEntry(ERRL_SEV_UNRECOVERABLE,
+                                IBSCOM_DO_IBSCOM,
+                                IBSCOM_BUS_FAILURE,
+                                TWO_UINT32_TO_UINT64(
+                                                     get_huid(i_target),
+                                                     i_addr),
+                                errData);
+
+                l_err->addHwCallout(i_target,
+                                    HWAS::SRCI_PRIORITY_HIGH,
+                                    HWAS::NO_DECONFIG,
+                                    HWAS::GARD_NULL);
+
+                //disable IBSCOM
+                ScomSwitches l_switches =
+                  i_target->getAttr<ATTR_SCOM_SWITCHES>();
+
+                // If IBSCOM is not already disabled.
+                if ((l_switches.useFsiScom != 1) ||
+                    (l_switches.useInbandScom != 0))
+                {
+                    l_switches.useFsiScom = 1;
+                    l_switches.useInbandScom = 0;
+
+                    // Turn off IBSCOM and turn on FSI SCOM.
+                    i_target->setAttr<ATTR_SCOM_SWITCHES>(l_switches);
+                }
+                break;
+            }
+        }
+        else
+        {
+            //Operation was a success, set buffer length
+            io_buflen = sizeof(uint64_t);
+        }
+
+        TRACDCOMP(g_trac_ibscom,"doIBScom: OpType 0x%.16llX, SCOM Address 0x%llX, Virtual Address 0x%llX",
                   static_cast<uint64_t>(i_opType),
-                  l_addr,
-                  &(l_virtAddr[l_addr]));
+                  i_addr,
+                  &(l_virtAddr[i_addr]));
 
     } while (0);
 
+    if( need_unlock && l_mutex )
+    {
+        mutex_unlock(l_mutex);
+    }
+
+    return l_err;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
+errlHndl_t ibscomPerformOp(DeviceFW::OperationType i_opType,
+                          Target* i_target,
+                          void* io_buffer,
+                          size_t& io_buflen,
+                          int64_t i_accessType,
+                          va_list i_args)
+{
+    errlHndl_t l_err = NULL;
+    uint64_t l_addr = va_arg(i_args,uint64_t);
+
+    l_err = doIBScom(i_opType,
+                     i_target,
+                     io_buffer,
+                     io_buflen,
+                     l_addr,
+                     false);
     return l_err;
 }
 
diff --git a/src/usr/ibscom/ibscom.H b/src/usr/ibscom/ibscom.H
index 7bb17b9ad..3ba3c34f1 100644
--- a/src/usr/ibscom/ibscom.H
+++ b/src/usr/ibscom/ibscom.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012                   */
+/* COPYRIGHT International Business Machines Corp. 2012,2013              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
diff --git a/src/usr/ibscom/test/ibscomtest.H b/src/usr/ibscom/test/ibscomtest.H
index 189f78049..e6be9f7c8 100644
--- a/src/usr/ibscom/test/ibscomtest.H
+++ b/src/usr/ibscom/test/ibscomtest.H
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* IBM CONFIDENTIAL                                                       */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012                   */
+/* COPYRIGHT International Business Machines Corp. 2012,2013              */
 /*                                                                        */
 /* p1                                                                     */
 /*                                                                        */
@@ -47,14 +47,11 @@ class IBscomTest: public CxxTest::TestSuite
 
     /**
      * @brief inband scom test #1
-     *        Write value and read back to verify
+     *        Write values and read back to verify
      */
     void test_IBscom(void)
     {
 
-        //TODO Enable and improve test cases  RTC: 52900
-        return;
-
         uint64_t fails = 0;
         uint64_t total = 0;
         errlHndl_t l_err = NULL;
@@ -88,77 +85,146 @@ class IBscomTest: public CxxTest::TestSuite
         }
         l_testTarget = *(centaur_list.begin());
 
-        uint64_t addr = 0x03010E03;
-        uint64_t orig_data = 0;
-        uint64_t data = 0x12345678FEEDB0B0;
+        TRACDCOMP(g_trac_ibscom,
+                  "IBscomTest::test_IBscom> Read orignal data from Centaur");
+
+
+        const uint64_t addrs[] = {0x0201164F, 0x0301069A};
+        uint64_t orig_data[2] = {0};
+        uint64_t new_data[] = {0x1234567ABABABA00, 0xFEEDB0B0FEDCBA00};
+        uint64_t read_data[2] = {0};
+        uint64_t read_data_fsi[2] = {0};
         size_t op_size = sizeof(uint64_t);
 
         //Save of initial register content
-        l_err = deviceRead( l_testTarget,
-                            &orig_data,
-                            op_size,
-                            DEVICE_SCOM_ADDRESS(addr) );
-        if( l_err )
+        for(uint32_t i=0; i<2; i++)
         {
-            TRACFCOMP(g_trac_ibscom,
-                      "IBscomTest::test_IBscom> Orig Read: Error from device : addr=0x%X, RC=%X",
-                      addr, l_err->reasonCode() );
-            TS_FAIL( "test_IBscom1> ERROR : Unexpected error log from read1" );
-            fails++;
-            errlCommit(l_err,IBSCOM_COMP_ID);
+            op_size = sizeof(uint64_t);
+            l_err = deviceRead( l_testTarget,
+                                &orig_data[i],
+                                op_size,
+                                DEVICE_SCOM_ADDRESS(addrs[i]) );
+            if( l_err )
+            {
+                TRACFCOMP(g_trac_ibscom,"IBscomTest::test_IBscom> Orig Read: Error from device : addr=0x%X, RC=%X",
+                          addrs[i], l_err->reasonCode() );
+                TS_FAIL( "test_IBscom1> ERROR : Unexpected error log from saving original data." );
+                fails++;
+                errlCommit(l_err,IBSCOM_COMP_ID);
+            }
+            total++;
+
         }
 
-        l_err = deviceOp( DeviceFW::WRITE,
-                          l_testTarget,
-                          &data,
-                          op_size,
-                          DEVICE_IBSCOM_ADDRESS(addr) );
-        if( l_err )
+        //Write in some new data
+        for(uint32_t i=0; i<2; i++)
         {
-            TRACFCOMP(g_trac_ibscom,
-                      "IBscomTest::test_IBscom> Write: Error from device : addr=0x%X, RC=%X",
-                      addr, l_err->reasonCode() );
-            TS_FAIL( "ScomTest::test_IBscom> ERROR : Error log from write1" );
-            fails++;
-            errlCommit(l_err,IBSCOM_COMP_ID);
+            TRACDCOMP(g_trac_ibscom,"IBscomTest::test_IBscom> Write new pattern: 0x%.16X", new_data[i]);
+
+            op_size = sizeof(uint64_t);
+            l_err = deviceOp( DeviceFW::WRITE,
+                              l_testTarget,
+                              &new_data[i],
+                              op_size,
+                              DEVICE_IBSCOM_ADDRESS(addrs[i]) );
+            if( l_err )
+            {
+                TRACFCOMP(g_trac_ibscom,"IBscomTest::test_IBscom> Write: Error from device : addr=0x%X, RC=%X",
+                          addrs[i], l_err->reasonCode() );
+                TS_FAIL( "ScomTest::test_IBscom> ERROR : Error log from IBSCOM Write" );
+                fails++;
+                errlCommit(l_err,IBSCOM_COMP_ID);
+            }
+
+            total++;
         }
 
-        total++;
-
-        l_err = deviceOp( DeviceFW::READ,
-                          l_testTarget,
-                          &data,
-                          op_size,
-                          DEVICE_IBSCOM_ADDRESS(addr) );
-        if( l_err )
+        //Read the data back with IBSCOM
+        for(uint32_t i=0; i<2; i++)
         {
-            TRACFCOMP(g_trac_ibscom,
-                      "IBscomTest::test_IBscom> Read: Error from device : addr=0x%X, RC=%X",
-                      addr, l_err->reasonCode() );
-            TS_FAIL( "test_IBscom> ERROR : Error log from read2" );
-            fails++;
-            errlCommit(l_err,IBSCOM_COMP_ID);
+            TRACDCOMP(g_trac_ibscom,
+                      "IBscomTest::test_IBscom> Read data back");
+
+            //reset size
+            op_size = sizeof(uint64_t);
+            l_err = deviceOp( DeviceFW::READ,
+                              l_testTarget,
+                              &read_data[i],
+                              op_size,
+                              DEVICE_IBSCOM_ADDRESS(addrs[i]) );
+            if( l_err )
+            {
+                TRACFCOMP(g_trac_ibscom,"IBscomTest::test_IBscom> Read: Error from device : addr=0x%X, RC=%X",
+                          addrs[i], l_err->reasonCode() );
+                TS_FAIL( "test_IBscom> ERROR : Error log from IBSCOM read" );
+                fails++;
+                errlCommit(l_err,IBSCOM_COMP_ID);
+            }
+
+            if(new_data[i] != read_data[i])
+            {
+                TS_FAIL( "test_IBscom> ERROR : Data miss-match on IBSCOM read-back check." );
+
+                TRACFCOMP(g_trac_ibscom,"IBscomTest::test_IBscom>ERROR: addr=0x%.8x: Write data=0x%.16X, IBSCOM read data=0x%.16X",
+                          addrs[i], new_data[i], read_data[i]);
+                fails++;
+            }
+
+            total++;
         }
 
-        total++;
+        //Read the data back with FSISCOM
+        for(uint32_t i=0; i<2; i++)
+        {
+            TRACDCOMP(g_trac_ibscom,
+                      "IBscomTest::test_IBscom> Read data back");
+
+            //reset size
+            op_size = sizeof(uint64_t);
+            l_err = deviceOp( DeviceFW::READ,
+                              l_testTarget,
+                              &read_data_fsi[i],
+                              op_size,
+                              DEVICE_FSISCOM_ADDRESS(addrs[i]) );
+            if( l_err )
+            {
+                TRACFCOMP(g_trac_ibscom, "IBscomTest::test_IBscom> Read: Error from device : addr=0x%X, RC=%X",
+                          addrs[i], l_err->reasonCode() );
+                TS_FAIL( "test_IBscom> ERROR : Error log from FSI Read" );
+                fails++;
+                errlCommit(l_err,IBSCOM_COMP_ID);
+            }
+
+            if(new_data[i] != read_data_fsi[i])
+            {
+                TS_FAIL( "test_IBscom> ERROR : Data miss-match on FSI read-back check." );
+
+                TRACFCOMP(g_trac_ibscom, "IBscomTest::test_IBscom>ERROR: addr=0x%.8x: Write data=0x%.16X, FSI read data=0x%.16X",
+                          addrs[i], new_data[i], read_data_fsi[i]);
+                fails++;
+            }
+            total++;
+        }
 
         //Restore original data.
-          l_err = deviceWrite( l_testTarget,
-                               &orig_data,
-                               op_size,
-                               DEVICE_SCOM_ADDRESS(addr) );
-
-        if( l_err )
+        for(uint32_t i=0; i<2; i++)
         {
-            TRACFCOMP(g_trac_ibscom,
-                      "IBscomTest::test_IBscom> Write Orig Data: Error from device : addr=0x%X, RC=%X",
-                      addr, l_err->reasonCode() );
-            TS_FAIL( "test_IBscom> ERROR : Error log from write2" );
-            fails++;
-            errlCommit(l_err,IBSCOM_COMP_ID);
+            op_size = sizeof(uint64_t);
+            l_err = deviceWrite( l_testTarget,
+                                 &orig_data[i],
+                                 op_size,
+                                 DEVICE_SCOM_ADDRESS(addrs[i]) );
+
+            if( l_err )
+            {
+                TRACFCOMP(g_trac_ibscom,"IBscomTest::test_IBscom> Write Orig Data: Error from device : addr=0x%X, RC=%X",
+                          addrs[i], l_err->reasonCode() );
+                TS_FAIL( "test_IBscom> ERROR : Error log from write2" );
+                fails++;
+                errlCommit(l_err,IBSCOM_COMP_ID);
+            }
         }
 
-
         TS_TRACE("test_IBscom runs successfully!");
         TRACFCOMP(g_trac_ibscom,
                   "IBscomTest::test_IBscom> %d/%d fails",
@@ -167,7 +233,8 @@ class IBscomTest: public CxxTest::TestSuite
         return;
     }
 
-
+    //TODO RTC: 68733: Add error path test cases when simics support
+    //is available
 
 };