Expand memory footprint to full 8MB cache.

    If fake PNOR isn't being used, we can expand our memory space to
    the full 8MB cache.  There will be follow up work with RTC: 49137
    to support 4MB degraded caches for bring-up.

Change-Id: I1248efa37965f39ebab62aae556349c34aa24b66
RTC: 47356
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/2319
Tested-by: Jenkins Server
Reviewed-by: Melissa J. Connell <missyc@us.ibm.com>
Reviewed-by: ADAM R. MUHLE <armuhle@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

11 files changed, 188 insertions, 70 deletions

diff --git a/src/include/kernel/misc.H b/src/include/kernel/misc.H
index c970c337e..5a818078e 100644
--- a/src/include/kernel/misc.H
+++ b/src/include/kernel/misc.H
@@ -1,26 +1,25 @@
-/*  IBM_PROLOG_BEGIN_TAG
- *  This is an automatically generated prolog.
- *
- *  $Source: src/include/kernel/misc.H $
- *
- *  IBM CONFIDENTIAL
- *
- *  COPYRIGHT International Business Machines Corp. 2011-2012
- *
- *  p1
- *
- *  Object Code Only (OCO) source materials
- *  Licensed Internal Code Source Materials
- *  IBM HostBoot Licensed Internal Code
- *
- *  The source code for this program is not published or other-
- *  wise divested of its trade secrets, irrespective of what has
- *  been deposited with the U.S. Copyright Office.
- *
- *  Origin: 30
- *
- *  IBM_PROLOG_END_TAG
- */
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/include/kernel/misc.H $                                   */
+/*                                                                        */
+/* IBM CONFIDENTIAL                                                       */
+/*                                                                        */
+/* COPYRIGHT International Business Machines Corp. 2011,2012              */
+/*                                                                        */
+/* p1                                                                     */
+/*                                                                        */
+/* Object Code Only (OCO) source materials                                */
+/* Licensed Internal Code Source Materials                                */
+/* IBM HostBoot Licensed Internal Code                                    */
+/*                                                                        */
+/* The source code for this program is not published or otherwise         */
+/* divested of its trade secrets, irrespective of what has been           */
+/* deposited with the U.S. Copyright Office.                              */
+/*                                                                        */
+/* Origin: 30                                                             */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
 /** @file misc.H
  *  @brief Misc. Kernel functions and utilities.
  */
@@ -88,5 +87,29 @@ namespace KernelMisc
                 /** Current time-base value for restore after winkle. */
             uint64_t iv_timebase;
     };
+
+    /** @fn expand_full_cache
+     *
+     *  @brief Expands the image footprint from a half-cache to full-cache
+     *         mode.
+     *
+     *  @return 0 or -errno
+     */
+    int expand_full_cache();
+
+    /** @fn populate_cache_lines
+     *
+     *  @brief Populates a range of memory addresses into the cache via dcbz.
+     *
+     *  This is meant to be used while the processor is in cache-contained
+     *  mode.  The entire cache line is zero'd so if the range is less then
+     *  a cache-line, or starts misaligned, then more memory might be zero'd
+     *  then would be expected.
+     *
+     *  @param[in] i_start - Address to start dcbz at.
+     *  @param[in] i_end - Address to stop dcbz at.
+     *                     (This address is not dcbz'd itself)
+     */
+    void populate_cache_lines(uint64_t* i_start, uint64_t* i_end);
 };
 #endif
diff --git a/src/include/kernel/pagemgr.H b/src/include/kernel/pagemgr.H
index ad94d0522..65da0de66 100644
--- a/src/include/kernel/pagemgr.H
+++ b/src/include/kernel/pagemgr.H
@@ -176,7 +176,7 @@ class PageManager
 
         enum
         {
-            MEMLEN = VmmManager::MBOX_DMA_ADDR,
+            MEMLEN = VmmManager::FIRST_RESERVED_PAGE,
             RESERVED_PAGES = 4,
 
             LOWMEM_NORM_LIMIT = 16,
diff --git a/src/include/kernel/vmmmgr.H b/src/include/kernel/vmmmgr.H
index 7b8bd07bc..87ae0caf6 100644
--- a/src/include/kernel/vmmmgr.H
+++ b/src/include/kernel/vmmmgr.H
@@ -40,11 +40,11 @@ class VmmManager
          */
         enum VMM_CONSTS
         {
-            FULL_MEM_SIZE = 4*MEGABYTE,
+            INITIAL_MEM_SIZE = 4*MEGABYTE,
 
             // put the Page Table at the end of our memory space
             PTSIZE = (1 << 18),
-            HTABORG_OFFSET = (FULL_MEM_SIZE - PTSIZE),
+            HTABORG_OFFSET = (INITIAL_MEM_SIZE - PTSIZE),
 
             // Put the DMA Pages just under the Page Table
             MBOX_DMA_PAGES = 64,    // must be <= 64
@@ -52,6 +52,9 @@ class VmmManager
             MBOX_DMA_ADDR = (HTABORG_OFFSET
                              - (MBOX_DMA_PAGES * MBOX_DMA_PAGESIZE)),
 
+            /** The base image and heap must be below this address. */
+            FIRST_RESERVED_PAGE = MBOX_DMA_ADDR,
+
             // Tells processor to ignore HRMOR
             FORCE_PHYS_ADDR = 0x8000000000000000,
         };
@@ -187,7 +190,7 @@ class VmmManager
 
         /**
          * @brief Allocates a block of virtual memory that extends the VMM
-         *          space upto 32MEG of Mainstore. 
+         *          space upto 32MEG of Mainstore.
          */
         static int mmExtend( void);
 
diff --git a/src/include/sys/mm.h b/src/include/sys/mm.h
index 34a60fc20..1b21816a8 100644
--- a/src/include/sys/mm.h
+++ b/src/include/sys/mm.h
@@ -96,12 +96,20 @@ int mm_remove_pages(PAGE_REMOVAL_OPS i_op, void* i_vaddr, uint64_t i_size);
  */
 int mm_set_permission(void* va, uint64_t size, uint64_t access_type);
 
+enum MM_EXTEND_SIZE
+{
+    MM_EXTEND_FULL_CACHE,      //< Extend memory to include full cache (8mb).
+    MM_EXTEND_REAL_MEMORY,     //< Extend memory into real mainstore.
+};
+
 /** @fn mm_extend()
- *  @brief System call to extend Memory to 32MEG
+ *  @brief System call to extend memory.
+ *
+ *  @param[in] i_size - Amount to extend memory by.
  *
  *  @return int - 0 for successful extension of memory, non-zero otherwise
  */
-int mm_extend(void);
+int mm_extend(MM_EXTEND_SIZE i_size = MM_EXTEND_REAL_MEMORY);
 
 /** @fn mm_linear_map()
  *  @brief   Allocates a block of memory of the given size at a specified
@@ -126,7 +134,7 @@ int mm_linear_map(void *i_paddr, uint64_t i_size);
 void mm_icache_invalidate(void * i_addr, size_t i_cpu_word_count);
 
 /** @fn mm_virt_to_phys()
- *  @brief System call to return the physical address backing a 
+ *  @brief System call to return the physical address backing a
  *      virtual address
  *
  *  @param[in] i_vaddr - Virtual address to translate
diff --git a/src/kernel/misc.C b/src/kernel/misc.C
index 329bc4fbc..d1a4b2833 100644
--- a/src/kernel/misc.C
+++ b/src/kernel/misc.C
@@ -29,6 +29,9 @@
 #include <assert.h>
 #include <kernel/terminate.H>
 #include <kernel/hbterminatetypes.H>
+#include <sys/mm.h>
+#include <errno.h>
+#include <kernel/pagemgr.H>
 
 extern "C"
     void kernel_shutdown(size_t, uint64_t, uint64_t, uint64_t) NO_RETURN;
@@ -231,4 +234,58 @@ namespace KernelMisc
         // the process.
         kassert(false);
     }
+
+    int expand_full_cache()
+    {
+        static bool executed = false;
+
+        if (executed) // Why are we being called a second time?
+        {
+            return -EFAULT;
+        }
+
+        uint64_t* startAddr = NULL;
+        uint64_t* endAddr = NULL;
+
+        switch(CpuID::getCpuType())
+        {
+            case CORE_POWER8_MURANO:
+            case CORE_POWER8_VENICE:
+                /* TODO: RTC 52972 - Change to INITIAL_MEM_SIZE once
+                 *                   SLW images are put in the "right" spot.
+                 */
+                startAddr =
+                    reinterpret_cast<uint64_t*>((OUTPUT_PORE_IMG_ADDR +
+                                                 MAX_OUTPUT_PORE_IMG_SIZE));
+                endAddr =
+                    reinterpret_cast<uint64_t*>(8 * MEGABYTE);
+
+            default:
+                break;
+        }
+
+        if (startAddr != NULL)
+        {
+            populate_cache_lines(startAddr, endAddr);
+            size_t pages = (reinterpret_cast<uint64_t>(endAddr) -
+                            reinterpret_cast<uint64_t>(startAddr)) / PAGESIZE;
+
+            PageManager::addMemory(reinterpret_cast<uint64_t>(startAddr),
+                                   pages);
+        }
+
+        executed = true;
+        return 0;
+    }
+
+    void populate_cache_lines(uint64_t* i_start, uint64_t* i_end)
+    {
+        size_t cache_line_size = getCacheLineWords();
+
+        while(i_start != i_end)
+        {
+            dcbz(i_start);
+            i_start += cache_line_size;
+        }
+    }
 };
diff --git a/src/kernel/pagemgr.C b/src/kernel/pagemgr.C
index ca534efdd..f1a1f2976 100644
--- a/src/kernel/pagemgr.C
+++ b/src/kernel/pagemgr.C
@@ -198,12 +198,9 @@ PageManager::PageManager()
 
     // Populate L3 cache lines.
     uint64_t* cache_line = reinterpret_cast<uint64_t*>(addr);
-    uint64_t* end_cache_line = (uint64_t*) VmmManager::FULL_MEM_SIZE;
-    while (cache_line != end_cache_line)
-    {
-        dcbz(cache_line);
-        cache_line += getCacheLineWords();
-    }
+    uint64_t* end_cache_line = (uint64_t*) VmmManager::INITIAL_MEM_SIZE;
+    KernelMisc::populate_cache_lines(cache_line, end_cache_line);
+
 
     // Allocate pages
     iv_heapKernel.addMemory( addr, RESERVED_PAGES );
diff --git a/src/kernel/syscall.C b/src/kernel/syscall.C
index 6535e7c19..b2a72d420 100644
--- a/src/kernel/syscall.C
+++ b/src/kernel/syscall.C
@@ -804,18 +804,37 @@ namespace Systemcalls
     }
 
     /**
-     * Allocates a block of virtual memory that extends the VMM
-     * space upto 32MEG of Mainstore.
-     * @param[in] t: The task used to extend Memory 
+     * Extends the initial footprint of the image further into memory.
+     *
+     * Depending on the syscall parameter, we will either switch from 4MB
+     * to 8MB cache-contained mode or expand into 32MB of space using real
+     * system memory.
+
+     * @param[in] t: The task used to extend Memory
      */
     void MmExtend(task_t* t)
     {
-        TASK_SETRTN(t, VmmManager::mmExtend());
+        uint64_t size = TASK_GETARG0(t);
+
+        switch (size)
+        {
+            case MM_EXTEND_FULL_CACHE:
+                TASK_SETRTN(t, KernelMisc::expand_full_cache());
+                break;
+
+            case MM_EXTEND_REAL_MEMORY:
+                TASK_SETRTN(t, VmmManager::mmExtend());
+                break;
+
+            default:
+                TASK_SETRTN(t, -EINVAL);
+                break;
+        }
     }
 
     /**
      * Allocates a block of memory of the given size
-     * to at a specified physical address 
+     * to at a specified physical address
      */
     void MmLinearMap(task_t* t)
     {
diff --git a/src/lib/syscall_mm.C b/src/lib/syscall_mm.C
index 7b6ba672d..5bb85a4fb 100644
--- a/src/lib/syscall_mm.C
+++ b/src/lib/syscall_mm.C
@@ -93,16 +93,16 @@ uint64_t mm_virt_to_phys( void* i_vaddr )
 }
 
 /**
- * System call to extend Memory to 32Meg. 
+ * System call to extend Memory to 32Meg.
  */
-int mm_extend(void)
+int mm_extend(MM_EXTEND_SIZE i_size)
 {
-    return (int64_t)_syscall0(MM_EXTEND);
+    return (int64_t)_syscall1(MM_EXTEND, reinterpret_cast<void*>(i_size));
 }
 
 
 /**
- * System call to create a block of memory at a specific physical address 
+ * System call to create a block of memory at a specific physical address
  */
 int mm_linear_map(void *i_paddr, uint64_t i_size)
 {
diff --git a/src/usr/hwpf/hwp/thread_activate/thread_activate.C b/src/usr/hwpf/hwp/thread_activate/thread_activate.C
index 20a345e47..5c192e7ec 100644
--- a/src/usr/hwpf/hwp/thread_activate/thread_activate.C
+++ b/src/usr/hwpf/hwp/thread_activate/thread_activate.C
@@ -40,7 +40,7 @@
 
 #include    <devicefw/userif.H>
 #include    <sys/misc.h>
-
+#include    <sys/mm.h>
 #include    <proc_thread_control.H>
 
 //  targeting support
@@ -220,6 +220,14 @@ void activate_threads( errlHndl_t& io_rtaskRetErrl )
 
     }
 
+    // Reclaim remainder of L3 cache if available.
+    // TODO: RTC 49137: Add calls to MVPD and PNOR to make decision.
+    if (!TARGETING::is_vpo())
+    {
+        mm_extend(MM_EXTEND_FULL_CACHE);
+    }
+
+
     TRACFCOMP( g_fapiTd,
                "activate_threads exit" );
 
diff --git a/src/usr/runtime/populate_attributes.C b/src/usr/runtime/populate_attributes.C
index 51b96668a..cba9612bc 100644
--- a/src/usr/runtime/populate_attributes.C
+++ b/src/usr/runtime/populate_attributes.C
@@ -36,6 +36,7 @@
 #include <targeting/common/targetservice.H>
 #include <targeting/common/utilFilter.H>
 #include <runtime/runtime_reasoncodes.H>
+#include <sys/mm.h> //@fixme RTC:49509 - remove if no longer needed.
 #include "common/hsvc_attribute_structs.H"
 //#include <arch/ppc.H> //for MAGIC_INSTRUCTION
 
@@ -60,7 +61,7 @@ TRAC_INIT(&g_trac_runtime, "RUNTIME", 4096);
     _cur_header->offset = (_output_ptr - _beginning); \
     memcpy( _output_ptr, &result_##__fid, sizeof(fapi::__fid##_Type) ); \
     _output_ptr += sizeof(fapi::__fid##_Type); \
-    (*_num_attr)++; 
+    (*_num_attr)++;
 
 /**
  * @brief Read a Privileged FAPI attribute and stick it into mainstore
@@ -80,7 +81,7 @@ TRAC_INIT(&g_trac_runtime, "RUNTIME", 4096);
     _cur_header->offset = (_output_ptr - _beginning); \
     memcpy( _output_ptr, &result_##__fid, sizeof(fapi::__fid##_Type) ); \
     _output_ptr += sizeof(fapi::__fid##_Type); \
-    (*_num_attr)++; 
+    (*_num_attr)++;
 
 /**
  * @brief Read the HUID attribute from targeting and stick it into mainstore
@@ -94,7 +95,7 @@ TRAC_INIT(&g_trac_runtime, "RUNTIME", 4096);
     _cur_header->offset = (_output_ptr - _beginning); \
     memcpy( _output_ptr, &_huid_temp, sizeof(uint32_t) ); \
     _output_ptr += sizeof(uint32_t); \
-    (*_num_attr)++; 
+    (*_num_attr)++;
 
 /**
  * @brief Read the PHYS_PATH attribute from targeting and stick it into mainstore
@@ -117,7 +118,7 @@ TRAC_INIT(&g_trac_runtime, "RUNTIME", 4096);
     _cur_header = &(_all_headers[(*_num_attr)]); \
     _cur_header->id = hsvc_attr_header_t::NO_ATTRIBUTE; \
     _cur_header->sizeBytes = 0; \
-    _cur_header->offset = 0; 
+    _cur_header->offset = 0;
 
 
 namespace RUNTIME
@@ -134,7 +135,7 @@ struct system_data_t
     // header data that HostServices uses
     hsvc_system_data_t hsvc;
 
-    // actual data content 
+    // actual data content
     hsvc_attr_header_t attrHeaders[MAX_ATTRIBUTES];
     char attributes[MAX_ATTRIBUTES*sizeof(uint32_t)];
 };
@@ -158,9 +159,9 @@ struct node_data_t
     // header data that HostServices uses
     hsvc_node_data_t hsvc;
 
-    // actual data content 
+    // actual data content
     hsvc_proc_header_t procs[MAX_PROCS_RSV];
-    hsvc_ex_header_t ex[MAX_PROCS_RSV*MAX_EX_PER_PROC];    
+    hsvc_ex_header_t ex[MAX_PROCS_RSV*MAX_EX_PER_PROC];
     hsvc_attr_header_t procAttrHeaders[MAX_PROCS_RSV][NUM_PROC_ATTRIBUTES];
     hsvc_attr_header_t exAttrHeaders[MAX_PROCS_RSV*MAX_EX_PER_PROC][NUM_EX_ATTRIBUTES];
     char attributes[MAX_ATTRIBUTES*sizeof(uint32_t)];
@@ -169,8 +170,8 @@ struct node_data_t
 
 //@fixme  RTC:49509
 // Steal the unused fake pnor space until we have mainstore
-#define SYSTEM_DATA_POINTER (5*MEGABYTE)
-#define NODE_DATA_POINTER (5*MEGABYTE+512*KILOBYTE)
+#define SYSTEM_DATA_POINTER (64*MEGABYTE)
+#define NODE_DATA_POINTER (SYSTEM_DATA_POINTER+512*KILOBYTE)
 
 
 /**
@@ -188,6 +189,8 @@ errlHndl_t populate_system_attributes( void )
         TRACFCOMP( g_trac_runtime, "-SYSTEM-" );
 
         // allocate memory and fill it with some junk data
+        // @fixme RTC:49509 - remove mm_linear_map and put in real HDAT.
+        mm_linear_map(reinterpret_cast<void*>(SYSTEM_DATA_POINTER), 4*MEGABYTE);
         system_data_t* sys_data =
           reinterpret_cast<system_data_t*>(SYSTEM_DATA_POINTER);
         memset( sys_data, 'A', sizeof(system_data_t) );
@@ -310,7 +313,7 @@ errlHndl_t populate_node_attributes( uint64_t i_nodeNum )
                          (const_cast<TARGETING::Target*>(all_procs[p])) );
 
             // Compute the processor id to match what HDAT uses
-            uint64_t node_id = 
+            uint64_t node_id =
               all_procs[p]->getAttr<TARGETING::ATTR_FABRIC_NODE_ID>();
             uint64_t chip_id =
               all_procs[p]->getAttr<TARGETING::ATTR_FABRIC_CHIP_ID>();
@@ -328,19 +331,19 @@ errlHndl_t populate_node_attributes( uint64_t i_nodeNum )
             // Prepare the variables for the HSVC_LOAD_ATTR calls
             _all_headers = &(node_data->procAttrHeaders[p][0]);
             _num_attr = &(node_data->procs[p].numAttr);
-            _target = &fapi_proc; 
+            _target = &fapi_proc;
 
             // Fill up the attributes
             ADD_HUID( (all_procs[p]) ); // for debug
             ADD_PHYS_PATH( (all_procs[p]) );
-            HSVC_LOAD_ATTR_P( ATTR_EC ); 
+            HSVC_LOAD_ATTR_P( ATTR_EC );
             // Use a generated file for the list of attributes to load
             #include "common/hsvc_procdata.C"
 
             // Add an empty attribute header to signal the end
             EMPTY_ATTRIBUTE;
 
-            
+
             // Loop around all of the EX chiplets for this proc
             TARGETING::TargetHandleList all_ex;
             TARGETING::getChildChiplets( all_ex, all_procs[p],
@@ -372,7 +375,7 @@ errlHndl_t populate_node_attributes( uint64_t i_nodeNum )
                 // Prepare the variables for the HSVC_LOAD_ATTR calls
                 _all_headers = &(node_data->exAttrHeaders[next_ex][0]);
                 _num_attr = &(node_data->ex[next_ex].numAttr);
-                _target = &fapi_ex; 
+                _target = &fapi_ex;
 
                 // Fill up the attributes
                 ADD_HUID( (all_ex[e]) ); // for debug
@@ -434,7 +437,7 @@ errlHndl_t populate_attributes( void )
 
     do {
         //@todo : Remove this before RTC:49137 is merged, fix with RTC:49509
-        // Skip this in VPO  
+        // Skip this in VPO
         if( TARGETING::is_vpo() )
         {
             TRACFCOMP( g_trac_runtime, "Skipping RUNTIME::populate_attributes in VPO mode" );
diff --git a/src/usr/runtime/test/runtimeattrstest.H b/src/usr/runtime/test/runtimeattrstest.H
index 037c779e3..bdf7b430e 100644
--- a/src/usr/runtime/test/runtimeattrstest.H
+++ b/src/usr/runtime/test/runtimeattrstest.H
@@ -77,7 +77,7 @@ class RuntimeTest: public CxxTest::TestSuite
         hsvc_attr_header_t* headers = reinterpret_cast<hsvc_attr_header_t*>
           ((beginning+sysdata->offset));
 
-        //Grab a couple of arbitrary attributes        
+        //Grab a couple of arbitrary attributes
         ReturnCode l_rc;
 
         fapi::ATTR_FREQ_PB_Type freq = 0;
@@ -161,15 +161,15 @@ class RuntimeTest: public CxxTest::TestSuite
         }
 
         //Make sure that the offsets are reasonable
-        if( nodedata->procOffset == 0 ) 
+        if( nodedata->procOffset == 0 )
         {
             TS_FAIL("testVerifyNodeAttributes> procOffset is invalid");
         }
-        if( nodedata->exOffset == 0 ) 
+        if( nodedata->exOffset == 0 )
         {
             TS_FAIL("testVerifyNodeAttributes> exOffset is invalid");
         }
-        if( nodedata->procOffset == nodedata->exOffset ) 
+        if( nodedata->procOffset == nodedata->exOffset )
         {
             TS_FAIL("testVerifyNodeAttributes> offsets are equal - that is bad");
         }
@@ -196,7 +196,7 @@ class RuntimeTest: public CxxTest::TestSuite
             fapi::Target* proc = getProcTarget( curproc->procid );
             TRACDCOMP( g_trac_runtime, "procid = %d", curproc->procid );
 
-            //Grab a couple of arbitrary attributes        
+            //Grab a couple of arbitrary attributes
             hsvc_attr_header_t* headers = reinterpret_cast<hsvc_attr_header_t*>
               ((beginning+curproc->offset));
             ReturnCode l_rc;
@@ -292,7 +292,7 @@ class RuntimeTest: public CxxTest::TestSuite
                                             curex->chiplet );
             TRACDCOMP( g_trac_runtime, "procid = %d, chiplet = %d", curex->parent_procid, curex->chiplet );
 
-            //Grab a couple of arbitrary attributes        
+            //Grab a couple of arbitrary attributes
             hsvc_attr_header_t* headers = reinterpret_cast<hsvc_attr_header_t*>
               ((beginning+curex->offset));
             ReturnCode l_rc;
@@ -377,11 +377,11 @@ class RuntimeTest: public CxxTest::TestSuite
     //@fixme  RTC:49509
     hsvc_system_data_t* getSysPtr(void)
     {
-        return( (hsvc_system_data_t*)(5*MEGABYTE) );
+        return( (hsvc_system_data_t*)(64*MEGABYTE) );
     };
     hsvc_node_data_t* getNodePtr(void)
     {
-        return( (hsvc_node_data_t*)(5*MEGABYTE+512*KILOBYTE) );
+        return( (hsvc_node_data_t*)(64*MEGABYTE+512*KILOBYTE) );
     };
 
     //utility to fetch a proc target based on a procid
@@ -410,7 +410,7 @@ class RuntimeTest: public CxxTest::TestSuite
         TARGETING::Target* proc_target = NULL;
         for( size_t p = 0; p < all_procs.size(); p++ )
         {
-            uint64_t node_id = 
+            uint64_t node_id =
               all_procs[p]->getAttr<TARGETING::ATTR_FABRIC_NODE_ID>();
             uint64_t CHIP_UNIT_POS =
               all_procs[p]->getAttr<TARGETING::ATTR_FABRIC_CHIP_ID>();