IOTA CME

- Default is to use PK

Change-Id: I7317f63558e42f098f8669dda9853bf99898e30a
RTC: 167247
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/42396
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: CHRISTOPHER M. RIEDL <cmriedl@us.ibm.com>
Reviewed-by: BRIAN D. VICTOR <brian.d.victor1@ibm.com>
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>

31 files changed, 1227 insertions, 167 deletions

diff --git a/import/chips/p9/procedures/ppe/iota/iota.c b/import/chips/p9/procedures/ppe/iota/iota.c
index 5068912a..d4c098c0 100644
--- a/import/chips/p9/procedures/ppe/iota/iota.c
+++ b/import/chips/p9/procedures/ppe/iota/iota.c
@@ -26,6 +26,10 @@
 #include "iota_uih.h"
 #include "iota_ppe42.h"
 
+// Force all kernel variables into .sdata
+// Order controlled by linker script
+uint64_t ppe42_64bit_timebase __attribute__((section(".sdata.ppe42_64bit_timebase"))) = 0;
+
 iotaMachineState g_iota_machine_state_stack[(IOTA_MAX_NESTED_INTERRUPTS)] __attribute__((aligned(8))) =
 {
     [ 0 ... (IOTA_MAX_NESTED_INTERRUPTS - 1) ] = IOTA_MACHINE_STATE_INIT
@@ -55,7 +59,7 @@ void _iota_evaluate_idle_tasks()
                 iota_halt();
             }
 
-            g_iota_idle_task_list[idle_task_idx].function(idle_task_idx);
+            g_iota_idle_task_list[idle_task_idx].function(idle_task_idx, 0);
 
 #if IOTA_AUTO_DISABLE_IDLE_TASKS
             uint32_t ctx = mfmsr();
@@ -74,16 +78,32 @@ extern void main(void);
 void _iota_boot()
 {
     g_iota_curr_machine_state_ptr = g_iota_machine_state_stack;
+
+    // Can CME be reset w/o reload?
+    // If see need to clean up entire state of ppe42 to known state
+    // enable DEC timer w interrupts
+
+    mtmsr(IOTA_INITIAL_MSR);
+
+    // DEC timer setup
+    mtspr(SPRN_DEC, 0xffffffff);
+    mtspr(SPRN_TSR, TSR_DIS);
+    or_spr(SPRN_TCR, TCR_DIE);
+
+    __hwmacro_setup(); // Configures external interrupt registers
+
     main();
 }
 
 void iota_run()
 {
+    mtmsr(IOTA_DEFAULT_MSR);
+
     while(1)
     {
         // Run any enabled idle tasks
         _iota_evaluate_idle_tasks();
-        // FIXME / TODO load default MSR here?
+
         uint32_t ctx = mfmsr() | MSR_WE | MSR_EE ;
         mtmsr(ctx);
     }
@@ -110,7 +130,8 @@ void iota_set_idle_task_state(uint32_t state, uint32_t idle_task_idx)
 void _iota_schedule(uint32_t schedule_reason)
 {
     // Increment machine state pointer
-    if(g_iota_curr_machine_state_ptr < &g_iota_machine_state_stack[IOTA_MAX_NESTED_INTERRUPTS - 1])
+    if(g_iota_curr_machine_state_ptr <
+       &g_iota_machine_state_stack[IOTA_MAX_NESTED_INTERRUPTS - 1])
     {
         g_iota_curr_machine_state_ptr++;
     }
@@ -119,20 +140,22 @@ void _iota_schedule(uint32_t schedule_reason)
         iota_halt();
     }
 
-    // TODO call appropriate interrupt handler here
+    // call appropriate interrupt handler here
     uint32_t task_idx;
 
     switch(schedule_reason)
     {
         case _IOTA_SCHEDULE_REASON_EXT:
+
             task_idx = iota_uih();
 
             if(g_iota_task_list[task_idx] != IOTA_NO_TASK)
             {
-                uint32_t ctx = mfmsr();
-                wrteei(1);
-                g_iota_task_list[task_idx](task_idx);
-                mtmsr(ctx);
+                //uint32_t ctx = mfmsr();
+                uint32_t irq = cntlz64(g_ext_irq_vector);
+                mtmsr(IOTA_DEFAULT_MSR);
+                g_iota_task_list[task_idx](task_idx, irq);
+                //mtmsr(ctx);
             }
             else
             {
@@ -150,11 +173,14 @@ void _iota_schedule(uint32_t schedule_reason)
             break;
     }
 
-    // TODO check for idle tasks here
-    // Rationale: if the g_iota_curr_machine_state_ptr == &g_iota_curr_machine_state_ptr[0],
-    //            then all interrupt tasks must be completed since this is the last
-    //            context about to be restored, which means enabled idle tasks
-    //            can be executed
+    wrteei(0);
+
+    // Check for idle tasks here
+    // Rationale: if the g_iota_curr_machine_state_ptr ==
+    //            &g_iota_curr_machine_state_ptr[0],
+    //            then all interrupt tasks must be completed since this is the
+    //            last context about to be restored, which means enabled idle
+    //            tasks can be executed
     iotaMachineState* p = g_iota_curr_machine_state_ptr;
 
     if(--p == g_iota_machine_state_stack)
@@ -175,3 +201,8 @@ void _iota_schedule(uint32_t schedule_reason)
         iota_halt();
     }
 }
+
+uint64_t pk_timebase_get()
+{
+    return (ppe42_64bit_timebase & 0xffffffff00000000ull) | (~(mfspr(SPRN_DEC)) + 1);
+}
diff --git a/import/chips/p9/procedures/ppe/iota/iota.h b/import/chips/p9/procedures/ppe/iota/iota.h
index d2c628ff..d68c0141 100644
--- a/import/chips/p9/procedures/ppe/iota/iota.h
+++ b/import/chips/p9/procedures/ppe/iota/iota.h
@@ -70,38 +70,57 @@ void iota_set_idle_task_state(uint32_t state, uint32_t idle_task_idx);
 #define IOTA_IDLE_DISABLED  0x00000000
 #define IOTA_IDLE_ENABLED   0x00000001
 
+#if !defined(IOTA_INITIAL_MSR)
+    #define IOTA_INITIAL_MSR (MSR_ME | MSR_IS0 | MSR_IS1 | MSR_IS2 | MSR_IPE)
+#endif
+
 #if !defined(IOTA_DEFAULT_MSR)
-    #define IOTA_DEFAULT_MSR    0x00000000
+#define IOTA_DEFAULT_MSR    (MSR_UIE | MSR_EE | MSR_ME | MSR_IS0 | \
+                             MSR_IS1 | MSR_IS2 | MSR_IPE | MSR_SEM6)
 #endif
 
 ///
 
 /// IOTA -- CONVENIENCE MACROS
-// This below macro-fu is an adaptation of a similar macro in the Linux kernel (BUILD_BUG_ON)
-// Allows compile time checking of conditions using sizeof()
+// This below macro-fu is an adaptation of a similar macro in the Linux kernel
+// (BUILD_BUG_ON). Allows compile time checking of conditions using sizeof()
 #define IOTA_TOKEN_PASTE0(a,b) a ## b
 #define IOTA_TOKEN_PASTE1(a,b) IOTA_TOKEN_PASTE0(a,b)
-#define IOTA_COMPILE_TIME_UNIQUE IOTA_TOKEN_PASTE1(_iota_compile_unique_,__COUNTER__)
-#define IOTA_COMPILE_TIME_CHECK(condition) extern int IOTA_COMPILE_TIME_UNIQUE [!!(condition)-1];
+#define IOTA_COMPILE_TIME_UNIQUE \
+    IOTA_TOKEN_PASTE1(_iota_compile_unique_,__COUNTER__)
+#define IOTA_COMPILE_TIME_CHECK(condition) \
+    extern int IOTA_COMPILE_TIME_UNIQUE [!!(condition)-1];
+
+#define SECTION_SBSS __attribute__((section(".sbss")))
+#define SECTION_SDATA __attribute__((section(".sdata")))
+#define SECTION(a) __attribute__((section(a)))
 
 #define IOTA_TASK(function) ((iotaTaskFuncPtr)function)
 
 #define IOTA_TIMER_HANDLER(function) ((iotaTimerFuncPtr)function)
-#define IOTA_DEC_HANDLER(function) g_iota_dec_handler = IOTA_TIMER_HANDLER(function);
-#define IOTA_FIT_HANDLER(function) g_iota_fit_handler = IOTA_TIMER_HANDLER(function);
+
+#define IOTA_DEC_HANDLER(function) g_iota_dec_handler \
+        = IOTA_TIMER_HANDLER(function);
+
+#define IOTA_FIT_HANDLER(function) g_iota_fit_handler \
+        = IOTA_TIMER_HANDLER(function);
 
 #define IOTA_BEGIN_IDLE_TASK_TABLE \
-    iotaIdleTask g_iota_idle_task_list[] = {
+    iotaIdleTask g_iota_idle_task_list[] \
+    SECTION(".sdata.g_iota_idle_task_list") = {
 #define IOTA_END_IDLE_TASK_TABLE \
     }; \
-    uint32_t const g_iota_idle_task_list_size = (uint32_t)(sizeof(g_iota_idle_task_list)/sizeof(iotaIdleTask));
+    uint32_t const g_iota_idle_task_list_size \
+        = (uint32_t)(sizeof(g_iota_idle_task_list)/sizeof(iotaIdleTask));
 
 #define IOTA_BEGIN_TASK_TABLE \
-    iotaTaskFuncPtr g_iota_task_list[] = {
+    iotaTaskFuncPtr g_iota_task_list[] SECTION(".sdata.g_iota_task_list") = {
 #define IOTA_END_TASK_TABLE \
     }; \
-    uint32_t const g_iota_task_list_size = (uint32_t)(sizeof(g_iota_task_list)/sizeof(iotaTaskFuncPtr)); \
-    IOTA_COMPILE_TIME_CHECK((sizeof(g_iota_task_list)/sizeof(iotaTaskFuncPtr)) == (IOTA_NUM_EXT_IRQ_PRIORITIES));
+    uint32_t const g_iota_task_list_size \
+        = (uint32_t)(sizeof(g_iota_task_list)/sizeof(iotaTaskFuncPtr)); \
+    IOTA_COMPILE_TIME_CHECK((sizeof(g_iota_task_list)/sizeof(iotaTaskFuncPtr)) \
+                            == (IOTA_NUM_EXT_IRQ_PRIORITIES));
 
 #define IOTA_MACHINE_STATE_INIT { \
         IOTA_32U_ARR_INIT, \
@@ -150,7 +169,7 @@ typedef struct
     uint32_t padding; // needs to be 8B aligned
 } iotaMachineState;
 
-typedef void (*iotaTaskFuncPtr )(uint32_t arg);
+typedef void (*iotaTaskFuncPtr )(uint32_t arg, uint32_t irq);
 typedef void (*iotaTimerFuncPtr)(void        );
 
 typedef struct
diff --git a/import/chips/p9/procedures/ppe/iota/iota_debug_ptrs.c b/import/chips/p9/procedures/ppe/iota/iota_debug_ptrs.c
new file mode 100644
index 00000000..0b713d7b
--- /dev/null
+++ b/import/chips/p9/procedures/ppe/iota/iota_debug_ptrs.c
@@ -0,0 +1,42 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: import/chips/p9/procedures/ppe/iota/iota_debug_ptrs.c $       */
+/*                                                                        */
+/* OpenPOWER HCODE Project                                                */
+/*                                                                        */
+/* COPYRIGHT 2017                                                         */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+#include "iota_trace.h"
+#include "iota_debug_ptrs.h"
+#include "iota_app_cfg.h"
+
+iota_debug_ptrs_t iota_debug_ptrs __attribute__((section(".debug_ptrs"))) =
+{
+    .debug_ptrs_size            = sizeof(iota_debug_ptrs_t),
+    .debug_ptrs_version         = IOTA_DEBUG_PTRS_VERSION,
+
+#if PK_TRACE_SUPPORT
+    .debug_trace_ptr            = &g_pk_trace_buf,
+    .debug_trace_size           = sizeof(g_pk_trace_buf),
+#else
+    .debug_trace_ptr            = 0,
+    .debug_trace_size           = 0,
+#endif /* PK_TRACE_SUPPORT */
+
+};
diff --git a/import/chips/p9/procedures/ppe/iota/iota_debug_ptrs.h b/import/chips/p9/procedures/ppe/iota/iota_debug_ptrs.h
new file mode 100644
index 00000000..f0295c79
--- /dev/null
+++ b/import/chips/p9/procedures/ppe/iota/iota_debug_ptrs.h
@@ -0,0 +1,47 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: import/chips/p9/procedures/ppe/iota/iota_debug_ptrs.h $       */
+/*                                                                        */
+/* OpenPOWER HCODE Project                                                */
+/*                                                                        */
+/* COPYRIGHT 2017                                                         */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+#if !defined(__IOTA_DEBUG_PTRS_H__)
+#define __IOTA_DEBUG_PTRS_H__
+/**
+ * @file iota_debug_ptrs.h
+ * @brief Structure for a table of pointers to debug data
+ */
+
+#define IOTA_DEBUG_PTRS_VERSION  1
+
+typedef struct
+{
+    // The size and version of this structure
+    unsigned short   debug_ptrs_size;
+    unsigned short   debug_ptrs_version;
+
+    // Trace buffer location and size
+    void*           debug_trace_ptr;
+    unsigned long   debug_trace_size;
+
+
+} iota_debug_ptrs_t;
+
+#endif
diff --git a/import/chips/p9/procedures/ppe/iota/iota_ppe42.S b/import/chips/p9/procedures/ppe/iota/iota_ppe42.S
index e1e4f47f..f00fab4d 100644
--- a/import/chips/p9/procedures/ppe/iota/iota_ppe42.S
+++ b/import/chips/p9/procedures/ppe/iota/iota_ppe42.S
@@ -24,6 +24,7 @@
 /* IBM_PROLOG_END_TAG                                                     */
 #include "iota.h"
 #include "iota_ppe42.h"
+#include "std_irq_config.h"
 
 .global g_iota_execution_stack
 
@@ -121,6 +122,9 @@ __iota_save_interrupt_state_and_schedule:
     mtcr0   %r9
     lwz     %r9, _IOTA_SAVE_CTR_OFFSET(%r3)
     mtctr   %r9
+    lvd     %d8, _IOTA_SAVE_SRR0_OFFSET(%r3)
+    mtsrr0  %r8
+    mtsrr1  %r9
     # Restore d8
     lvd     %d8, _IOTA_SAVE_D8_OFFSET(%r3)
     # Restore r3
@@ -131,3 +135,7 @@ __iota_save_interrupt_state_and_schedule:
 .global __iota_halt
 __iota_halt:
     trap
+
+## pull in the macro that calculates all the external interrupt config
+## defined in pk/std/std_irq_config.h
+    .std_irq_cfg_bitmaps
diff --git a/import/chips/p9/procedures/ppe/iota/iota_ppe42.h b/import/chips/p9/procedures/ppe/iota/iota_ppe42.h
index 8a6f29dd..8dcbc787 100644
--- a/import/chips/p9/procedures/ppe/iota/iota_ppe42.h
+++ b/import/chips/p9/procedures/ppe/iota/iota_ppe42.h
@@ -22,9 +22,26 @@
 /* permissions and limitations under the License.                         */
 /*                                                                        */
 /* IBM_PROLOG_END_TAG                                                     */
+
+#ifndef __ASSEMBLER__
+    #include <stdint.h>
+#endif
+
+#include "ppe42_asm.h"
+#include "ppe42_msr.h"
+#include "ppe42_spr.h"
+
 #ifndef __IOTA_PPE42_H__
 #define __IOTA_PPE42_H__
 
+#ifdef __PPE42_CORE_C__
+    #define IF__PPE42_CORE_C__(x) x
+    #define UNLESS__PPE42_CORE_C__(x)
+#else
+    #define IF__PPE42_CORE_C__(x)
+    #define UNLESS__PPE42_CORE_C__(x) x
+#endif
+
 #define _IOTA_SAVE_D0_OFFSET    0
 #define _IOTA_SAVE_D4_OFFSET    8
 #define _IOTA_SAVE_D6_OFFSET   16
@@ -69,8 +86,6 @@ b      __iota_save_interrupt_state_and_schedule
 // *INDENT-ON*
 #else
 
-#include "ppe42_msr.h"
-
 /// 8-bit MMIO Write
 #define out8(addr, data) \
     do {*(volatile uint8_t *)(addr) = (data);} while(0)
@@ -122,8 +137,109 @@ b      __iota_save_interrupt_state_and_schedule
         __d; \
     })
 
+/// CouNT Leading Zeros Word
+#define cntlzw(x) \
+    ({uint32_t __x = (x); \
+        uint32_t __lzw; \
+        asm volatile ("cntlzw %0, %1" : "=r" (__lzw) : "r" (__x)); \
+        __lzw;})
+
+/// CouNT Leading Zeros : uint32_t
+static inline int
+cntlz32(uint32_t x)
+{
+    return cntlzw(x);
+}
+
+/// CouNT Leading Zeros : uint64_t
+static inline int
+cntlz64(uint64_t x)
+{
+    if (x > 0xffffffff)
+    {
+        return cntlz32(x >> 32);
+    }
+    else
+    {
+        return 32 + cntlz32(x);
+    }
+}
 
-#endif // __ASSEMBLER__
 
+#define sync() asm volatile ("sync" : : : "memory")
+
+// machine context is simply the MSR
+typedef unsigned long PkMachineContext;
+
+/// Enter a critical section, saving the current machine
+/// context.
+UNLESS__PPE42_CORE_C__(extern)
+inline
+void pk_interrupt_disable(PkMachineContext* context)
+{
+    *context = mfmsr();
+    wrteei(0);
+}
+
+UNLESS__PPE42_CORE_C__(extern)
+inline
+void pk_machine_context_set(PkMachineContext* context)
+{
+    mtmsr(*context);
+}
+
+#define pk_critical_section_enter(pctx) \
+    pk_interrupt_disable(pctx)
+
+/// Exit a critical section by restoring the previous machine context.
+
+#define pk_critical_section_exit(pctx) \
+    pk_machine_context_set(pctx)
+
+typedef union
+{
+
+    uint32_t value;
+
+    struct
+    {
+
+        /// A flag indicating that PK is in thread mode after a call of
+        /// pk_start_threads().
+        unsigned thread_mode : 1;
+
+        /// If this field is non-zero then PK is processing an interrupt
+        /// and the \c irq field will contain the PkIrqId of the interrupt
+        /// that kicked off interrupt processing.
+        unsigned processing_interrupt : 1;
+
+        /// The priority of the currently running thread.  In an interrupt
+        /// context, this is the priority of the thread that was interrupted.
+        unsigned thread_priority : 6;
+
+        /// This bit tracks whether the current context can be discarded or
+        /// if the context must be saved.  If the processor takes an interrupt
+        /// and this bit is set, then the current context will be discarded.
+        /// This bit is set at the end of handling an interrupt and prior
+        /// to entering the wait enabled state.
+        unsigned discard_ctx : 1;
+
+        /// The PkIrqId of the currently running (or last run) handler.  If
+        /// \c processing_interrupt is set, then this is the
+        /// PkIrqId of the IRQ that is currently executing.
+        unsigned irq : 7;
+
+        /// Each PPE application will define (or not) the interpretation of
+        /// this field.  Since SPRG0 is saved and restored during during thread
+        /// context switches, this field can be used to record the progress of
+        /// individual threads.  The kernel and/or application will provide
+        /// APIs or macros to read and write this field.
+        unsigned app_specific : 16;
+
+    } fields;
+
+} __KernelContext;
+
+#endif // __ASSEMBLER__
 
 #endif // __IOTA_PPE42_H__
diff --git a/import/chips/p9/procedures/ppe/iota/iota_ppe42_vectors.S b/import/chips/p9/procedures/ppe/iota/iota_ppe42_vectors.S
index 782f63ad..b594e831 100644
--- a/import/chips/p9/procedures/ppe/iota/iota_ppe42_vectors.S
+++ b/import/chips/p9/procedures/ppe/iota/iota_ppe42_vectors.S
@@ -48,6 +48,7 @@ __data_storage:
 __instruction_storage:
     trap
 
+.global ppe42_64bit_timebase
 .global __external_interrupt
 .org    __vectors + 0x00a0
 __external_interrupt:
@@ -66,7 +67,17 @@ __program_exception:
 .global __dec_interrupt
 .org    __vectors + 0x0100
 __dec_interrupt:
-    __m_iota_interrupt_and_exception_handler _IOTA_SCHEDULE_REASON_DEC
+    stw     %r3,_IOTA_TEMPORARY_R3_STACK_OFFSET(%r1)
+    // Increment upper 32 bits of 64 bit timebase
+    lwz     %r3,ppe42_64bit_timebase@sda21(0)
+    addi    %r3,%r3,1
+    stw     %r3,ppe42_64bit_timebase@sda21(0)
+    _liwa   %r3, TSR_DIS
+    mttsr   %r3
+    lwz     %r3,_IOTA_TEMPORARY_R3_STACK_OFFSET(%r1)
+    rfi
+    // remove rfi and restore following line to share DEC timer ovrflow
+    //__m_iota_interrupt_and_exception_handler _IOTA_SCHEDULE_REASON_DEC
 
 .global __fit_interrupt
 .org    __vectors + 0x0120
diff --git a/import/chips/p9/procedures/ppe/iota/iota_trace.h b/import/chips/p9/procedures/ppe/iota/iota_trace.h
new file mode 100644
index 00000000..7de82b15
--- /dev/null
+++ b/import/chips/p9/procedures/ppe/iota/iota_trace.h
@@ -0,0 +1,181 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: import/chips/p9/procedures/ppe/iota/iota_trace.h $            */
+/*                                                                        */
+/* OpenPOWER HCODE Project                                                */
+/*                                                                        */
+/* COPYRIGHT 2017                                                         */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+#if !defined(__IOTA_TRACE__)
+#define __IOTA_TRACE__
+
+#include "pk_app_cfg.h"
+#include "pk_trace.h"
+
+// Enable PK application tracing for latency measurments
+#ifndef PK_TRACE_PERF_ENABLE
+    #define PK_TRACE_PERF_ENABLE 0
+#endif
+
+/// Enable PK application tracing (enabled by default)
+#ifndef PK_TRACE_ENABLE
+    #define PK_TRACE_ENABLE 1
+#endif
+
+/// Enable PK ctrl  (enabled by default)
+#ifndef PK_TRACE_CTRL_ENABLE
+    #define PK_TRACE_CTRL_ENABLE 1
+#endif
+
+/// Enable PK crit  (disabled by default)
+#ifndef PK_TRACE_CRIT_ENABLE
+    #define PK_TRACE_CRIT_ENABLE 0
+#endif
+
+/// Enable PK ckpt  (disabled by default)
+#ifndef PK_TRACE_CKPT_ENABLE
+    #define PK_TRACE_CKPT_ENABLE 0
+#endif
+
+/// Enable Debug suppress  (disabled by default)
+// a.k.a. enabled means turn off PK_TRACE(), but keep crit trace
+#ifndef PK_TRACE_DBG_SUPPRESS
+    #define PK_TRACE_DBG_SUPPRESS 0
+#endif
+
+/// Enable PK kernel tracing (disabled by default)
+#ifndef PK_KERNEL_TRACE_ENABLE
+    #define PK_KERNEL_TRACE_ENABLE 0
+#endif
+
+/// pk trace disabled implies no tracing at all
+//   override any other trace settings
+#if !PK_TRACE_ENABLE
+    #undef PK_TRACE_DBG_SUPPRESS
+    #undef PK_TRACE_CTRL_ENABLE
+    #undef PK_TRACE_CRIT_ENABLE
+    #undef PK_TRACE_CKPT_ENABLE
+
+    #define PK_TRACE_DBG_SUPPRESS 1
+    #define PK_TRACE_CTRL_ENABLE 0
+    #define PK_TRACE_CRIT_ENABLE 0
+    #define PK_TRACE_CKPT_ENABLE 0
+#endif
+
+// PK TRACE enabled implies all default tracing on.
+// PK TRACE enabled & PK CKPT DEBUG disabled implies PK CRIT INFO and CTRL ERROR tracing only.
+// PK TRACE enabled & PK CKPT DEBUG and CRIT INFO disabled implies PK CTRL ERROR tracing only.
+// PK TRACE enabled & PK CRIT INFO disabled && PK DEBUG disabled && PK ERROR disabled implies
+//          PK TRACE disabled
+#if PK_TRACE_ENABLE &&  PK_TRACE_DBG_SUPPRESS && !PK_TRACE_CRIT_ENABLE && !PK_TRACE_CKPT_ENABLE && !PK_TRACE_CTRL_ENABLE
+    #undef PK_TRACE_ENABLE
+    #define PK_TRACE_ENABLE 0
+#endif
+
+
+//Application trace macros
+#if !PK_TRACE_PERF_ENABLE
+    #define PK_TRACE_PERF(...)
+#else
+    #define PK_TRACE_PERF(...) PKTRACE(__VA_ARGS__)
+#endif
+
+#if PK_TRACE_DBG_SUPPRESS
+    #define PK_TRACE(...)
+    #define PK_TRACE_BIN(str, bufp, buf_size)
+#else
+    #define PK_TRACE(...) PKTRACE(__VA_ARGS__)
+    #define PK_TRACE_BIN(str, bufp, buf_size) PKTRACE_BIN(str, bufp, buf_size)
+#endif
+
+#if !PK_TRACE_CTRL_ENABLE
+    #define PK_TRACE_ERR(...)
+#else
+    #define PK_TRACE_ERR(...) PKTRACE(__VA_ARGS__)
+#endif
+
+#if !PK_TRACE_CRIT_ENABLE
+    #define PK_TRACE_INF(...)
+#else
+    #define PK_TRACE_INF(...) PKTRACE(__VA_ARGS__)
+#endif
+
+#if !PK_TRACE_CKPT_ENABLE
+    #define PK_TRACE_DBG(...)
+#else
+    #define PK_TRACE_DBG(...) PKTRACE(__VA_ARGS__)
+#endif
+
+/// The following macros are helper macros for tracing.  They should not be called
+/// directly.
+#define VARG_COUNT_HELPER(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N
+#define VARG_COUNT(...) VARG_COUNT_HELPER(, ##__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0)
+
+/// Trace macros for C functions
+#define HASH_ARG_COMBO(str, arg) \
+    ((((uint32_t)trace_ppe_hash(str, PK_TRACE_HASH_PREFIX)) << 16) | ((uint32_t)(arg) & 0x0000ffff))
+
+#define PKTRACE0(...) pk_trace_tiny() //will fail at compile time
+
+#define PKTRACE1(str) \
+    pk_trace_tiny((trace_ppe_hash(str, PK_TRACE_HASH_PREFIX) << 16))
+
+#define PKTRACE2(str, parm0) \
+    ((sizeof(parm0) <= 2)? \
+     pk_trace_tiny(HASH_ARG_COMBO(str, parm0)): \
+     pk_trace_big(HASH_ARG_COMBO(str, 1), ((uint64_t)parm0) << 32, 0))
+
+#define PKTRACE3(str, parm0, parm1) \
+    pk_trace_big(HASH_ARG_COMBO(str, 2), ((((uint64_t)parm0) << 32) | parm1), 0)
+
+#define PKTRACE4(str, parm0, parm1, parm2) \
+    pk_trace_big(HASH_ARG_COMBO(str, 3), ((((uint64_t)parm0) << 32) | parm1),\
+                 ((uint64_t)parm2) << 32 )
+
+#define PKTRACE5(str, parm0, parm1, parm2, parm3) \
+    pk_trace_big(HASH_ARG_COMBO(str, 4), ((((uint64_t)parm0) << 32) | parm1),\
+                 ((((uint64_t)parm2) << 32) | parm3) )
+
+#define PKTRACE6(...) pk_trace_tiny() //will fail at compile time
+#define PKTRACE7(...) pk_trace_tiny() //will fail at compile time
+
+#define PKTRACE_HELPER2(count, ...) PKTRACE ## count (__VA_ARGS__)
+#define PKTRACE_HELPER(count, ...) PKTRACE_HELPER2(count, __VA_ARGS__)
+
+#if (PK_TRACE_SUPPORT)
+#define PKTRACE(...) PKTRACE_HELPER(VARG_COUNT(__VA_ARGS__), __VA_ARGS__)
+#define PKTRACE_BIN(str, bufp, buf_size) \
+    pk_trace_binary(((buf_size < 255)? HASH_ARG_COMBO(str, buf_size): HASH_ARG_COMBO(str, 255)), bufp)
+#else
+#define PKTRACE(...)
+#define PKTRACE_BIN(str, bufp, buf_size)
+#endif //PK_TRACE_SUPPORT
+
+//Trace function prototypes
+void pk_trace_tiny(uint32_t i_parm);
+void pk_trace_big(uint32_t i_hash_and_count,
+                  uint64_t i_parm1, uint64_t i_parm2);
+void pk_trace_binary(uint32_t i_hash_and_size, void* bufp);
+//void pk_trace_set_timebase(PkTimebase timebase);
+
+
+
+
+
+#endif
diff --git a/import/chips/p9/procedures/ppe/iota/iota_uih.c b/import/chips/p9/procedures/ppe/iota/iota_uih.c
index 2dd3d1c7..62afce93 100644
--- a/import/chips/p9/procedures/ppe/iota/iota_uih.c
+++ b/import/chips/p9/procedures/ppe/iota/iota_uih.c
@@ -36,6 +36,8 @@ const uint64_t ext_irq_vectors_cme[IOTA_NUM_EXT_IRQ_PRIORITIES][2] =
     /* 0: IDX_PRTY_VEC    1: IDX_MASK_VEC */
     {
         IRQ_VEC_PRTY0_CME, /* 0: IDX_PRTY_LVL_HIPRTY */
+        IRQ_VEC_PRTY12_CME |
+        IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
         IRQ_VEC_PRTY8_CME |
@@ -50,6 +52,8 @@ const uint64_t ext_irq_vectors_cme[IOTA_NUM_EXT_IRQ_PRIORITIES][2] =
     },
     {
         IRQ_VEC_PRTY1_CME, /* 1: IDX_PRTY_LVL_DB3 */
+        IRQ_VEC_PRTY12_CME |
+        IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
         IRQ_VEC_PRTY8_CME |
@@ -63,6 +67,8 @@ const uint64_t ext_irq_vectors_cme[IOTA_NUM_EXT_IRQ_PRIORITIES][2] =
     },
     {
         IRQ_VEC_PRTY2_CME, /* 2: IDX_PRTY_LVL_DB2 */
+        IRQ_VEC_PRTY12_CME |
+        IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
         IRQ_VEC_PRTY8_CME |
@@ -75,6 +81,8 @@ const uint64_t ext_irq_vectors_cme[IOTA_NUM_EXT_IRQ_PRIORITIES][2] =
     },
     {
         IRQ_VEC_PRTY3_CME, /* 3: IDX_PRTY_LVL_SPWU */
+        IRQ_VEC_PRTY12_CME |
+        IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
         IRQ_VEC_PRTY8_CME |
@@ -86,6 +94,8 @@ const uint64_t ext_irq_vectors_cme[IOTA_NUM_EXT_IRQ_PRIORITIES][2] =
     },
     {
         IRQ_VEC_PRTY4_CME, /* 4: IDX_PRTY_LVL_WAKE */
+        IRQ_VEC_PRTY12_CME |
+        IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
         IRQ_VEC_PRTY8_CME |
@@ -96,6 +106,8 @@ const uint64_t ext_irq_vectors_cme[IOTA_NUM_EXT_IRQ_PRIORITIES][2] =
     },
     {
         IRQ_VEC_PRTY5_CME, /* 5: IDX_PRTY_LVL_STOP */
+        IRQ_VEC_PRTY12_CME |
+        IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
         IRQ_VEC_PRTY8_CME |
@@ -105,6 +117,8 @@ const uint64_t ext_irq_vectors_cme[IOTA_NUM_EXT_IRQ_PRIORITIES][2] =
     },
     {
         IRQ_VEC_PRTY6_CME, /* 6: IDX_PRTY_LVL_DB1 */
+        IRQ_VEC_PRTY12_CME |
+        IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
         IRQ_VEC_PRTY8_CME |
@@ -114,6 +128,8 @@ const uint64_t ext_irq_vectors_cme[IOTA_NUM_EXT_IRQ_PRIORITIES][2] =
 
     {
         IRQ_VEC_PRTY7_CME, /* 7: IDX_PRTY_LVL_DB0 */
+        IRQ_VEC_PRTY12_CME |
+        IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
         IRQ_VEC_PRTY8_CME |
@@ -122,27 +138,44 @@ const uint64_t ext_irq_vectors_cme[IOTA_NUM_EXT_IRQ_PRIORITIES][2] =
 
     {
         IRQ_VEC_PRTY8_CME, /* 8: IDX_PRTY_LVL_INTERCME_IN0 */
+        IRQ_VEC_PRTY12_CME |
+        IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME |
         IRQ_VEC_PRTY8_CME
     },
     {
         IRQ_VEC_PRTY9_CME, /* 9: IDX_PRTY_LVL_PMCR */
+        IRQ_VEC_PRTY12_CME |
+        IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME |
         IRQ_VEC_PRTY9_CME
     },
     {
         IRQ_VEC_PRTY10_CME, /* 10: IDX_PRTY_LVL_DISABLED */
+        IRQ_VEC_PRTY12_CME |
+        IRQ_VEC_PRTY11_CME |
         IRQ_VEC_PRTY10_CME
+    },
+    {
+        IRQ_VEC_PRTY11_CME, /* 11: IDX_PRTY_LVL_COMM_RECVD */
+        IRQ_VEC_PRTY12_CME |
+        IRQ_VEC_PRTY11_CME
+    },
+    {
+        IRQ_VEC_PRTY12_CME, /* 12: IDX_PRTY_LVL_DISABLED */
+        IRQ_VEC_PRTY12_CME
     }
 
+
 };
 
-uint8_t  g_current_prty_level = IOTA_NUM_EXT_IRQ_PRIORITIES - 1;
+uint32_t g_current_prty_level = IOTA_NUM_EXT_IRQ_PRIORITIES - 1;
 uint8_t  g_eimr_stack[IOTA_NUM_EXT_IRQ_PRIORITIES];
 int      g_eimr_stack_ctr = -1;
 uint64_t g_eimr_override_stack[IOTA_NUM_EXT_IRQ_PRIORITIES];
 uint64_t g_eimr_override = 0x0000000000000000;
+uint64_t g_ext_irq_vector = 0;
 
 // Unified IRQ priority and masking handler.
 // - Locates the highest priority IRQ task vector that has at least one of its
@@ -150,14 +183,14 @@ uint64_t g_eimr_override = 0x0000000000000000;
 uint32_t iota_uih(void)
 {
     uint32_t  iPrtyLvl = 0, bFound = 0;
-    uint64_t  ext_irq_vector;
+    //uint64_t  ext_irq_vector;
 
     // 1. Identify the priority level of the interrupt.
-    ext_irq_vector = in64(CME_LCL_EISTR);
+    g_ext_irq_vector = in64(CME_LCL_EISTR);
 
     do
     {
-        if(ext_irq_vectors_cme[iPrtyLvl][IDX_PRTY_VEC] & ext_irq_vector)
+        if(ext_irq_vectors_cme[iPrtyLvl][IDX_PRTY_VEC] & g_ext_irq_vector)
         {
             bFound = 1;
             break;
diff --git a/import/chips/p9/procedures/ppe/iota/iota_uih.h b/import/chips/p9/procedures/ppe/iota/iota_uih.h
index c3250c2f..5fe86457 100644
--- a/import/chips/p9/procedures/ppe/iota/iota_uih.h
+++ b/import/chips/p9/procedures/ppe/iota/iota_uih.h
@@ -36,6 +36,26 @@
 
 #include "iota.h"
 
+// Need to generically pull this in
+// iota_uih_cfg.h
+#include "iota_app_cfg.h"
+#include "std_register_addresses.h"
+#include "std_irq_config.h"
+
+extern uint32_t iota_uih(void);
+extern void __hwmacro_setup(void);
+
+extern uint64_t g_ext_irq_vector;
+
+// Needed by ppehw_common.h
+#define PK_IRQ_POLARITY_ACTIVE_LOW  0
+#define PK_IRQ_POLARITY_ACTIVE_HIGH 1
+#define PK_IRQ_TRIGGER_LEVEL_SENSITIVE 0
+#define PK_IRQ_TRIGGER_EDGE_SENSITIVE  1
+
+
+// Changed to pick this up from p9_cme_irq.h
+#if defined(__COMMENT_OUT__)
 // Priority Levels
 #define IDX_PRTY_LVL_HIPRTY         0
 #define IDX_PRTY_LVL_DB3            1
@@ -88,11 +108,10 @@ extern const uint64_t ext_irq_vectors_cme[IOTA_NUM_EXT_IRQ_PRIORITIES][2];
                               IRQ_VEC_PRTY9_CME | \
                               IRQ_VEC_PRTY10_CME )
 
-extern uint8_t  g_current_prty_level;
-extern uint8_t  g_eimr_stack[IOTA_NUM_EXT_IRQ_PRIORITIES];
+extern uint32_t g_current_prty_level;
+extern uint8_t  g_eimr_stack[IOTA_NUM_EXT_IRQ_PRIORITIES] SECTION_SBSS;
 extern int      g_eimr_stack_ctr;
-extern uint64_t g_eimr_override_stack[IOTA_NUM_EXT_IRQ_PRIORITIES];
+extern uint64_t g_eimr_override_stack[IOTA_NUM_EXT_IRQ_PRIORITIES]SECTION_SBSS;
 extern uint64_t g_eimr_override;
-extern uint32_t iota_uih(void);
-
+#endif
 #endif //__IOTA_UIH_H__
diff --git a/import/chips/p9/procedures/ppe/iota/link.ld b/import/chips/p9/procedures/ppe/iota/link.ld
index 484c4955..297f2514 100644
--- a/import/chips/p9/procedures/ppe/iota/link.ld
+++ b/import/chips/p9/procedures/ppe/iota/link.ld
@@ -24,31 +24,127 @@
 /* IBM_PROLOG_END_TAG                                                     */
 OUTPUT_FORMAT(elf32-powerpc)
 
+#define  __LINKERSCRIPT__
+#include "iota_lnk_cfg.h"
+
+#if ((SRAM_START % 512) != 0)
+#error SRAM_START must be 512 byte aligned
+#endif
+
 MEMORY
 {
-    sram : ORIGIN = 0xffff8000, LENGTH = 0x8000
+    sram : ORIGIN = SRAM_START, LENGTH = SRAM_LENGTH
 }
 
+// The linker will discard any symbols and sections it deems are unused.
+// Declare as EXTERN any symbols only used externally to ensure the linker
+// keeps them even though APP code never references them.
+EXTERN(iota_debug_ptrs);
+EXTERN(g_cme_magic_word);
+
 SECTIONS
 {
-    . = 0xffff8000;
+    . = SRAM_START;
+    _VECTOR_START = .;
+
+    .vectors _VECTOR_START  : { *(.vectors) } > sram
+
+    /*=====================================
+     * IMAGE HEADER
+     ======================================*/
+    _IMG_HEADER = _VECTOR_START +  PPE_HEADER_IMAGE_OFFSET;
+    PPE_HEADER_IMAGE_NAME _IMG_HEADER : { *(PPE_HEADER_IMAGE_NAME) } > sram
 
-    .vectors   : { *(.vectors) } > sram
-    . = ALIGN(512);
+    /*====================================
+     * Debug pointers table
+     ====================================*/
+    _DEBUG_PTRS_START = _VECTOR_START + PPE_DEBUG_PTRS_OFFSET;
+    .debug_ptrs _DEBUG_PTRS_START : { *(.debug_ptrs) } > sram
+
+    _DUMP_PTRS_START = _DEBUG_PTRS_START + PPE_DEBUG_PTRS_SIZE;
+    .dump_ptrs _DUMP_PTRS_START : { *(.dump_ptrs) } > sram
+
+    /*======================================
+     * Code
+     ======================================*/
     .text    . : { *(.text)    } > sram
 
-    . = ALIGN(8);
-    _SDA_BASE_ = . ;
-    .sdata   . : { *(.sdata*) } > sram
-    .sbss    . : { *(.sbss*)  } > sram
 
+    /*======================================
+     * Read only data
+     *====================================*/
+    .rodata . : { *(.rodata*) *(.got2) } > sram
+
+    /*===================================================
+     * Small data area 2. Must be 8 byte aligned
+     * .sdata2 and .sbss2 should be next to each other.
+     * .sdata2 is intended for read-only data (const).
+     * .sbss2 is for zero initailized or non-initialized
+     * const data, but the POWER EABI allow it to be written.
+     * This area is referenced by an offset to gpr2
+     ====================================================*/
     . = ALIGN(8);
     _SDA2_BASE_ = . ;
     .sdata2  . : { *(.sdata2*) } > sram
     .sbss2   . : { *(.sbss2*)  } > sram
 
-    .data    . : { *(.data*)   } > sram
-    .bss     . : { *(.bss*)    } > sram
-
+    /*================================================
+     * The Small data area. Must by 8 byte aligned
+     * .sdata and .sbss should be next to each other
+     * This area is referenced by an offset to gpr13.
+     * The compiler will place any static/global data
+     * type 8 bytes or less in length in these sections
+     * .sdata is for initialized data.
+     * .sbss is for uninitialized or zero initialized data
+     * Any data object can be forced into this secion
+     * using __attribute__((section(".sdata")))
+     * @note
+     * This would be good place to put iota kernel data
+     * such that multiple ppe engines could share the
+     * same kernel, but have their own context (not complete)
+     ================================================*/
     . = ALIGN(8);
+    _SDA_BASE_ = . ;
+    .sdata   . : {  *(.sdata.ppe42_64bit_timebase)
+                    *(.sdata.g_current_prty_level)
+                    *(.sdata.g_eimr_stack_ctr)
+                    *(.sdata.g_iota_curr_machine_state_ptr)
+                    *(.sdata.g_iota_fit_handler)
+                    *(.sdata.g_iota_dec_handler)
+                    *(.sdata.g_iota_idle_task_list)
+                    *(.sdata.g_iota_task_list)
+                    *(.sdata*)
+                 } > sram
+
+    .sbss    . : {  *(.sbss.sbss.g_ext_irq_vector)
+                    *(.sbss.g_eimr_override)
+                    *(.sbss*)  } > sram
+
+    /*=====================================================
+     * Large data area.
+     * data types > 8 bytes go here by default.
+     * The linker will throw out sections at the end of
+     * an image that only contain zeros, so use KEEP()
+     =====================================================*/
+    .rwdata . : { KEEP(*(.data*))
+                  KEEP(*(.bss*))
+                  FILL(0xA55A);  /* mark end of image     */
+                  . = ALIGN(16); /* keeps the linker from */
+                                 /* truncating the image  */
+              } > sram
+
+    _IMAGE_END = .;
+    _IMAGE_SIZE = . - SRAM_START;
+
+    .rela     : { *(.rela*) }
+    .eh_frame : { *(.eh_frame) }
+    .iplt     : { *(.iplt) }
+
+    /DISCARD/ : {
+        *(.comment)
+        *(.gnu.attributes)
+        *(.dtors)
+        *(.interp)
+    }
+
 }
diff --git a/import/chips/p9/procedures/ppe/iota/main.c b/import/chips/p9/procedures/ppe/iota/main.c
index ea978e26..ee76e6b8 100644
--- a/import/chips/p9/procedures/ppe/iota/main.c
+++ b/import/chips/p9/procedures/ppe/iota/main.c
@@ -23,6 +23,7 @@
 /*                                                                        */
 /* IBM_PROLOG_END_TAG                                                     */
 #include "iota.h"
+#include "iota_trace.h"
 
 /**
  * @file main.c
@@ -88,6 +89,10 @@ __attribute__((weak)) int main()
     IOTA_DEC_HANDLER(dec_handler);
     IOTA_FIT_HANDLER(func0);
 
+    PK_TRACE("Test Trace");
+    PK_TRACE("Tiny Trace %d", (uint16_t)1234);
+    PK_TRACE("Big trace %d", 0x12345678);
+
     out64(CME_LCL_EITR_OR, 0x8000000000000000ull);
     out64(CME_LCL_EIMR   , 0x7fffffffffffffffull);
 
diff --git a/import/chips/p9/procedures/ppe/iota/pk.h b/import/chips/p9/procedures/ppe/iota/pk.h
index 7bebf8be..296813f1 100644
--- a/import/chips/p9/procedures/ppe/iota/pk.h
+++ b/import/chips/p9/procedures/ppe/iota/pk.h
@@ -26,16 +26,93 @@
 #define __PK_H__
 
 /** @file pk.h
- * @brief Pk defs so that IOTA can use pk parts
+ * @brief Pk replacements so that IOTA can use pk parts
  */
 
 #include "ppe42_asm.h"
 #include "iota_ppe42.h"
+#include "iota_uih.h"
+#include "iota_app_cfg.h"
+
+#if !defined(NULL)
+    #define NULL ((void*)0)
+#endif
 
 #ifndef __ASSEMBLER__
-    #include <stdint.h>
-    typedef uint64_t PkTimebase;
-    extern uint64_t pk_timebase_get();
+#include <stdint.h>
+#include "iota_trace.h"
+
+typedef uint32_t size_t;
+
+typedef uint64_t PkTimebase;
+extern uint64_t pk_timebase_get();
+
+#define pk_halt() iota_halt()
+#define pk_irq_vec_restore(...) iota_uih_irq_vec_restore()
+
+// This sets the SPRG0 in pk.
+static inline void ppe42_app_ctx_set(uint16_t app_ctx)
+{
+    PkMachineContext    mctx;
+    __KernelContext   __ctx;
+    mctx = mfmsr();
+    wrteei(0);
+    __ctx.value = mfspr(SPRN_SPRG0);
+    __ctx.fields.app_specific = app_ctx;
+    mtspr(SPRN_SPRG0, __ctx.value);
+    mtmsr(mctx);
+}
+
+
+typedef uint32_t PkIrqId; //location of left-most bit on in irq reg (64 bits)
+
+#if !defined(PK_PANIC)
+
+#if SIMICS_ENVIRONMENT
+#define PK_PANIC(code)                  \
+    do {                                \
+        asm volatile ("stw     %r3, __pk_panic_save_r3@sda21(0)");      \
+        asm volatile ("lwz     %r3, __pk_panic_dbcr@sda21(0)");         \
+        asm volatile ("mtdbcr  %r3");                                   \
+        asm volatile (".long %0" : : "i" (code));                       \
+    } while(0)
+#else
+#define PK_PANIC(code)                                                  \
+    do {                                                                \
+        asm volatile ("tw 31, %0, %1" : : "i" (code/256) , "i" (code%256)); \
+    } while (0)
+#endif
+#endif // SIMICS_ENVIRONMENT
+
+// These variables are used by the PK_PANIC() definition above to save and
+// restore state. __pk_panic_dbcr is the value loaded into DBCR to force
+// traps to halt the PPE and freeze the timers.
+
+#if SIMICS_ENVIRONMENT
+#ifdef __PPE42_CORE_C__
+uint32_t __pk_panic_save_r3;
+uint32_t __pk_panic_dbcr = DBCR_RST_HALT;
+#define __PK_PANIC_DEFS__
+#else
+#define __PK_PANIC_DEFS__                                               \
+    extern uint32_t __pk_panic_save_r3;                             \
+    extern uint32_t __pk_panic_dbcr;
+#endif //SIMICS_ENVIRONMENT
+
+#endif // PK_PANIC
+
+#define MIN(X, Y)                               \
+    ({                                          \
+        typeof (X) __x = (X);                   \
+        typeof (Y) __y = (Y);                   \
+        (__x < __y) ? __x : __y; })
+
+#define MAX(X, Y)                               \
+    ({                                          \
+        typeof (X) __x = (X);                   \
+        typeof (Y) __y = (Y);                   \
+        (__x > __y) ? __x : __y;                \
+    })
 
 #endif //__ASSEMBLER__
 #endif
diff --git a/import/chips/p9/procedures/ppe/iota/ppe42.h b/import/chips/p9/procedures/ppe/iota/ppe42.h
new file mode 100644
index 00000000..81a02cc7
--- /dev/null
+++ b/import/chips/p9/procedures/ppe/iota/ppe42.h
@@ -0,0 +1,33 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: import/chips/p9/procedures/ppe/iota/ppe42.h $                 */
+/*                                                                        */
+/* OpenPOWER HCODE Project                                                */
+/*                                                                        */
+/* COPYRIGHT 2017                                                         */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+#ifndef __PPE42_H__
+#define __PPE42_H__
+
+// pk/ppe42/ppe42.h is polluted with pk dependencies.
+// This file is to override that in IOTA
+// GOAL Make ppe42 kernel independent
+#include "iota_ppe42.h"
+
+#endif
diff --git a/import/chips/p9/procedures/ppe_closed/cme/cme_common.mk b/import/chips/p9/procedures/ppe_closed/cme/cme_common.mk
index b3ac216d..5eb1624b 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/cme_common.mk
+++ b/import/chips/p9/procedures/ppe_closed/cme/cme_common.mk
@@ -31,55 +31,104 @@ IMAGE_EDITOR=cmeImgEdit.exe
 #  set _TARGET = PPE to use the new native compiler
 $(IMAGE)_TARGET=PPE
 
-## PPE_TYPE can be std or gpe
+## PPE_TYPE is std for CME
 _PPE_TYPE=std
 
-$(IMAGE)_LINK_SCRIPT=link.cmd
-
-$(IMAGE)_LDFLAGS=-e __system_reset -N -gc-sections -Bstatic
-
-include $(PK_SRCDIR)/trace/pktracefiles.mk
-CME_OBJS := $(PKTRACE_OBJECTS)
-$(call ADD_PPEIMAGE_SRCDIR,$(IMAGE),$(PK_SRCDIR)/trace)
-PKTRACE_OBJECTS:=
+ifeq "__IOTA__" "$($(IMAGE)_KERNEL)"
+######################################################
+# IOTA and carefully picked PK objects
+######################################################
+IOTA_SRCDIR=$(ROOTPATH)/chips/p9/procedures/ppe/iota
+
+$(IMAGE)_LINK_SCRIPT=../../ppe/iota/link.ld
+
+CME_OBJS:= iota_ppe42.o
+CME_OBJS += iota_ppe42_vectors.o
+CME_OBJS += iota_uih.o
+CME_OBJS += iota.o
+CME_OBJS += iota_debug_ptrs.o
+CME_OBJS += eabi.o
+CME_OBJS += math.o
+CME_OBJS += div32.o
+CME_OBJS += ppe42_gcc.o
+CME_OBJS += ppe42_string.o
+CME_OBJS += p9_cme_iota_main.o
+CME_OBJS += std_init.o
+
+# Add source code directories for the above objects
+$(call ADD_PPEIMAGE_SRCDIR,$(IMAGE),$(IOTA_SRCDIR))
+$(call ADD_PPEIMAGE_SRCDIR,$(IMAGE),$(PK_SRCDIR)/ppe42)
+$(call ADD_PPEIMAGE_SRCDIR,$(IMAGE),$(PK_SRCDIR)/$(_PPE_TYPE))
 
+else
+#  __PK__
+$(IMAGE)_LINK_SCRIPT=link.cmd
+################################
+# PK KERNEL
+################################
 include $(PK_SRCDIR)/kernel/pkkernelfiles.mk
-CME_OBJS += $(PK_OBJECTS)
+CME_OBJS := $(PK_OBJECTS)
 CME_OBJS += $(PK_TIMER_OBJECTS)
 CME_OBJS += $(PK_THREAD_OBJECTS)
 $(call ADD_PPEIMAGE_SRCDIR,$(IMAGE),$(PK_SRCDIR)/kernel)
 
+###################################
+# PK PPE42
+###################################
 include $(PK_SRCDIR)/ppe42/pkppe42files.mk
 CME_OBJS += $(PPE42_OBJECTS)
 CME_OBJS += $(PPE42_THREAD_OBJECTS)
 $(call ADD_PPEIMAGE_SRCDIR,$(IMAGE),$(PK_SRCDIR)/ppe42)
 
+###############################
+# PK PPE_TYPE
+###############################
 include $(PK_SRCDIR)/$(_PPE_TYPE)/pk$(_PPE_TYPE)files.mk
 CME_OBJS += $(STD_OBJECTS)
 $(call ADD_PPEIMAGE_SRCDIR,$(IMAGE),$(PK_SRCDIR)/$(_PPE_TYPE))
 
-include $(PM_LIBDIR)/common/libcommonfiles.mk
-CME_OBJS += $(LIBCOMMON_OBJECTS)
-$(call ADD_PPEIMAGE_SRCDIR,$(IMAGE),$(PM_LIBDIR)/common)
+endif
 
+$(IMAGE)_LDFLAGS=-e __system_reset -N -gc-sections -Bstatic
+
+##############################
+# PK TRACE
+##############################
+include $(PK_SRCDIR)/trace/pktracefiles.mk
+CME_OBJS += $(PKTRACE_OBJECTS)
+$(call ADD_PPEIMAGE_SRCDIR,$(IMAGE),$(PK_SRCDIR)/trace)
+PKTRACE_OBJECTS:=
+
+#include $(PM_LIBDIR)/common/libcommonfiles.mk
+#CME_OBJS += $(LIBCOMMON_OBJECTS)
+#$(call ADD_PPEIMAGE_SRCDIR,$(IMAGE),$(PM_LIBDIR)/common)
+
+##################################
+# HCODE LIB
+##################################
 include $(HCODE_LIBDIR)/hcodelibfiles.mk
 CME_OBJS += $(HCODE_OBJECTS)
 $(call ADD_PPEIMAGE_SRCDIR,$(IMAGE),$(HCODE_LIBDIR))
 
-include $(P2P_SRCDIR)/p2pfiles.mk
-CME_OBJS += $(P2P_OBJECTS)
-$(call ADD_PPEIMAGE_SRCDIR,$(IMAGE),$(P2P_SRCDIR))
-
+####################################
+# CME files
+####################################
 # It's important that the final included *.mk is in the $(CME_SCRDIR)
 include $(CME_SRCDIR)/topfiles.mk
+
+ifeq "__PK__" "$($(IMAGE)_KERNEL)"
 CME_OBJS+=$(TOP_OBJECTS)
+endif
+
 CME_OBJS+=$(PSTATE_OBJECTS)
 CME_OBJS+=$(UTILS_OBJECTS)
 CME_OBJS+=$(STOP_OBJECTS)
 
+
 # add include paths
+ifeq "__PK__" "$($(IMAGE)_KERNEL)"
 $(call ADD_PPEIMAGE_INCDIR,$(IMAGE),\
-    $(CME_SRCDIR)/stop_cme \
+	$(CME_SRCDIR)/stop_cme \
 	$(CME_SRCDIR)/pstate_cme \
 	$(CME_SRCDIR)/utils \
 	$(PK_SRCDIR)/kernel \
@@ -94,7 +143,25 @@ $(call ADD_PPEIMAGE_INCDIR,$(IMAGE),\
         $(ROOTPATH)/chips/p9/procedures/hwp/lib/ \
         $(ROOTPATH)/chips/p9/utils/imageProcs/ \
 	)
-
+else
+# __IOTA__
+$(call ADD_PPEIMAGE_INCDIR,$(IMAGE),\
+	$(CME_SRCDIR)/stop_cme \
+	$(CME_SRCDIR)/pstate_cme \
+	$(CME_SRCDIR)/utils \
+	$(IOTA_SRCDIR) \
+	$(PK_SRCDIR)/ppe42 \
+	$(PK_SRCDIR)$(_PPE_TYPE) \
+	$(PM_LIBDIR)/include/registers \
+	$(PK_SRCDIR)/trace \
+	$(PM_LIBDIR)/include \
+	$(PM_LIBDIR)/include/registers \
+	$(PM_LIBDIR)/common \
+	$(HCODE_LIBDIR) \
+	$(HCODE_COMMON_LIBDIR) \
+	$(ROOTPATH)/chips/p9/utils/imageProcs/ \
+	)
+endif
 
 $(IMAGE)_TRACE_HASH_PREFIX := $(shell echo $(IMAGE) | md5sum | cut -c1-4 \
 	| xargs -i printf "%d" 0x{})
@@ -103,33 +170,14 @@ $(IMAGE)_TRACE_HASH_PREFIX := $(shell echo $(IMAGE) | md5sum | cut -c1-4 \
 # Don't build up COMMONFLAGS based on other local variables.
 # The local variables won't be be available/assigned at the time
 # they are resolved to build the COMMONFLAGS
+CME_COMMONFLAGS:=
 
+ifeq "__PK__" "$($(IMAGE)_KERNEL)"
 # Options for PK_TRACE
-
+#CME_COMMONFLAGS+= -DPK_TRACE_LEVEL=0
 CME_COMMONFLAGS+= -DPK_TRACE_TIMER_OUTPUT=0
 CME_COMMONFLAGS+= -DDEV_DEBUG=0
 
-# Options for Functions being skipped
-
-CME_COMMONFLAGS+= -DSPWU_AUTO=0
-CME_COMMONFLAGS+= -DSTOP_PRIME=0
-
-CME_COMMONFLAGS+= -DSKIP_ABORT=0
-CME_COMMONFLAGS+= -DSKIP_L2_PURGE_ABORT=0
-
-CME_COMMONFLAGS+= -DSKIP_ENTRY_CATCHUP=0
-CME_COMMONFLAGS+= -DSKIP_EXIT_CATCHUP=0
-
-CME_COMMONFLAGS+= -DSKIP_ARRAYINIT=0
-CME_COMMONFLAGS+= -DSKIP_SCAN0=0
-CME_COMMONFLAGS+= -DSKIP_INITF=0
-
-CME_COMMONFLAGS+= -DSKIP_SELF_RESTORE=0
-CME_COMMONFLAGS+= -DSKIP_RAM_HRMOR=0
-
-CME_COMMONFLAGS+= -DSKIP_BCE_SCAN_RING=0
-CME_COMMONFLAGS+= -DSKIP_BCE_SCOM_RESTORE=0
-
 # Options for Kernel configuration
 
 # Force CME tasks to use the unified interrupt handler
@@ -149,8 +197,38 @@ CME_COMMONFLAGS+= -DPK_THREAD_SUPPORT=1
 CME_COMMONFLAGS+= -DPK_TRACE_SUPPORT=1
 CME_COMMONFLAGS+= -DUSE_PK_APP_CFG_H=1
 CME_COMMONFLAGS+= -D__PK__=1
+else
+#__IOTA__
+CME_COMMONFLAGS+= -DPK_TRACE_SUPPORT=1
+CME_COMMONFLAGS+= -DUSE_PK_APP_CFG_H=1
+CME_COMMONFLAGS+= -DPK_TIMER_SUPPORT=0
+CME_COMMONFLAGS+= -D__IOTA__
+#__IOTA__
+endif
+
+## Common to both __PK__ and __IOTA__
 CME_COMMONFLAGS+= -D__PPE_PLAT
 CME_COMMONFLAGS+= -D__PPE_CME
+# Options for Functions being skipped
+
+CME_COMMONFLAGS+= -DSPWU_AUTO=0
+CME_COMMONFLAGS+= -DSTOP_PRIME=0
+
+CME_COMMONFLAGS+= -DSKIP_ABORT=0
+CME_COMMONFLAGS+= -DSKIP_L2_PURGE_ABORT=0
+
+CME_COMMONFLAGS+= -DSKIP_ENTRY_CATCHUP=0
+CME_COMMONFLAGS+= -DSKIP_EXIT_CATCHUP=0
+
+CME_COMMONFLAGS+= -DSKIP_ARRAYINIT=0
+CME_COMMONFLAGS+= -DSKIP_SCAN0=0
+CME_COMMONFLAGS+= -DSKIP_INITF=0
+
+CME_COMMONFLAGS+= -DSKIP_SELF_RESTORE=0
+CME_COMMONFLAGS+= -DSKIP_RAM_HRMOR=0
+
+CME_COMMONFLAGS+= -DSKIP_BCE_SCAN_RING=0
+CME_COMMONFLAGS+= -DSKIP_BCE_SCOM_RESTORE=0
 
 ## end ifdef IMAGE
 endif
diff --git a/import/chips/p9/procedures/ppe_closed/cme/cme_p9c10.mk b/import/chips/p9/procedures/ppe_closed/cme/cme_p9c10.mk
index c267ad68..93ef1892 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/cme_p9c10.mk
+++ b/import/chips/p9/procedures/ppe_closed/cme/cme_p9c10.mk
@@ -26,7 +26,12 @@
 CME_TARGET := cme_p9c10
 IMAGE := $(CME_TARGET)
 
+#Select KERNEL
+#$(IMAGE)_KERNEL:=__IOTA__
+$(IMAGE)_KERNEL:=__PK__
+
 # Options for Platforms specific
+$(IMAGE)_COMMONFLAGS+= -DPK_TRACE_LEVEL=0
 $(IMAGE)_COMMONFLAGS = -DNIMBUS_DD_LEVEL=0
 $(IMAGE)_COMMONFLAGS+= -DCUMULUS_DD_LEVEL=10
 
diff --git a/import/chips/p9/procedures/ppe_closed/cme/cme_p9n10.mk b/import/chips/p9/procedures/ppe_closed/cme/cme_p9n10.mk
index 573db339..2e5bbb84 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/cme_p9n10.mk
+++ b/import/chips/p9/procedures/ppe_closed/cme/cme_p9n10.mk
@@ -26,6 +26,10 @@
 CME_TARGET := cme_p9n10
 IMAGE:=$(CME_TARGET)
 
+#Select KERNEL
+#$(IMAGE)_KERNEL:=__IOTA__
+$(IMAGE)_KERNEL:=__PK__
+
 # Options for Platforms specific DD lvl
 
 $(IMAGE)_COMMONFLAGS+= -DPK_TRACE_LEVEL=0
@@ -59,7 +63,7 @@ $(call BUILD_PPEIMAGE)
  OBJS := stop_cme/p9_cme_cpmr.o
 
 $(call ADD_BINHEADER_INCDIR,$(IMAGE),\
-    $(CME_SRCDIR)/stop_cme \
+	$(CME_SRCDIR)/stop_cme \
 	$(CME_SRCDIR)/pstate_cme \
 	$(PK_SRCDIR)/kernel \
 	$(PK_SRCDIR)/ppe42 \
@@ -71,7 +75,9 @@ $(call ADD_BINHEADER_INCDIR,$(IMAGE),\
 	$(HCODE_LIBDIR) \
 	$(HCODE_COMMON_LIBDIR) \
 	$(ROOTPATH)/chips/p9/procedures/hwp/lib/ \
-        $(ROOTPATH)/chips/p9/utils/imageProcs/ \
+	$(ROOTPATH)/chips/p9/utils/imageProcs/ \
 	)
 
- $(call BUILD_BINHEADER,$(IMAGEPATH)/$(CME_TARGET)/$(CME_TARGET).bin,$(ROOTPATH)/chips/p9/procedures/utils/stopreg/selfRest.bin)
+$(call BUILD_BINHEADER, \
+	$(IMAGEPATH)/$(CME_TARGET)/$(CME_TARGET).bin, \
+	$(ROOTPATH)/chips/p9/procedures/utils/stopreg/selfRest.bin)
diff --git a/import/chips/p9/procedures/ppe_closed/cme/cme_p9n20.mk b/import/chips/p9/procedures/ppe_closed/cme/cme_p9n20.mk
index f080befd..31a8a459 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/cme_p9n20.mk
+++ b/import/chips/p9/procedures/ppe_closed/cme/cme_p9n20.mk
@@ -25,7 +25,10 @@
 
 CME_IMAGE:=cme_p9n20
 IMAGE:=$(CME_IMAGE)
-IMAGE_EDITOR:=cmeImgEdit.exe
+
+#Select KERNEL
+#$(IMAGE)_KERNEL:=__IOTA__
+$(IMAGE)_KERNEL:=__PK__
 
 # Options for Platforms specific tuning
 
@@ -74,4 +77,6 @@ $(call ADD_BINHEADER_INCDIR,$(IMAGE),\
         $(ROOTPATH)/chips/p9/utils/imageProcs/ \
 	)
 
- $(call BUILD_BINHEADER,$(IMAGEPATH)/$(CME_IMAGE)/$(CME_IMAGE).bin,$(ROOTPATH)/chips/p9/procedures/utils/stopreg/selfRest.bin)
+ $(call BUILD_BINHEADER, \
+	$(IMAGEPATH)/$(CME_IMAGE)/$(CME_IMAGE).bin, \
+	$(ROOTPATH)/chips/p9/procedures/utils/stopreg/selfRest.bin)
diff --git a/import/chips/p9/procedures/ppe_closed/cme/iota_app_cfg.h b/import/chips/p9/procedures/ppe_closed/cme/iota_app_cfg.h
index a5ffadc6..ddb26a07 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/iota_app_cfg.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/iota_app_cfg.h
@@ -24,15 +24,26 @@
 /* IBM_PROLOG_END_TAG                                                     */
 #if !defined(__IOTA_APP_CFG__)
 #define __IOTA_APP_CFG__
-
 /**
  * @file iota_app_cfg.h
  * @brief IOTA CONFIGURATION SETTINGS
  */
 
-//Maximum number of external interrupt priority levels
-// and thus also the maximum number of prioritized tasks allowed.
-#define IOTA_NUM_EXT_IRQ_PRIORITIES  11
+// parts of pk that are included require this
+// Having two  app_cfg files seems confusing
+#include "pk_app_cfg.h"
+
+#define PLATFORM_PANIC_CODES_H "cme_panic_codes.h"
+#include "pk_panic_codes.h"
+
+#if !defined(__ASSEMBLER__)
+    #include "p9_cme_irq.h"
+
+    //Maximum number of external interrupt priority levels
+    // and thus also the maximum number of prioritized tasks allowed.
+    #define IOTA_NUM_EXT_IRQ_PRIORITIES  NUM_EXT_IRQ_PRTY_LEVELS
+
+#endif
 
 // Maximum number of expected nested interrupts
 #define IOTA_MAX_NESTED_INTERRUPTS    4
@@ -45,4 +56,5 @@
 // Main "execution" stack size in bytes, must be multiple of 8
 #define IOTA_EXECUTION_STACK_SIZE  2048
 
+
 #endif
diff --git a/import/chips/p9/procedures/ppe_closed/cme/iota_lnk_cfg.h b/import/chips/p9/procedures/ppe_closed/cme/iota_lnk_cfg.h
new file mode 100644
index 00000000..65db8eca
--- /dev/null
+++ b/import/chips/p9/procedures/ppe_closed/cme/iota_lnk_cfg.h
@@ -0,0 +1,41 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: import/chips/p9/procedures/ppe_closed/cme/iota_lnk_cfg.h $    */
+/*                                                                        */
+/* OpenPOWER HCODE Project                                                */
+/*                                                                        */
+/* COPYRIGHT 2017                                                         */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+#if !defined(__IOTA_LNK_CFG__)
+#define __IOTA_LNK_CFG__
+/**
+ * @file iota_lnk_cfg.h
+ * @brief Control IOTA linkerscript configuration
+ */
+#include "p9_hcd_memmap_cme_sram.H"
+
+#define SRAM_START              CME_SRAM_BASE_ADDR
+#define SRAM_LENGTH             CME_SRAM_SIZE
+#define PPE_HEADER_IMAGE_OFFSET CME_HEADER_IMAGE_OFFSET
+#define PPE_HEADER_IMAGE_NAME   .cme_image_header
+#define PPE_DEBUG_PTRS_OFFSET   CME_DEBUG_PTRS_OFFSET
+#define PPE_DEBUG_PTRS_SIZE     CME_DEBUG_PTRS_SIZE
+
+
+#endif
diff --git a/import/chips/p9/procedures/ppe_closed/cme/link.cmd b/import/chips/p9/procedures/ppe_closed/cme/link.cmd
index 599dd4de..d9ad8322 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/link.cmd
+++ b/import/chips/p9/procedures/ppe_closed/cme/link.cmd
@@ -120,7 +120,10 @@ SECTIONS
     // offsets.
 
     _SDA_BASE_ = .;
-    .sdata  . : { *(.sdata*)  } > sram
+    .sdata  . : { *(.sdata.ppe42_64bit_timebase)
+                  *(.sdata*)
+                } > sram
+
     .sbss   . : { *(.sbss*)   } > sram
 
     // Other read-write data
diff --git a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_iota_main.c b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_iota_main.c
new file mode 100644
index 00000000..80e131af
--- /dev/null
+++ b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_iota_main.c
@@ -0,0 +1,105 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: import/chips/p9/procedures/ppe_closed/cme/p9_cme_iota_main.c $ */
+/*                                                                        */
+/* OpenPOWER HCODE Project                                                */
+/*                                                                        */
+/* COPYRIGHT 2017                                                         */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+#include "iota.h"
+#include "iota_trace.h"
+
+#include "cmehw_common.h"
+#include "cme_register_addresses.h"
+
+// CME Pstate Header and Structure
+#include "p9_cme.h"
+CmeRecord G_cme_record ;
+
+// CME Pstate Header and Structure
+#include "p9_cme_pstate.h"
+CmePstateRecord G_cme_pstate_record;
+
+// CME Stop Header and Structure
+#include "p9_cme_stop.h"
+CmeStopRecord G_cme_stop_record __attribute__((section (".dump_ptrs"))) = {{0}, {0}, 0};
+
+
+void fit_handler()
+{
+    // Default is to do nothing
+}
+
+void dec_handler()
+{
+    // Currently not available
+}
+
+void ext_handler(uint32_t task_idx)
+{
+}
+
+// List of low priority tasks that run when the cme engine would
+// otheriwse be idle.
+IOTA_BEGIN_IDLE_TASK_TABLE
+{ IOTA_IDLE_DISABLED, IOTA_NO_TASK },
+IOTA_END_IDLE_TASK_TABLE
+
+IOTA_BEGIN_TASK_TABLE
+IOTA_TASK(ext_handler), // bits 0-6   default
+          IOTA_TASK(ext_handler), // bits 10,11 default
+          IOTA_TASK(p9_cme_stop_db2_handler), // bits 18,19 p9_cme_stop_db2_handler
+          IOTA_TASK(p9_cme_stop_spwu_handler), // bits 14,15 p9_cme_stop_spwu_handler
+          IOTA_TASK(p9_cme_stop_rgwu_handler), // bits 16,17 p9_cme_stop_rgwu_handler
+          IOTA_TASK(p9_cme_stop_pcwu_handler), // bits 12,13 p9_cme_stop_pcwu_handler
+          IOTA_TASK(p9_cme_stop_enter_handler), // bits 20,21 p9_cme_stop_enter_handler
+          IOTA_TASK(p9_cme_stop_db1_handler), // bits 40,41 p9_cme_stop_db1_handler
+          IOTA_TASK(p9_cme_pstate_db_handler), // bits 36,37 p9_cme_pstate_db_handler
+          IOTA_TASK(p9_cme_pstate_intercme_in0_handler), // bit  7     p9_cme_pstate_intercme_in0_handler
+          IOTA_TASK(p9_cme_pstate_pmcr_handler), // bits 34,35 p9_cme_pstate_pmcr_handler
+          IOTA_TASK(p9_cme_pstate_intercme_msg_handler), // bit  29    p9_cme_pstate_intercme_msg_handler
+          IOTA_NO_TASK  // Should never see these
+          IOTA_END_TASK_TABLE;
+
+int main()
+{
+    IOTA_DEC_HANDLER(dec_handler);
+    IOTA_FIT_HANDLER(fit_handler);
+
+    PK_TRACE("E>CME MAIN");
+
+#if defined(USE_CME_QUEUED_SCOM) || defined(USE_CME_QUEUED_SCAN)
+    out32(CME_LCL_LMCR_OR, BITS32(8, 2));
+#endif
+    //Read which cores are good
+    G_cme_record.core_enabled = in32(CME_LCL_FLAGS) &
+                                (BIT32(CME_FLAGS_CORE0_GOOD) | BIT32(CME_FLAGS_CORE1_GOOD));
+
+
+    // In IOTA, these have become initialization routines, not threads
+    p9_cme_stop_exit_thread(NULL);
+    p9_cme_stop_enter_thread(NULL);
+    p9_cme_pstate_db_thread(NULL);
+    p9_cme_pstate_pmcr_thread(NULL);
+
+
+    iota_run();
+
+    return 0;
+}
diff --git a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.c b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.c
index 1c7c4d11..e060832f 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.c
@@ -184,7 +184,7 @@ const uint64_t ext_irq_vectors_cme[NUM_EXT_IRQ_PRTY_LEVELS][2] =
 
 };
 
-uint8_t       g_current_prty_level = NUM_EXT_IRQ_PRTY_LEVELS - 1;
+uint32_t      g_current_prty_level = NUM_EXT_IRQ_PRTY_LEVELS - 1;
 uint8_t       g_eimr_stack[NUM_EXT_IRQ_PRTY_LEVELS];
 int           g_eimr_stack_ctr = -1;
 uint64_t      g_eimr_override_stack[NUM_EXT_IRQ_PRTY_LEVELS];
diff --git a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.h b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.h
index 2eef1960..45e88596 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/p9_cme_irq.h
@@ -46,6 +46,8 @@
 // - The variable names and actions in this file must perfectly match associated
 //   definitions in cme_irq_common.c
 
+#include <stdint.h>
+
 // Priority Levels
 #define IDX_PRTY_LVL_HIPRTY         0
 #define IDX_PRTY_LVL_DB3            1
@@ -107,13 +109,20 @@ extern const uint64_t ext_irq_vectors_cme[NUM_EXT_IRQ_PRTY_LEVELS][2];
                               IRQ_VEC_PRTY11_CME | \
                               IRQ_VEC_PRTY12_CME )
 
-extern uint8_t       g_current_prty_level;
-extern uint8_t       g_eimr_stack[NUM_EXT_IRQ_PRTY_LEVELS];
+extern uint32_t      g_current_prty_level;
+
+extern uint8_t
+g_eimr_stack[NUM_EXT_IRQ_PRTY_LEVELS] __attribute__((section(".sbss")));
+
 extern int           g_eimr_stack_ctr;
-extern uint64_t      g_eimr_override_stack[NUM_EXT_IRQ_PRTY_LEVELS];
+
+extern uint64_t
+g_eimr_override_stack[NUM_EXT_IRQ_PRTY_LEVELS] __attribute__((section(".sbss")));
+
 extern uint64_t      g_eimr_override;
 
 /// Restore a vector of interrupts by overwriting EIMR.
+#if !defined(__IOTA__)
 UNLESS__PPE42_IRQ_CORE_C__(extern)
 inline void
 pk_irq_vec_restore(PkMachineContext* context)
@@ -141,3 +150,4 @@ pk_irq_vec_restore(PkMachineContext* context)
 
     //pk_critical_section_exit(context);
 }
+#endif
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
index 66734f86..e326115d 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_intercme.c
@@ -51,7 +51,7 @@ void p9_cme_pstate_intercme_in0_handler(void* arg, PkIrqId irq)
     cppm_cmedb0_t dbData;
     dbData.value = 0;
     uint32_t cme_flags = in32(CME_LCL_FLAGS);
-    PkMachineContext  ctx;
+    PkMachineContext  ctx __attribute__((unused));
 
     PK_TRACE("INTER0: Enter\n");
 
@@ -119,7 +119,7 @@ void p9_cme_pstate_intercme_in0_handler(void* arg, PkIrqId irq)
 
 void p9_cme_pstate_intercme_msg_handler(void* arg, PkIrqId irq)
 {
-    PkMachineContext ctx;
+    PkMachineContext ctx __attribute__((unused));
     // Override mask, disable every interrupt except high-priority ones via the
     // priority mask for this interrupt (p9_pk_irq.c)
     uint32_t msg;
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c
index c172d0b0..f4922873 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.c
@@ -235,7 +235,8 @@ void p9_cme_analog_control(uint32_t core_mask, ANALOG_CONTROL enable)
             CME_PUTSCOM(CPPM_CACCR_OR, core_mask, (BITS64(13, 3)));
 
             // Update PMSRS (only on stop-exit)
-            // TODO Revisit during CME code review, should use common PMSRS
+            // TODO RTC 152965
+            //      Revisit during CME code review, should use common PMSRS
             //      function instead (for when other fields are added in the
             //      future)
             uint64_t pmsrs = ((((uint64_t)pstate << 48) & BITS64(8, 8))
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h
index b051c220..720038d7 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_pstate.h
@@ -134,7 +134,9 @@ typedef enum
 
 typedef struct
 {
+#if !defined(__IOTA__)
     PkSemaphore sem[2];
+#endif
     uint32_t qmFlag;
     uint32_t siblingCMEFlag;
     uint32_t quadPstate;
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
index beb78c31..962bdb0f 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_db.c
@@ -79,7 +79,15 @@ void p9_cme_pstate_pig_send();
 void p9_cme_pstate_db_handler(void* arg, PkIrqId irq)
 {
     PK_TRACE_INF("DB_HDL: Entered\n");
+#if defined(__IOTA__)
+    wrteei(1);
+
+    p9_cme_pstate_process_db0();
+
+    iota_uih_irq_vec_restore();
+#else
     pk_semaphore_post((PkSemaphore*)arg);
+#endif
     PK_TRACE_INF("DB_HDL: Exited\n");
 }
 
@@ -89,7 +97,7 @@ void p9_cme_pstate_db_handler(void* arg, PkIrqId irq)
 void p9_cme_pstate_db_thread(void* arg)
 {
     PK_TRACE_INF("DB_TH: Started\n");
-    PkMachineContext  ctx;
+    PkMachineContext  ctx __attribute__((unused));
     uint32_t cores = 0;
     uint64_t scom_data;
     uint32_t resclk_data;
@@ -117,7 +125,9 @@ void p9_cme_pstate_db_thread(void* arg)
     //if quadManager
     if (G_cme_pstate_record.qmFlag)
     {
+#if !defined(__IOTA__)
         pk_semaphore_create(&G_cme_pstate_record.sem[1], 0, 1);
+#endif
 
         if (G_cme_flags & BIT32(CME_FLAGS_CORE0_GOOD))
         {
@@ -299,9 +309,10 @@ void p9_cme_pstate_db_thread(void* arg)
         //Register with PGPE
         p9_cme_pstate_register();
 
-
         PK_TRACE_INF("DB_TH: Inited\n");
 
+#if !defined(__IOTA__)
+
         while(1)
         {
             //pend on sempahore
@@ -312,6 +323,8 @@ void p9_cme_pstate_db_thread(void* arg)
 
             pk_irq_vec_restore(&ctx);
         }
+
+#endif
     }
 
     PK_TRACE_INF("DB_TH: Exit\n");
diff --git a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_pmcr.c b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_pmcr.c
index 95afb01b..af42303f 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_pmcr.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/pstate_cme/p9_cme_thread_pmcr.c
@@ -49,13 +49,74 @@
 //
 cme_pstate_pmcr_data_t G_pmcr_thread_data;
 extern CmePstateRecord G_cme_pstate_record;
+uint32_t G_pmcr_cme_flags;
+
+void cme_pstate_pmcr_action()
+{
+    wrteei(1);
+
+    PkMachineContext  ctx __attribute__((unused));
+    uint64_t pmcr;
+    ppm_pig_t ppmPigData;
+    uint32_t eisr;
+    static
+    uint32_t coreMask[CORES_PER_EX] __attribute__((aligned(8))) =
+    {
+        CME_MASK_C0,
+        CME_MASK_C1
+    };
+
+    int c;
+
+    //Determine which core have pending request
+    for(c = 0; c < CORES_PER_EX; c++)
+    {
+        if (G_pmcr_cme_flags & (BIT32(CME_FLAGS_CORE0_GOOD) >> c))
+        {
+            eisr = in32_sh(CME_LCL_EISR); //EISR
+
+            if (eisr & (BIT32(2) >> c))
+            {
+                //Clear interrupt and read PMCR
+                out32_sh(CME_LCL_EISR_CLR, BIT32(2) >> c);
+                //We read the pmcr into a local variable, so that
+                //both phases use the same PMCR value. Otherwise, it's possible
+                //for pmcr to change between sending phase 1 and phase 2
+                pmcr = in64(CME_LCL_PMCRS0 + (c << 5));
+
+                //Send Phase 1
+                ppmPigData.value = 0;
+                ppmPigData.fields.req_intr_type = 0;
+                ppmPigData.value |= ((pmcr & PIG_PAYLOAD_PS_PHASE1_MASK) >> 8);
+                ppmPigData.value |= ((uint64_t)(G_pmcr_thread_data.seqNum & 0x6) << 57);
+                send_pig_packet(ppmPigData.value, coreMask[c]);
+                G_pmcr_thread_data.seqNum++;
+
+                //Send Phase 2
+                ppmPigData.value = 0;
+                ppmPigData.fields.req_intr_type = 1;
+                ppmPigData.value |= (pmcr & PIG_PAYLOAD_PS_PHASE2_MASK);
+                ppmPigData.value |= ((uint64_t)(G_pmcr_thread_data.seqNum & 0x6) << 57);
+                send_pig_packet(ppmPigData.value, coreMask[c]);
+                G_pmcr_thread_data.seqNum++;
+                PK_TRACE_INF("PMCR_TH: Fwd PMCR[%d]=0x%08x%08x\n", c, pmcr >> 32, pmcr);
+            }
+        }
+    }
+
+    pk_irq_vec_restore(&ctx);
+}
 
 //
 //PMCR Interrupt Handler
 //
 void p9_cme_pstate_pmcr_handler(void* arg, PkIrqId irq)
 {
+#if defined(__IOTA__)
+    cme_pstate_pmcr_action();
+#else
     pk_semaphore_post((PkSemaphore*)arg);
+#endif
 }
 
 //
@@ -64,20 +125,15 @@ void p9_cme_pstate_pmcr_handler(void* arg, PkIrqId irq)
 void p9_cme_pstate_pmcr_thread(void* arg)
 {
     PK_TRACE_INF("PMCR_TH: Enter\n");
-    int32_t c;
-    PkMachineContext  ctx;
-    uint64_t pmcr;
-    uint32_t cme_flags;
-    ppm_pig_t ppmPigData;
-    uint32_t eisr;
-    uint32_t coreMask[CORES_PER_EX] = {CME_MASK_C0, CME_MASK_C1};
     uint32_t msg;
 
     G_pmcr_thread_data.seqNum = 0; //Initialize seqNum to zero
 
-    cme_flags = in32(CME_LCL_FLAGS);
+    G_pmcr_cme_flags = in32(CME_LCL_FLAGS);;
 
+#if !defined(__IOTA__)
     pk_semaphore_create(&G_cme_pstate_record.sem[0], 0, 1);
+#endif
 
     // Synchronization between QM and Sibling
     // @todo RTC173279 move into CME init function
@@ -102,51 +158,16 @@ void p9_cme_pstate_pmcr_thread(void* arg)
 
     PK_TRACE_INF("PMCR_TH: Inited\n");
 
+#if !defined(__IOTA__)
+
     while(1)
     {
         //pend on sempahore
         pk_semaphore_pend(&G_cme_pstate_record.sem[0], PK_WAIT_FOREVER);
-        wrteei(1);
-
-        //Determine which core have pending request
-        for(c = 0; c < CORES_PER_EX; c++)
-        {
-            if (cme_flags & (BIT32(CME_FLAGS_CORE0_GOOD) >> c))
-            {
-                eisr = in32_sh(CME_LCL_EISR); //EISR
-
-                if (eisr & (BIT32(2) >> c))
-                {
-                    //Clear interrupt and read PMCR
-                    out32_sh(CME_LCL_EISR_CLR, BIT32(2) >> c);
-                    //We read the pmcr into a local variable, so that
-                    //both phases use the same PMCR value. Otherwise, it's possible
-                    //for pmcr to change between sending phase 1 and phase 2
-                    pmcr = in64(CME_LCL_PMCRS0 + (c << 5));
-
-                    //Send Phase 1
-                    ppmPigData.value = 0;
-                    ppmPigData.fields.req_intr_type = 0;
-                    ppmPigData.value |= ((pmcr & PIG_PAYLOAD_PS_PHASE1_MASK) >> 8);
-                    ppmPigData.value |= ((uint64_t)(G_pmcr_thread_data.seqNum & 0x6) << 57);
-                    send_pig_packet(ppmPigData.value, coreMask[c]);
-                    G_pmcr_thread_data.seqNum++;
-
-                    //Send Phase 2
-                    ppmPigData.value = 0;
-                    ppmPigData.fields.req_intr_type = 1;
-                    ppmPigData.value |= (pmcr & PIG_PAYLOAD_PS_PHASE2_MASK);
-                    ppmPigData.value |= ((uint64_t)(G_pmcr_thread_data.seqNum & 0x6) << 57);
-                    send_pig_packet(ppmPigData.value, coreMask[c]);
-                    G_pmcr_thread_data.seqNum++;
-                    PK_TRACE_INF("PMCR_TH: Fwd PMCR[%d]=0x%08x%08x\n", c, pmcr >> 32, pmcr);
-
-                }
-            }
-        }
-
-        pk_irq_vec_restore(&ctx);
+        cme_pstate_pmcr_action();
     }
 
+#endif
+
     PK_TRACE_INF("PMCR_TH: Exit\n");
 }
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
index 133339f4..4b1d4411 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop.h
@@ -146,7 +146,7 @@
 
 enum CME_IRQ_VECTORS
 {
-// if auto mask eimr.spwu else never mask eimr.spwu
+    // if auto mask eimr.spwu else never mask eimr.spwu
 #if SPWU_AUTO
     IRQ_VEC_WAKE_C0                  = BIT64(12) | BIT64(14) | BIT64(16),
     IRQ_VEC_WAKE_C1                  = BIT64(13) | BIT64(15) | BIT64(17),
@@ -236,7 +236,9 @@ typedef struct
     uint32_t      core_blockey;
     // core in special wakeup, can be used as core select in scom address or data
     uint32_t      core_in_spwu;
+#if !defined(__IOTA__)
     PkSemaphore   sem[2];
+#endif
 } CmeStopRecord;
 
 
diff --git a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
index cac72c07..24df2a1a 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
+++ b/import/chips/p9/procedures/ppe_closed/cme/stop_cme/p9_cme_stop_irq_handlers.c
@@ -37,7 +37,7 @@ p9_cme_stop_pcwu_handler(void* arg, PkIrqId irq)
 {
     MARK_TRAP(STOP_PCWU_HANDLER)
 
-    PkMachineContext ctx;
+    PkMachineContext ctx __attribute__((unused));
     uint32_t  core_mask = 0;
     uint32_t  core      = (in32(CME_LCL_EISR) & BITS32(12, 2)) >> SHIFT32(13);
     data64_t  scom_data = {0};
@@ -76,7 +76,16 @@ p9_cme_stop_pcwu_handler(void* arg, PkIrqId irq)
     if (core)
     {
         out32(CME_LCL_EIMR_OR, BITS32(12, 6) | BITS32(20, 2));
+#if defined(__IOTA__)
+        wrteei(1);
+        p9_cme_stop_exit();
+
+        // re-evaluate g_eimr_override then restore eimr
+        p9_cme_stop_eval_eimr_override();
+        iota_uih_irq_vec_restore();
+#else
         pk_semaphore_post((PkSemaphore*)arg);
+#endif
     }
     else
     {
@@ -98,7 +107,7 @@ p9_cme_stop_spwu_handler(void* arg, PkIrqId irq)
 {
     MARK_TRAP(STOP_SPWU_HANDLER)
 
-    PkMachineContext ctx;
+    PkMachineContext ctx __attribute__((unused));
     int      sem_post   = 0;
     uint32_t core_mask  = 0;
     uint32_t core_index = 0;
@@ -166,7 +175,16 @@ p9_cme_stop_spwu_handler(void* arg, PkIrqId irq)
     {
         out32(CME_LCL_EIMR_OR, BITS32(12, 6) | BITS32(20, 2));
         PK_TRACE_INF("Launching exit thread");
+#if defined(__IOTA__)
+        wrteei(1);
+        p9_cme_stop_exit();
+
+        // re-evaluate g_eimr_override then restore eimr
+        p9_cme_stop_eval_eimr_override();
+        iota_uih_irq_vec_restore();
+#else
         pk_semaphore_post((PkSemaphore*)arg);
+#endif
     }
     else
     {
@@ -182,7 +200,16 @@ p9_cme_stop_rgwu_handler(void* arg, PkIrqId irq)
     MARK_TRAP(STOP_RGWU_HANDLER)
     PK_TRACE_INF("RGWU Handler Trigger %d", irq);
     out32(CME_LCL_EIMR_OR, BITS32(12, 6) | BITS32(20, 2));
+#if defined(__IOTA__)
+    wrteei(1);
+    p9_cme_stop_exit();
+
+    // re-evaluate g_eimr_override then restore eimr
+    p9_cme_stop_eval_eimr_override();
+    iota_uih_irq_vec_restore();
+#else
     pk_semaphore_post((PkSemaphore*)arg);
+#endif
 }
 
 
@@ -193,13 +220,24 @@ p9_cme_stop_enter_handler(void* arg, PkIrqId irq)
     MARK_TRAP(STOP_ENTER_HANDLER)
     PK_TRACE_INF("PM_ACTIVE Handler Trigger %d", irq);
     out32(CME_LCL_EIMR_OR, BITS32(12, 6) | BITS32(20, 2));
+#if defined(__IOTA__)
+    wrteei(1);
+
+    // The actual entry sequence
+    p9_cme_stop_entry();
+
+    // re-evaluate g_eimr_override then restore eimr
+    p9_cme_stop_eval_eimr_override();
+    iota_uih_irq_vec_restore();
+#else
     pk_semaphore_post((PkSemaphore*)arg);
+#endif
 }
 
 void
 p9_cme_stop_db2_handler(void* arg, PkIrqId irq)
 {
-    PkMachineContext ctx;
+    PkMachineContext ctx __attribute__((unused));
 
     MARK_TRAP(STOP_DB2_HANDLER)
     PK_TRACE_DBG("DB2 Handler Trigger %d", irq);
@@ -220,7 +258,7 @@ p9_cme_stop_db2_handler(void* arg, PkIrqId irq)
 void
 p9_cme_stop_db1_handler(void* arg, PkIrqId irq)
 {
-    PkMachineContext ctx;
+    PkMachineContext ctx __attribute__((unused));
     cppm_cmedb1_t    db1         = {0};
     ppm_pig_t        pig         = {0};
     uint32_t         core        = 0;