From 8fac481d825883154ca20612ca0a69bc52f02618 Mon Sep 17 00:00:00 2001 From: Doug Gilbert Date: Tue, 23 Apr 2013 14:10:52 -0500 Subject: Inter-processor communications for multinode RTC: 63128 Change-Id: Ica27c7f714bc8b874c9bccb663a32d3cfba37c5a Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/4193 Tested-by: Jenkins Server Reviewed-by: A. Patrick Williams III --- src/kernel/intmsghandler.C | 11 +----- src/kernel/ipc.C | 97 ++++++++++++++++++++++++++++++++++++++++++++++ src/kernel/makefile | 2 +- src/kernel/syscall.C | 73 ++++++++++++++++++++-------------- src/kernel/terminate.C | 7 +++- 5 files changed, 150 insertions(+), 40 deletions(-) create mode 100644 src/kernel/ipc.C (limited to 'src/kernel') diff --git a/src/kernel/intmsghandler.C b/src/kernel/intmsghandler.C index 8bf68950c..610335731 100644 --- a/src/kernel/intmsghandler.C +++ b/src/kernel/intmsghandler.C @@ -119,15 +119,6 @@ void InterruptMsgHdlr::handleInterrupt() printk("InterrurptMsgHdlr got called before IPC was setup\n"); hit = true; } - - - // else we got an external interrupt before we got things set up. - // TODO Is there anything that can be done other than - // leave the interrupt presenter locked. - // Does the code that sets up the IP registers need to check to see if - // there is an interrupt sitting there and send an EOI? - // Story 41868 - Mask off all interrupts very early - might - // resolve this TODO. } } @@ -155,6 +146,8 @@ void InterruptMsgHdlr::sendIPI(uint64_t i_pir) mfrrAddress += mmio_offset(i_pir); mfrrAddress += MFRR_ADDR_OFFSET; + mfrrAddress |= 0x8000000000000000ul; + register uint8_t data = 0; eieio(); sync(); diff --git a/src/kernel/ipc.C b/src/kernel/ipc.C new file mode 100644 index 000000000..3f522c4ee --- /dev/null +++ b/src/kernel/ipc.C @@ -0,0 +1,97 @@ +/* IBM_PROLOG_BEGIN_TAG */ +/* This is an automatically generated prolog. */ +/* */ +/* $Source: src/kernel/ipc.C $ */ +/* */ +/* IBM CONFIDENTIAL */ +/* */ +/* COPYRIGHT International Business Machines Corp. 2013 */ +/* */ +/* p1 */ +/* */ +/* Object Code Only (OCO) source materials */ +/* Licensed Internal Code Source Materials */ +/* IBM HostBoot Licensed Internal Code */ +/* */ +/* The source code for this program is not published or otherwise */ +/* divested of its trade secrets, irrespective of what has been */ +/* deposited with the U.S. Copyright Office. */ +/* */ +/* Origin: 30 */ +/* */ +/* IBM_PROLOG_END_TAG */ +#include +#include +#include +#include +#include + +using namespace KernelIpc; + +namespace KernelIpc +{ + void send(uint64_t i_q, msg_t * i_msg); +}; + +/** + * IPC communication area. Interrupt service provider initializes. + * @see intrrp.C + */ +KernelIpc::ipc_data_area_t KernelIpc::ipc_data_area; + +// Put the IPC message in the other nodes memory space +// Two potential issues: +// 1. The destination node is not there - memory location is nonexistant. +// 2. The destination node never responds, potentially hanging this thread. +void KernelIpc::send(uint64_t i_q, msg_t * i_msg) +{ + // @note + // Point to memory in the destination image. + // All host boot images are assured to be at the same code level. + // PIR node and physical node are not always the same. For instance a + // single node system with an alt-master could be designed such that the + // alt-master were on a different logical (power bus numbering) node. + // Since it's not in plan to use this IPC mechanism in a single node + // system, this case will ge ignored for now. + uint64_t this_node = getPIR()/KERNEL_MAX_SUPPORTED_CPUS_PER_NODE; + uint64_t hrmor_offset = getHRMOR()-(this_node*(ipc_data_area.hrmor_base)); + uint64_t dest_node = (i_q >> 32) & 0x07; + uint64_t dest_hrmor = (ipc_data_area.hrmor_base*dest_node) + hrmor_offset; + + uint64_t dest_addr = reinterpret_cast(&ipc_data_area); + dest_addr += dest_hrmor; + dest_addr |= 0x8000000000000000ul; + + printkd("IPC Dest addr %lx\n",dest_addr); + + // pointer to the ipc_data_area in the destination node + ipc_data_area_t * p_dest = + reinterpret_cast(dest_addr); + + // get lock on IPC data area in other node + while(false == __sync_bool_compare_and_swap(&(p_dest->msg_queue_id), + 0, + 0xFFFFFFFFFFFFFFFFul)) + { + setThreadPriorityLow(); + } + setThreadPriorityHigh(); + + p_dest->msg_payload = *i_msg; // copy in message + lwsync(); + + p_dest->msg_queue_id = i_q; // set destination queue id + lwsync(); + + // send IPI + InterruptMsgHdlr::sendIPI(p_dest->pir); + + // The message allocation is freed here to make msg_send for IPC + // messages behave the same as non-IPC msg_send; that is, the message + // is freed by the consumer; however, i_msg was allocated in user space + // code and freed here in kernel space. The assumption is that this is OK. + msg_free(i_msg); + + return; +} + diff --git a/src/kernel/makefile b/src/kernel/makefile index a3a750dcb..3627798a6 100644 --- a/src/kernel/makefile +++ b/src/kernel/makefile @@ -27,7 +27,7 @@ OBJS += syscall.o scheduler.o spinlock.o exception.o vmmmgr.o timemgr.o OBJS += futexmgr.o ptmgr.o segmentmgr.o devicesegment.o basesegment.o OBJS += block.o cpuid.o misc.o msghandler.o blockmsghdlr.o stacksegment.o OBJS += softpatch_p8.o barrier.o idebug.o intmsghandler.o deferred.o -OBJS += shutdown.o forceattn_p8.o terminate.o +OBJS += shutdown.o forceattn_p8.o terminate.o ipc.o include ${ROOTPATH}/config.mk diff --git a/src/kernel/syscall.C b/src/kernel/syscall.C index 985d9c0ab..0a8e644a0 100644 --- a/src/kernel/syscall.C +++ b/src/kernel/syscall.C @@ -42,6 +42,10 @@ #include #include +namespace KernelIpc +{ + void send(uint64_t i_q, msg_t * i_msg); +}; extern "C" void kernel_execute_decrementer() @@ -308,44 +312,55 @@ namespace Systemcalls void MsgSend(task_t* t) { - MessageQueue* mq = (MessageQueue*) TASK_GETARG0(t); + uint64_t q_handle = TASK_GETARG0(t); msg_t* m = (msg_t*) TASK_GETARG1(t); - if ((NULL == mq) || (NULL == m)) + if(((q_handle >> 32) & MSGQ_TYPE_IPC) != 0) { - printkd("NULL pointer for message queue (%p) or message (%p).\n", - mq, m); - TASK_SETRTN(t, -EINVAL); - return; + // IPC message + KernelIpc::send(q_handle, m); } + else + { - m->__reserved__async = 0; // set to async msg. + MessageQueue* mq = reinterpret_cast(q_handle); - if (m->type >= MSG_FIRST_SYS_TYPE) - { - printkd("Invalid type for msg_send, type=%d.\n", m->type); - TASK_SETRTN(t, -EINVAL); - return; - } + if ((NULL == mq) || (NULL == m)) + { + printkd("NULL pointer for message queue (%p) or message (%p).\n", + mq, m); + TASK_SETRTN(t, -EINVAL); + return; + } - mq->lock.lock(); + m->__reserved__async = 0; // set to async msg. - // Get waiting (server) task. - task_t* waiter = mq->waiting.remove(); - if (NULL == waiter) // None found, add to 'messages' queue. - { - MessagePending* mp = new MessagePending(); - mp->key = m; - mp->task = t; - mq->messages.insert(mp); - } - else // Add waiter back to its scheduler. - { - TASK_SETRTN(waiter, (uint64_t) m); - waiter->cpu->scheduler->addTask(waiter); - } + if (m->type >= MSG_FIRST_SYS_TYPE) + { + printkd("Invalid type for msg_send, type=%d.\n", m->type); + TASK_SETRTN(t, -EINVAL); + return; + } - mq->lock.unlock(); + mq->lock.lock(); + + // Get waiting (server) task. + task_t* waiter = mq->waiting.remove(); + if (NULL == waiter) // None found, add to 'messages' queue. + { + MessagePending* mp = new MessagePending(); + mp->key = m; + mp->task = t; + mq->messages.insert(mp); + } + else // Add waiter back to its scheduler. + { + TASK_SETRTN(waiter, (uint64_t) m); + waiter->cpu->scheduler->addTask(waiter); + } + + mq->lock.unlock(); + } TASK_SETRTN(t, 0); } diff --git a/src/kernel/terminate.C b/src/kernel/terminate.C index 2c103c5af..bbb8d4aa9 100644 --- a/src/kernel/terminate.C +++ b/src/kernel/terminate.C @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -37,7 +38,11 @@ extern "C" void p8_force_attn() NO_RETURN; HB_TI_DataArea kernel_TIDataArea; /* Instance of the HB desriptor struct */ -HB_Descriptor kernel_hbDescriptor = {&kernel_TIDataArea}; +HB_Descriptor kernel_hbDescriptor = +{ + &kernel_TIDataArea, + &KernelIpc::ipc_data_area +}; -- cgit v1.2.1