diff options
Diffstat (limited to 'src/ssx/pgp/pgp_async.c')
| -rwxr-xr-x | src/ssx/pgp/pgp_async.c | 1024 |
1 files changed, 1024 insertions, 0 deletions
diff --git a/src/ssx/pgp/pgp_async.c b/src/ssx/pgp/pgp_async.c new file mode 100755 index 0000000..3bf9a63 --- /dev/null +++ b/src/ssx/pgp/pgp_async.c @@ -0,0 +1,1024 @@ +// $Id: pgp_async.c,v 1.4 2014/02/14 12:18:05 bcbrock Exp $ +// $Source: /afs/awd/projects/eclipz/KnowledgeBase/.cvsroot/eclipz/chips/p8/working/procedures/ssx/pgp/pgp_async.c,v $ +//----------------------------------------------------------------------------- +// *! (C) Copyright International Business Machines Corp. 2013 +// *! All Rights Reserved -- Property of IBM +// *! *** IBM Confidential *** +//----------------------------------------------------------------------------- + +/// \file pgp_async.c +/// \brief Support for asynchronous request queuing and callback mechanisms +/// +/// This file implements device drivers for asynchronous requests. The model +/// for devices like the PORE engines and the PBA block copy engines is that +/// the application creates self-contained requests for jobs to run on the +/// engine. The application then schedules the request and continues normal +/// processing, or threads can request to simply block in place until the +/// request finishes. Queue management is handled in the interrupt handler +/// for the engine. As each queued job finishes, the next request (if any) is +/// started on the engine and the optional callback is invoked or scheduled. +/// +/// The application can either use a polling protocol or the asynchronous +/// callback mechanism to determine the state of the request. A timeout +/// mechanism is also provided that cancels or kills a job that does not +/// complete within a fixed time. Error handling in the event that the request +/// does not complete in time or has failed is standardized by the +/// implementation, but can be overridden by the application. +/// +/// Asynchronous request interrupt handlers can run either as critical or +/// noncritical handlers. Some engines may support queuing requests that +/// complete immediately; therefore the kernel context of the callback may be +/// either a thread or interupt context, and in general, callbacks should not +/// make assumptions about the kernel context when they run. +/// +/// If the callback is non-NULL and the request was created with \c +/// ASYNC_CALLBACK_IMMEDIATE, the callback is then immediately invoked in the +/// current context. In general, immediate callbacks should be short and sweet +/// (to reduce interrupt latency) and should not make SSX kernel calls. +/// +/// If the request has a non-NULL callback and was created with \c +/// ASYNC_CALLBACK_DEFERRED then the callback will be deferred to a +/// noncritical interrupt context. Deferred callbacks are queued, then run +/// later in response to a reserved IPI. Deferred callbacks always run as +/// noncritical interrupt handlers with noncritical interrupts \e enabled, +/// similar to SSX timer callbacks. +/// +/// If the request has a non-null callback and was created with \c +/// ASYNC_CALLBACK_NONCRITICAL, then the callback will be run immediately from +/// a noncritical handler or thread environment, or deferred from a critical +/// interrupt context. Similar to immediate callbacks, callbacks marked +/// noncritical should be short and sweet (to reduce interrupt latency), but +/// may make SSX kernel calls. +/// +/// Regardless of whether the callback is critical or noncritical, the +/// callback is coded as a normal C or assembler subroutine with a single +/// void* argument. +/// +/// As a programming shortcut, the AsyncRequest includes a semaphore object. +/// A special AsyncRequest option ASYNC_REQUEST_BLOCKING indicates that the +/// thread scheduling the request should block on the semaphore of the +/// AsyncRequest (with SSX_WAIT_FOREVER) until the request is complete. Note +/// that a separate callback can be specified even if ASYNC_REQUEST_BLOCKING +/// is specified. +/// +/// Requests are always timestamped. The \a start_time field of the request is +/// set from the SSX timebase when the request is first 'run' on the device. +/// Request types that may require multiple back-to-back 'runs' (like PBA +/// block copies with more than 4K data) only record the initial 'run'. The \a +/// end_time field of the request is set from the SSX timebase when the job +/// finishes on the hardware, but before any callback is run. These +/// timestamps cover time intervals not visible to the application; The +/// application is responsible for timestamping the period between request +/// queing and the \a start_time, and between the \a end_time and callback +/// completion if required. The timestamps can be robustly recovered from the +/// request using the API async_request_timestamps_get(). +/// +/// This is largely a generic implementation, designed to reduce code space by +/// allowing the PORE, PBA and OCB drivers to use the same generic data +/// structures and code. This is supported by the 'single-inheritence class +/// hierarchy' described in the comments for pgp_async.h. +/// +/// <b> Request Completion and Callback States </b> +/// +/// The application can determine what happend to the job by observing the +/// request state and callback return code when the request becomes idle. +/// A request is not considered idle until the job has run to completion or +/// been aborted, any callback has been run, any timeout has been cancelled, +/// and any thread pending on request completion has been maxde runnable. +/// +/// Normally the application should see the request state (\a request->state) +/// ASYNC_REQUEST_STATE_COMPLETE and a callback return code +/// (\a request=->callback_rc) of 0 which indicates that the job and callback +/// both completed normally. If the request did not complete normally it +/// could be due to one of several reasons. For further analysis the request +/// also includes a field \a abort_state that records the state the job was in +/// when it was aborted (i.e., cancelled, killed, timed out or error-out). +/// +/// <b> Timeout Semantics </b> +/// +/// Any timeout other than SSX_WAIT_FOREVER specifies a timeout covering the +/// interval spanning the time a job is scheduled until the time the job +/// completes on the hardware. If the job is still wating to execute when it +/// times out then the job is simply removed from the queue and marked as +/// having timed out. If the job is running when it times out then the job is +/// forceably removed from the hardware, which may have unintended or +/// indeterminate consequences. The application may need to consider whether +/// it is safe to continue after a forced timeout. +/// +/// Specifying a timeout involves quite a bit of overhead, since a timer needs +/// to be scheduled and cancelled each time a job is run. If the interrupt +/// handler for a device is a critical handler then the timeout cancellation +/// will need to be deferred to the callback queue, potentially increasing +/// overhead even further. +/// +/// <b> Implementation Notes </b> +/// +/// - The \e queue objects will normally be global data structures that +/// persist throughout the life of the application. +/// +/// - The \e request objects may be either global or local data +/// structures. However, since \e request objects are not copied, and pointers +/// to them are stored in the \e queue objects, \a request objects should not +/// be allocated on the stack if the stack frame could become invalid before +/// the request completes. +/// +/// \todo Once all function is developed and tested, convert interrupt +/// handling to fast-mode assembler routines. + +#include "ssx.h" + + +//////////////////////////////////////////////////////////////////////////// +// Global Data +//////////////////////////////////////////////////////////////////////////// + +/// Queue of deferred callbacks + +static SsxDeque G_async_callback_queue; + + +//////////////////////////////////////////////////////////////////////////// +// FFDC +//////////////////////////////////////////////////////////////////////////// + +/// Collect FFDC for the PLB (OCI) arbiter +/// +/// \param ffdc A pointer to an OciFfdc structure to be filled in. +/// +/// \param master_id The PLB (OCI) master Id of the master of interest. +/// +/// Note: The PEAR and PESR hold error information for all OCI masters +/// _except_ the OCC ICU and DCU. ICU and DCU errors are in the STO_PESR and +/// STO_PEAR. Currently there is no need to collect those DCRs for 'async' +/// errors. + +void +oci_ffdc(OciFfdc* ffdc, int master_id) +{ + uint32_t pesr_lock_mask; + + ffdc->pearl.value = mfdcr(PLB_PEARL); + ffdc->pesr.value = mfdcr(PLB_PESR); + + pesr_lock_mask = 0x30000000 >> (4 * master_id); + if (ffdc->pesr.value & pesr_lock_mask) { + ffdc->mine = 1; + mtdcr(PLB_PESR, pesr_lock_mask); + } else { + ffdc->mine = 0; + } +} + + + +//////////////////////////////////////////////////////////////////////////// +// AsyncQueue +//////////////////////////////////////////////////////////////////////////// + +// Start an asynchronous request on the device with timestamping. This will +// always be called from a critical section, and any timestamp is collected +// immediately before the request is kicked off on the device. Devices like +// the BCE engines and the OCB queue drivers may run the same request multiple +// times to get all of the data moved. Therefore the initial timestamp is only +// captured the first time the request is run on the device. + +static inline int +async_request_run(AsyncRequest *request) +{ + if (request->state != ASYNC_REQUEST_STATE_RUNNING) { + request->start_time = ssx_timebase_get(); + } + request->state = ASYNC_REQUEST_STATE_RUNNING; + return request->run_method(request); +} + + +// Create (initialize) a generic AsyncQueue +// +// This is an internal API used to initialize generic request queues. The +// caller is assumed to have done all error checking on the parameters. +// +// This is a simple initialization that resets the queues and sets the state +// to QUEUE_STATE_IDLE. This routine should only be called on uninitialized +// AsyncQueue objects. + +int +async_queue_create(AsyncQueue *queue, AsyncEngine engine) +{ + ssx_deque_sentinel_create(&(queue->deque)); + queue->current = 0; + queue->engine = engine; + queue->state = ASYNC_QUEUE_STATE_IDLE; + + return 0; +} + + +// Generic completion of a request. This is called both by async_handler() +// and async_request_deque(). + +static void +async_request_complete(AsyncRequest *request) +{ + SsxMachineContext ctx; + AsyncRequestCallback callback; + int completed; + + // Handle callbacks and deferred processing of the job that just + // finished. No callback is easy. If the job does have a callback + // that we can execute immediately then that is done immediately. + // Note that 'completed' here means only that the callback is complete. + + callback = request->callback; + if (!callback) { + + completed = 1; + + } else if ((request->options & ASYNC_CALLBACK_IMMEDIATE) || + ((request->options & ASYNC_CALLBACK_NONCRITICAL) && + !__ssx_kernel_context_critical_interrupt())) { + + request->state = ASYNC_REQUEST_STATE_CALLBACK_RUNNING; + request->callback_rc = callback(request->arg); + completed = 1; + + } else { + + request->state = ASYNC_REQUEST_STATE_CALLBACK_QUEUED; + completed = 0; + } + + + // If the callback completed then we go ahead and cancel any timeout + // and/or wake the thread if possible. In critical interrupt contexts + // we always have to defer these operations, so we may lose 'complete' + // status. + + if (completed && + ((request->timeout != SSX_WAIT_FOREVER) || + (request->options & ASYNC_REQUEST_BLOCKING))) { + + if (__ssx_kernel_context_critical_interrupt()) { + + request->state = ASYNC_REQUEST_STATE_POSTPROCESSING; + completed = 0; + + } else { + if (request->timeout != SSX_WAIT_FOREVER) { + ssx_timer_cancel(&(request->timer)); + } + if (request->options & ASYNC_REQUEST_BLOCKING) { + ssx_semaphore_post(&(request->sem)); + } + } + } + + + // A truly completed job gets its completion state here. Otherwise we + // have to schedule the deferred postprocessing. + + if (completed) { + + request->state = request->completion_state; + + } else { + + ssx_critical_section_enter(SSX_CRITICAL, &ctx); + + if (request->options & ASYNC_CALLBACK_PRIORITY) { + ssx_deque_push_front(&G_async_callback_queue, + &(request->deque)); + } else { + ssx_deque_push_back(&G_async_callback_queue, + &(request->deque)); + } + + ssx_critical_section_exit(&ctx); + ssx_irq_status_set(PGP_IRQ_ASYNC_IPI, 1); + } +} + + +// The generic handler for asynchonous device completion. +// +// This handler processes completions of generic device requests, as well as +// the initial running of jobs when the queue is idle. Error completions are +// initially handled by async_error_handler() which then calls async_handler() +// to finish the failed job and start the next job if possible. The initial +// device handler must manage interrupts and provide a method to run the +// 'current' job in the queue. The engine-specific handler may also iterate +// the current job until it is complete - this handler should only be called +// when the current job is complete. +// +// Some engines may have jobs that can complete immediately. This handler +// iterates over jobs in the queue as long as the run method of the current +// job returns the code -ASYNC_REQUEST_COMPLETE. +// +// NB : Normally this call is made from an interrupt handler, however it may +// be called from job scheduling code if the engine is idle. Regardless, the +// caller must insure that any call for a queue is protected against +// interrupts for that queue. + +void +async_handler(AsyncQueue *queue) +{ + AsyncRequest *finished, *current; + int rc; + + // This loop is repeated as long as any job started in a loop completes + // immediately. + + do { + + // This API may be called on an idle queue, which indicates that we + // should simply start the job on the head of the queue. Otherwise + // save a pointer to the job that just finished and update its + // end_time. + + if (queue->state == ASYNC_QUEUE_STATE_IDLE) { + + finished = 0; + + } else { + + finished = (AsyncRequest *)(queue->current); + + if (SSX_ERROR_CHECK_KERNEL && (finished == 0)) { + SSX_PANIC(ASYNC_PHANTOM_INTERRUPT); + } + + finished->end_time = ssx_timebase_get(); + + } + + + // If the queue is in an error state we will not schedule any further + // jobs on this queue. Otherwise we start the next job running on the + // engine. + + if (queue->state == ASYNC_QUEUE_STATE_ERROR) { + + queue->current = 0; + rc = 0; + + } else { + + current = (AsyncRequest *)ssx_deque_pop_front(&(queue->deque)); + queue->current = current; + + if (current) { + queue->state = ASYNC_QUEUE_STATE_RUNNING; + rc = async_request_run(current); + } else { + queue->state = ASYNC_QUEUE_STATE_IDLE; + rc = 0; + } + } + + + // If no job just finished, continue with the loop. If the job we + // just enqueued finished immediately it will be 'finished' on the + // next loop (it would have given an rc == + // -ASYNC_REQUEST_COMPLETE). Otherwise complete the request. + + if (finished != 0) { + + async_request_complete(finished); + } + + } while (rc == -ASYNC_REQUEST_COMPLETE); +} + + +/// Schedule (queue for execution) a generic asynchronous request. +/// +/// \param request An initialized and idle AsyncRequest +/// +/// +/// The request is queued for execution with all of the parameters provided +/// when the request was created. It is considered an error to (re)schedule a +/// request that is currently scheduled, running, or has the callback queued or +/// in execution. Requests either need to be idle, or must be explicitly +/// cancelled or killed before thay can be (re)scheduled. Because engine queue +/// interrupt handlers may run as critical interrupts, this routine operates in +/// a short \c SSX_CRITICAL critical section. +/// +/// If the request is made to an otherwise empty queue then the async_handler() +/// must also be called immediately to begin the job. This will extend the +/// critical section slightly. This routine will not start a request on a +/// queue that has halted due to an error. The request will simply be enqueued +/// in that case. +/// +/// \retval 0 Success +/// +/// \retval -ASYNC_INVALID_OBJECT_SCHEDULE The \a request is NULL (0). +/// +/// \retval -ASYNC_REQUEST_NOT_IDLE The \a request is (still) active in some +/// way at entry. +/// +/// \retval -ASYNC_REQUEST_NOT_COMPLETE This code is returned for requests that +/// do not complete successfully, but only if ASYNC_REQUEST_BLOCKING is +/// specified. The caller will need to examine the request state if necessary +/// to determine whether the request failed, was cancelled or timed out. +/// +/// \todo Consider making the critical section priority depend on the device +/// queue priority here, by adding a priority field to the queue. Without this +/// we always have to run the async_handler() in an SSX_CRITICAL critical +/// section. + +int +async_request_schedule(AsyncRequest *request) +{ + SsxMachineContext ctx = SSX_THREAD_MACHINE_CONTEXT_DEFAULT; // For GCC + AsyncQueue *queue; + int rc; + + if (SSX_ERROR_CHECK_API) { + SSX_ERROR_IF(request == 0, ASYNC_INVALID_OBJECT_SCHEDULE); + } + + rc = 0; + ssx_critical_section_enter(SSX_CRITICAL, &ctx); + + do { + + // Check to insure the request is idle (which check must be done in + // the critical section), then start any required timeout. + + if (SSX_ERROR_CHECK_API) { + if (!async_request_is_idle(request)) { + rc = -ASYNC_REQUEST_NOT_IDLE; + break; + } + } + + if (request->timeout != SSX_WAIT_FOREVER) { + rc = ssx_timer_schedule(&(request->timer), request->timeout, 0); + if (rc) break; + } + + + // Enqueue the request and initialize the request state. + + queue = request->queue; + + if (request->options & ASYNC_REQUEST_PRIORITY) { + ssx_deque_push_front(&(queue->deque), &(request->deque)); + } else { + ssx_deque_push_back(&(queue->deque), &(request->deque)); + } + + request->state = ASYNC_REQUEST_STATE_QUEUED; + request->completion_state = ASYNC_REQUEST_STATE_COMPLETE; + request->callback_rc = 0; + + + // If the queue is idle, call the async_handler() to start the job. + // Then block the calling thread if required. + + if (queue->state == ASYNC_QUEUE_STATE_IDLE) { + async_handler(queue); + } + + if (request->options & ASYNC_REQUEST_BLOCKING) { + rc = ssx_semaphore_pend(&(request->sem), SSX_WAIT_FOREVER); + if (rc) break; + + if (!async_request_completed(request)) { + rc = -ASYNC_REQUEST_NOT_COMPLETE; + break; + } + } + + } while (0); + + ssx_critical_section_exit(&ctx); + + return rc; +} + + +// The generic error handler +// +// This is a generic handler called in response to an error interrupt or +// timeout. This handler calls a request-specific error method to stop the +// hardware device, collect FFDC and make the hardware device runnable again +// if possible. Then the current request is marked as having failed, and the +// generic async_handler() is called which will start the next job (if any) +// and take care of the callbacks for the failed request. If the error +// method returns a non-0 return code then the error is non-recoverable and +// the queue is marked with the error state. + +void +async_error_handler(AsyncQueue* queue, uint8_t completion_state) +{ + AsyncRequest *finished; + + finished = (AsyncRequest *)(queue->current); + + if (SSX_ERROR_CHECK_KERNEL && (finished == 0)) { + SSX_PANIC(ASYNC_PHANTOM_ERROR); + } + + if (finished->error_method) { + if (finished->error_method(finished)) { + queue->state = ASYNC_QUEUE_STATE_ERROR; + } + } + finished->abort_state = finished->state; + finished->completion_state = completion_state; + async_handler(queue); +} + + +//////////////////////////////////////////////////////////////////////////// +// AsyncRequest +//////////////////////////////////////////////////////////////////////////// + +// Dequeue a queued AsyncRequest +// +// This is an internal API, always called from an SSX_CRITICAL critical +// section. The request is known to be queued in one of the async queues. It +// is removed from the queue and its state is updated. + +static void +async_request_dequeue(AsyncRequest* request, uint8_t completion_state) +{ + ssx_deque_delete((SsxDeque*)request); + request->abort_state = request->state; + request->completion_state = completion_state; + async_request_complete(request); +} + + +// Time out an AsyncRequest +// +// This is an internal API, the timer callback used to time out long-running +// AsyncRequest. +// +// This timer callback must run in an SSX_CRITICAL critical section to +// guarantee atomic access to the AsyncRequest object. + +static void +async_timeout(void* arg) +{ + AsyncRequest *request = (AsyncRequest*)arg; + SsxMachineContext ctx; + + ssx_critical_section_enter(SSX_CRITICAL, &ctx); + + // The behavior at timeout depends on the mode. We'll handle the cases + // from easiest to hardest. + + if (request->state & ASYNC_REQUEST_CALLBACK_GROUP) { + + // If the request has already queued or is running the callback (which + // could happen due to interrupt interleaving) then the request is + // already finished, and it will eventually make a (redundant) call to + // ssx_timer_cancel() to cancel the timer. So there's nothing to do. + + + } else if (request->state & ASYNC_REQUEST_IDLE_GROUP) { + + // If the request is idle we panic - This can't happen as it would + // indicate that the timer was not cancelled when the request + // finished. + + SSX_PANIC(ASYNC_TIMEOUT_BUG); + + + } else if (request->state & ASYNC_REQUEST_QUEUED_GROUP) { + + // If the request is still in the queue then we can simply cancel it, + // which includes running the callback. + + async_request_dequeue(request, ASYNC_REQUEST_STATE_TIMED_OUT); + + + } else if (request->state & ASYNC_REQUEST_RUNNING_GROUP) { + + // If the request is running on the hardware, then we need to call + // the async_error_handler() to remove the job and restart the + // hardware, which includes running the callback. + + async_error_handler(request->queue, ASYNC_REQUEST_STATE_TIMED_OUT); + + + } else { + + SSX_PANIC(ASYNC_INVALID_STATE); + + } + + ssx_critical_section_exit(&ctx); +} + + +// Create (initialize) a generic AsyncRequest +// +// This is an internal API used to initialize generic requests. However this +// API does some error checking, and any errors will be returned by the +// higher-level call. +// +// \retval -ASYNC_INVALID_OBJECT_REQUEST The \a request or \a queue was +/// null (0). +// +// \retval -ASYNC_INVALID_OPTIONS The \a options argument contains invalid +// options, or more than one callback protocol was selected. +// +// \retval -ASYNC_INVALID_ARGUMENT The \a run_method was null. +// +// \retval -ASYNC_CALLBACK_PROTOCOL_UNSPECIFIED The request includes a +// non-NULL callback, but no protocol for running the callback was specified. + +int +async_request_create(AsyncRequest *request, + AsyncQueue *queue, + AsyncRunMethod run_method, + AsyncErrorMethod error_method, + SsxInterval timeout, + AsyncRequestCallback callback, + void *arg, + int options) +{ + if (SSX_ERROR_CHECK_API) { + SSX_ERROR_IF((request == 0) || (queue == 0), + ASYNC_INVALID_OBJECT_REQUEST); + SSX_ERROR_IF((options & ~ASYNC_GENERIC_OPTIONS) || + (__builtin_popcount(options & ASYNC_CALLBACK_OPTIONS) > 1), + ASYNC_INVALID_OPTIONS); + SSX_ERROR_IF(run_method == 0, ASYNC_INVALID_ARGUMENT); + SSX_ERROR_IF((callback != 0) && + ((options & ASYNC_CALLBACK_OPTIONS) == 0), + ASYNC_CALLBACK_PROTOCOL_UNSPECIFIED); + } + + ssx_deque_element_create(&(request->deque)); + request->queue = queue; + request->run_method = run_method; + request->error_method = error_method; + request->timeout = timeout; + request->state = ASYNC_REQUEST_STATE_INITIALIZED; + request->callback = callback; + request->arg = arg; + request->options = options; + + if (request->timeout != SSX_WAIT_FOREVER) { + ssx_timer_create(&(request->timer), async_timeout, (void*)request); + } + + if (options & ASYNC_REQUEST_BLOCKING) { + ssx_semaphore_create(&(request->sem), 0, 1); + } + + + return 0; +} + + +/// Get timestamps from an AsyncRequest object +/// +/// \param request A pointer to an AsyncRequest +/// +/// \param start_time A pointer to a location to get the \a start_time of the +/// request, or NULL (0) if this data is not required. +/// +/// \param end_time A pointer to a location to get the \a end_time of the +/// request, or NULL (0) is this data is not required. +/// +/// \retval 0 The request contains valid timestamps and they have been +/// returned. +/// +/// \retval -ASYNC_INVALID_TIMESTAMPS The caller's timestamps have been +/// updated, but the timestamps are fully or partially invalid. This could be +/// due to several reasons: +/// +/// - The request has never been scheduled +/// - The request has been scheduled but has not completed on the device +/// - For space/time reasons, timestamps are not supported + +int +async_request_timestamps_get(AsyncRequest* request, + SsxTimebase* start_time, + SsxTimebase* end_time) +{ + int rc; + SsxMachineContext ctx; + + ssx_critical_section_enter(SSX_CRITICAL, &ctx); + + if (start_time) { + *start_time = request->start_time; + } + if (end_time) { + *end_time = request->end_time; + } + + if ((request->state & ASYNC_REQUEST_IDLE_GROUP) && + (request->state != ASYNC_REQUEST_STATE_INITIALIZED)) { + rc = 0; + } else { + rc = -ASYNC_INVALID_TIMESTAMPS; + } + + ssx_critical_section_exit(&ctx); + + return rc; +} + + +/// Compute the latency of an AsyncRequest +/// +/// \param request A pointer to an AsyncRequest +/// +/// \param latency A pointer to a location to receive the latency (end time - +/// start time) computed from the timestamps of \a request. +/// +/// \retval 0 The request contains valid timestamps and they have been +/// returned. +/// +/// \retval -ASYNC_INVALID_TIMESTAMPS The latancy has been computed but may be +/// invalid. This could be due to several reasons: +/// +/// - The request has never been scheduled +/// - The request has been scheduled but has not completed on the device +/// - For space/time reasons, timestamps are not supported + +int +async_request_latency(AsyncRequest* request, SsxTimebase* latency) +{ + int rc; + SsxTimebase start, end; + + rc = async_request_timestamps_get(request, &start, &end); + *latency = end - start; + return rc; +} + + +// Dump an AsyncRequest + +void +async_request_printk(AsyncRequest *request) +{ + printk("----------------------------------------\n"); + printk("-- AsyncRequest @ %p\n", request); + printk("-- deque = %p\n", &(request->deque)); + printk("-- start_time = 0x%016llx\n", request->start_time); + printk("-- end_time = 0x%016llx\n", request->end_time); + printk("-- sem = %p\n", &(request->sem)); + printk("-- queue = %p\n", request->queue); + printk("-- run_method = %p\n", request->run_method); + printk("-- error_method = %p\n", request->error_method); + printk("-- callback = %p\n", request->callback); + printk("-- arg = %p\n", request->arg); + printk("-- state = 0x%02x\n", request->state); + printk("-- completion_state = 0x%02x\n", request->completion_state); + printk("-- options = 0x%04x\n", request->options); + printk("----------------------------------------\n"); +} + + +//////////////////////////////////////////////////////////////////////////// +// Callback Queue +//////////////////////////////////////////////////////////////////////////// + +SSX_IRQ_FAST2FULL(async_callback_handler, async_callback_handler_full); + +// The handler for the asynchronous callback queue +// +// This is a full-mode noncritical interrupt handler. It is activated to run +// 1) all deferred callbacks, and 2) noncritical callbacks invoked from +// critical interrupt handlers, and 3) Thread-unblock and/or timeout-cancel +// requests that needed to be deferred to a noncritical context. The callback +// is enqueued in the SsxDeque passed as the private argument, and an IPI is +// used to activate this handler. +// +// This handler runs each callback in order. Since the callback queue may be +// managed by critical interrupt handlers we need to disable critical +// interrupts when popping the next element from the queue. +// +// Deferred callbacks are run with noncritical interrupts \e enabled, similar +// to how timer callbacks are run. +// +// Noncritical callbacks that were deferred here (by being invoked from a +// critical handler) run with interrupts \e disabled, to be consistent with +// the expected environment. Interrupts are then renabled briefly for +// interrupt latency mitigation. +// +// Note that NULL callbacks may be enqueued here but only in the state +// ASYNC_REQUEST_STATE_POSTPROCESSING. + +// The handler runs with its own IRQ disabled to avoid infinite interrupt +// loops caused by enabling interrupt preemption. The IRQ status can only be +// cleared inside an SSX_CRITICAL critical section. For efficiency we only do +// this when we know that no more callbacks are queued. + +// Final request completion has been broken out into a generic routine, +// async_request_finalize(). This routine is also called by the PTS +// completion queue handler, which handles its requests ion a slightly +// different way from the other async handlers. + +void +async_request_finalize(AsyncRequest* request) +{ + if (request->state == ASYNC_REQUEST_STATE_CALLBACK_QUEUED) { + + request->state = ASYNC_REQUEST_STATE_CALLBACK_RUNNING; + + if (request->options & ASYNC_CALLBACK_DEFERRED) { + + ssx_interrupt_preemption_enable(); + request->callback_rc = request->callback(request->arg); + + } else { + + request->callback_rc = request->callback(request->arg); + ssx_interrupt_preemption_enable(); + } + ssx_interrupt_preemption_disable(); + } + + request->state = request->completion_state; + + if (request->timeout != SSX_WAIT_FOREVER) { + ssx_timer_cancel(&(request->timer)); + } + + if (request->options & ASYNC_REQUEST_BLOCKING) { + ssx_semaphore_post(&(request->sem)); + } +} + + +void +async_callback_handler_full(void *arg, SsxIrqId irq, int priority) +{ + SsxMachineContext ctx; + SsxDeque *queue = (SsxDeque *)arg; + AsyncRequest *request; + + ssx_irq_disable(irq); + + do { + + ssx_critical_section_enter(SSX_CRITICAL, &ctx); + + request = (AsyncRequest *)ssx_deque_pop_front(queue); + if (!request) { + ssx_irq_status_clear(irq); + break; + } + + ssx_critical_section_exit(&ctx); + + async_request_finalize(request); + + } while (1); + + ssx_critical_section_exit(&ctx); + ssx_irq_enable(irq); +} + + +//////////////////////////////////////////////////////////////////////////// +// Initialization +//////////////////////////////////////////////////////////////////////////// + +// These initialization routines are for boot-time initialization, or +// test-mode reinitialization of interface parameters when the interface is +// known to be idle. They are not robust enough for mid-application +// reset/reprogramming of the asynchronous interfaces in the event of errors. + +// For the interrupt setup, whether or not the interrupt is enabled at the +// exit of setup depends on the particular driver being initialized. + +void +async_edge_handler_setup(SsxIrqHandler handler, + void *arg, + SsxIrqId irq, + int priority) +{ + ssx_irq_disable(irq); + ssx_irq_setup(irq, + SSX_IRQ_POLARITY_ACTIVE_HIGH, + SSX_IRQ_TRIGGER_EDGE_SENSITIVE); + ssx_irq_handler_set(irq, handler, arg, priority); + ssx_irq_status_clear(irq); +} + + +void +async_level_handler_setup(SsxIrqHandler handler, + void *arg, + SsxIrqId irq, + int priority, + int polarity) +{ + ssx_irq_disable(irq); + ssx_irq_setup(irq, + polarity, + SSX_IRQ_TRIGGER_LEVEL_SENSITIVE); + ssx_irq_handler_set(irq, handler, arg, priority); +} + + +void +async_callbacks_initialize(SsxDeque *queue, SsxIrqId irq) +{ + ssx_deque_sentinel_create(queue); + async_edge_handler_setup(async_callback_handler, + (void *)queue, + irq, SSX_NONCRITICAL); + ssx_irq_enable(irq); +} + + +/// Create all of the PgP asynchronous request structures and install and +/// activate the interrupt handlers. + +void +async_initialize() +{ + // This is the callback queue used e.g. when critical interrupts need to + // run non-critical callbacks. + + async_callbacks_initialize(&G_async_callback_queue, PGP_IRQ_ASYNC_IPI); + + // PORE + + async_pore_initialize(&G_pore_gpe0_queue, ASYNC_ENGINE_PORE_GPE0); + async_pore_initialize(&G_pore_gpe1_queue, ASYNC_ENGINE_PORE_GPE1); + async_pore_initialize(&G_pore_slw_queue, ASYNC_ENGINE_PORE_SLW); + +#if CONFIGURE_PTS + // PTS + + async_pts_initialize(&G_pts_gpe0_queue, ASYNC_ENGINE_PORE_GPE0); + async_pts_initialize(&G_pts_gpe1_queue, ASYNC_ENGINE_PORE_GPE1); +#endif + + // BCE + + async_bce_initialize(&G_pba_bcde_queue, + ASYNC_ENGINE_BCDE, + PGP_IRQ_PBA_BCDE_ATTN); + + async_bce_initialize(&G_pba_bcue_queue, + ASYNC_ENGINE_BCUE, + PGP_IRQ_PBA_BCUE_ATTN); + + // OCB + + async_ocb_initialize(&(G_ocb_read_queue[0]), + ASYNC_ENGINE_OCB_PUSH0, + G_ocb_read0_buffer, + OCB_READ0_LENGTH, + OCB_READ0_PROTOCOL); + + async_ocb_initialize(&(G_ocb_read_queue[1]), + ASYNC_ENGINE_OCB_PUSH1, + G_ocb_read1_buffer, + OCB_READ1_LENGTH, + OCB_READ1_PROTOCOL); + + async_ocb_initialize(&(G_ocb_read_queue[2]), + ASYNC_ENGINE_OCB_PUSH2, + G_ocb_read2_buffer, + OCB_READ2_LENGTH, + OCB_READ2_PROTOCOL); + + async_ocb_initialize(&(G_ocb_write_queue[0]), + ASYNC_ENGINE_OCB_PULL0, + G_ocb_write0_buffer, + OCB_WRITE0_LENGTH, + OCB_WRITE0_PROTOCOL); + + async_ocb_initialize(&(G_ocb_write_queue[1]), + ASYNC_ENGINE_OCB_PULL1, + G_ocb_write1_buffer, + OCB_WRITE1_LENGTH, + OCB_WRITE1_PROTOCOL); + + async_ocb_initialize(&(G_ocb_write_queue[2]), + ASYNC_ENGINE_OCB_PULL2, + G_ocb_write2_buffer, + OCB_WRITE2_LENGTH, + OCB_WRITE2_PROTOCOL); + + // PBAX + + async_pbax_initialize(&G_pbax_read_queue[0], + ASYNC_ENGINE_PBAX_PUSH0, + PGP_IRQ_PBA_OCC_PUSH0, + G_pbax_read0_buffer, + PBAX_READ0_LENGTH, + PBAX_READ0_PROTOCOL); + + async_pbax_initialize(&G_pbax_read_queue[1], + ASYNC_ENGINE_PBAX_PUSH1, + PGP_IRQ_PBA_OCC_PUSH1, + G_pbax_read1_buffer, + PBAX_READ1_LENGTH, + PBAX_READ1_PROTOCOL); +} |
