summaryrefslogtreecommitdiffstats
path: root/src/ssx/pgp/pgp_async.c
blob: f9c446f1aeb930e018bb267aa2c92c2196926e50 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
// $Id: pgp_async.c,v 1.4 2014/02/14 12:18:05 bcbrock Exp $
// $Source: /afs/awd/projects/eclipz/KnowledgeBase/.cvsroot/eclipz/chips/p8/working/procedures/ssx/pgp/pgp_async.c,v $
//-----------------------------------------------------------------------------
// *! (C) Copyright International Business Machines Corp. 2013
// *! All Rights Reserved -- Property of IBM
// *! *** IBM Confidential ***
//-----------------------------------------------------------------------------

/// \file pgp_async.c
/// \brief Support for asynchronous request queuing and callback mechanisms
///
/// This file implements device drivers for asynchronous requests.  The model
/// for devices like the PORE engines and the PBA block copy engines is that
/// the application creates self-contained requests for jobs to run on the
/// engine.  The application then schedules the request and continues normal
/// processing, or threads can request to simply block in place until the
/// request finishes.  Queue management is handled in the interrupt handler
/// for the engine. As each queued job finishes, the next request (if any) is
/// started on the engine and the optional callback is invoked or scheduled.
///
/// The application can either use a polling protocol or the asynchronous
/// callback mechanism to determine the state of the request.  A timeout
/// mechanism is also provided that cancels or kills a job that does not
/// complete within a fixed time. Error handling in the event that the request
/// does not complete in time or has failed is standardized by the
/// implementation, but can be overridden by the application.
///
/// Asynchronous request interrupt handlers can run either as critical or
/// noncritical handlers.  Some engines may support queuing requests that
/// complete immediately; therefore the kernel context of the callback may be
/// either a thread or interupt context, and in general, callbacks should not
/// make assumptions about the kernel context when they run.
///
/// If the callback is non-NULL and the request was created with \c
/// ASYNC_CALLBACK_IMMEDIATE, the callback is then immediately invoked in the
/// current context. In general, immediate callbacks should be short and sweet
/// (to reduce interrupt latency) and should not make SSX kernel calls.
///
/// If the request has a non-NULL callback and was created with \c
/// ASYNC_CALLBACK_DEFERRED then the callback will be deferred to a
/// noncritical interrupt context. Deferred callbacks are queued, then run
/// later in response to a reserved IPI.  Deferred callbacks always run as
/// noncritical interrupt handlers with noncritical interrupts \e enabled,
/// similar to SSX timer callbacks.
///
/// If the request has a non-null callback and was created with \c
/// ASYNC_CALLBACK_NONCRITICAL, then the callback will be run immediately from
/// a noncritical handler or thread environment, or deferred from a critical
/// interrupt context.  Similar to immediate callbacks, callbacks marked
/// noncritical should be short and sweet (to reduce interrupt latency), but
/// may make SSX kernel calls.
///
/// Regardless of whether the callback is critical or noncritical, the
/// callback is coded as a normal C or assembler subroutine with a single
/// void* argument.
///
/// As a programming shortcut, the AsyncRequest includes a semaphore object.
/// A special AsyncRequest option ASYNC_REQUEST_BLOCKING indicates that the
/// thread scheduling the request should block on the semaphore of the
/// AsyncRequest (with SSX_WAIT_FOREVER) until the request is complete. Note
/// that a separate callback can be specified even if ASYNC_REQUEST_BLOCKING
/// is specified.
///
/// Requests are always timestamped. The \a start_time field of the request is
/// set from the SSX timebase when the request is first 'run' on the device.
/// Request types that may require multiple back-to-back 'runs' (like PBA
/// block copies with more than 4K data) only record the initial 'run'. The \a
/// end_time field of the request is set from the SSX timebase when the job
/// finishes on the hardware, but before any callback is run.  These
/// timestamps cover time intervals not visible to the application; The
/// application is responsible for timestamping the period between request
/// queing and the \a start_time, and between the \a end_time and callback
/// completion if required.  The timestamps can be robustly recovered from the
/// request using the API async_request_timestamps_get().
///
/// This is largely a generic implementation, designed to reduce code space by
/// allowing the PORE, PBA and OCB drivers to use the same generic data
/// structures and code.  This is supported by the 'single-inheritence class
/// hierarchy' described in the comments for pgp_async.h.
///
/// <b> Request Completion and Callback States </b>
///
/// The application can determine what happend to the job by observing the
/// request state and callback return code when the request becomes idle.
/// A request is not considered idle until the job has run to completion or
/// been aborted, any callback has been run, any timeout has been cancelled,
/// and any thread pending on request completion has been maxde runnable.
///
/// Normally the application should see the request state (\a request->state)
/// ASYNC_REQUEST_STATE_COMPLETE and a  callback return code 
/// (\a request=->callback_rc) of 0 which indicates that the job and callback
/// both completed normally.  If the request did not complete normally it
/// could be due to one of several reasons.  For further analysis the request
/// also includes a field \a abort_state that records the state the job was in
/// when it was aborted (i.e., cancelled, killed, timed out or error-out).
///
/// <b> Timeout Semantics </b>
///
/// Any timeout other than SSX_WAIT_FOREVER specifies a timeout covering the
/// interval spanning the time a job is scheduled until the time the job
/// completes on the hardware.  If the job is still wating to execute when it
/// times out then the job is simply removed from the queue and marked as
/// having timed out.  If the job is running when it times out then the job is
/// forceably removed from the hardware, which may have unintended or
/// indeterminate consequences. The application may need to consider whether
/// it is safe to continue after a forced timeout.
///
/// Specifying a timeout involves quite a bit of overhead, since a timer needs
/// to be scheduled and cancelled each time a job is run.  If the interrupt
/// handler for a device is a critical handler then the timeout cancellation
/// will need to be deferred to the callback queue, potentially increasing
/// overhead even further.
///
/// <b> Implementation Notes </b>
///
/// - The \e queue objects will normally be global data structures that
/// persist throughout the life of the application.
///
/// - The \e request objects may be either global or local data
/// structures. However, since \e request objects are not copied, and pointers
/// to them are stored in the \e queue objects, \a request objects should not
/// be allocated on the stack if the stack frame could become invalid before
/// the request completes.
///
/// \todo Once all function is developed and tested, convert interrupt
/// handling to fast-mode assembler routines.

#include "ssx.h"


////////////////////////////////////////////////////////////////////////////
// Global Data
////////////////////////////////////////////////////////////////////////////

/// Queue of deferred callbacks

static SsxDeque G_async_callback_queue;


////////////////////////////////////////////////////////////////////////////
// FFDC 
////////////////////////////////////////////////////////////////////////////

/// Collect FFDC for the PLB (OCI) arbiter
///
/// \param ffdc A pointer to an OciFfdc structure to be filled in.
///
/// \param master_id The PLB (OCI) master Id of the master of interest.
///
/// Note: The PEAR and PESR hold error information for all OCI masters
/// _except_ the OCC ICU and DCU. ICU and DCU errors are in the STO_PESR and
/// STO_PEAR. Currently there is no need to collect those DCRs for 'async'
/// errors. 

void
oci_ffdc(OciFfdc* ffdc, int master_id)
{
    uint32_t pesr_lock_mask;

    ffdc->pearl.value = mfdcr(PLB_PEARL);
    ffdc->pesr.value = mfdcr(PLB_PESR);

    pesr_lock_mask = 0x30000000 >> (4 * master_id);
    if (ffdc->pesr.value & pesr_lock_mask) {
        ffdc->mine = 1;
        mtdcr(PLB_PESR, pesr_lock_mask);
    } else {
        ffdc->mine = 0;
    }
}



////////////////////////////////////////////////////////////////////////////
// AsyncQueue
////////////////////////////////////////////////////////////////////////////

// Start an asynchronous request on the device with timestamping.  This will
// always be called from a critical section, and any timestamp is collected
// immediately before the request is kicked off on the device. Devices like
// the BCE engines and the OCB queue drivers may run the same request multiple
// times to get all of the data moved. Therefore the initial timestamp is only
// captured the first time the request is run on the device.

static inline int
async_request_run(AsyncRequest *request)
{
    if (request->state != ASYNC_REQUEST_STATE_RUNNING) {
        request->start_time = ssx_timebase_get();
    }
    request->state = ASYNC_REQUEST_STATE_RUNNING;
    return request->run_method(request);
}


// Create (initialize) a generic AsyncQueue
//
// This is an internal API used to initialize generic request queues.  The
// caller is assumed to have done all error checking on the parameters.
//
// This is a simple initialization that resets the queues and sets the state
// to QUEUE_STATE_IDLE.  This routine should only be called on uninitialized
// AsyncQueue objects.

int
async_queue_create(AsyncQueue *queue, AsyncEngine engine)
{
    ssx_deque_sentinel_create(&(queue->deque));
    queue->current = 0;
    queue->engine = engine;
    queue->state = ASYNC_QUEUE_STATE_IDLE;

    return 0;
}


// Generic completion of a request.  This is called both by async_handler()
// and async_request_deque().

static void
async_request_complete(AsyncRequest *request)
{
    SsxMachineContext ctx;
    AsyncRequestCallback callback;
    int completed;

    // Handle callbacks and deferred processing of the job that just
    // finished.  No callback is easy.  If the job does have a callback
    // that we can execute immediately then that is done immediately.
    // Note that 'completed' here means only that the callback is complete.

    callback = request->callback;
    if (!callback) {

        completed = 1;

    } else if ((request->options & ASYNC_CALLBACK_IMMEDIATE) ||
               ((request->options & ASYNC_CALLBACK_NONCRITICAL) &&
                !__ssx_kernel_context_critical_interrupt())) {

        request->state = ASYNC_REQUEST_STATE_CALLBACK_RUNNING;
        request->callback_rc = callback(request->arg);
        completed = 1;

    } else {

        request->state = ASYNC_REQUEST_STATE_CALLBACK_QUEUED;
        completed = 0;
    }


    // If the callback completed then we go ahead and cancel any timeout
    // and/or wake the thread if possible.  In critical interrupt contexts
    // we always have to defer these operations, so we may lose 'complete'
    // status. 

    if (completed && 
        ((request->timeout != SSX_WAIT_FOREVER) ||
         (request->options & ASYNC_REQUEST_BLOCKING))) {

        if (__ssx_kernel_context_critical_interrupt()) {

            request->state = ASYNC_REQUEST_STATE_POSTPROCESSING;
            completed = 0;

        } else {
            if (request->timeout != SSX_WAIT_FOREVER) {
                ssx_timer_cancel(&(request->timer));
            }
            if (request->options & ASYNC_REQUEST_BLOCKING) {
                ssx_semaphore_post(&(request->sem));
            }
        }
    }


    // A truly completed job gets its completion state here.  Otherwise we
    // have to schedule the deferred postprocessing.

    if (completed) {
            
        request->state = request->completion_state;

    } else {

        ssx_critical_section_enter(SSX_CRITICAL, &ctx);

        if (request->options & ASYNC_CALLBACK_PRIORITY) {
            ssx_deque_push_front(&G_async_callback_queue,
                                 &(request->deque));
        } else {
            ssx_deque_push_back(&G_async_callback_queue,
                                &(request->deque));
        }

        ssx_critical_section_exit(&ctx);
        ssx_irq_status_set(PGP_IRQ_ASYNC_IPI, 1);
    }
}


// The generic handler for asynchonous device completion.
//
// This handler processes completions of generic device requests, as well as
// the initial running of jobs when the queue is idle.  Error completions are
// initially handled by async_error_handler() which then calls async_handler()
// to finish the failed job and start the next job if possible.  The initial
// device handler must manage interrupts and provide a method to run the
// 'current' job in the queue. The engine-specific handler may also iterate
// the current job until it is complete - this handler should only be called
// when the current job is complete.
//
// Some engines may have jobs that can complete immediately. This handler
// iterates over jobs in the queue as long as the run method of the current
// job returns the code -ASYNC_REQUEST_COMPLETE.
//
// NB : Normally this call is made from an interrupt handler, however it may
// be called from job scheduling code if the engine is idle. Regardless, the
// caller must insure that any call for a queue is protected against
// interrupts for that queue.

void
async_handler(AsyncQueue *queue)
{
    AsyncRequest *finished, *current;
    int rc;

    // This loop is repeated as long as any job started in a loop completes
    // immediately.

    do {

        // This API may be called on an idle queue, which indicates that we
        // should simply start the job on the head of the queue.  Otherwise
        // save a pointer to the job that just finished and update its
        // end_time.

        if (queue->state == ASYNC_QUEUE_STATE_IDLE) {

            finished = 0;

        } else {

            finished = (AsyncRequest *)(queue->current);

            if (SSX_ERROR_CHECK_KERNEL && (finished == 0)) {
                SSX_PANIC(ASYNC_PHANTOM_INTERRUPT);
            }

            finished->end_time = ssx_timebase_get();

        }


        // If the queue is in an error state we will not schedule any further
        // jobs on this queue.  Otherwise we start the next job running on the
        // engine. 

        if (queue->state == ASYNC_QUEUE_STATE_ERROR) {

            queue->current = 0;
            rc = 0;

        } else {

            current = (AsyncRequest *)ssx_deque_pop_front(&(queue->deque));
            queue->current = current;

            if (current) {
                queue->state = ASYNC_QUEUE_STATE_RUNNING;
                rc = async_request_run(current);
            } else {
                queue->state = ASYNC_QUEUE_STATE_IDLE;
                rc = 0;
            }
        }


        // If no job just finished, continue with the loop.  If the job we
        // just enqueued finished immediately it will be 'finished' on the
        // next loop (it would have given an rc ==
        // -ASYNC_REQUEST_COMPLETE). Otherwise complete the request.

        if (finished != 0) {

            async_request_complete(finished);
        }

   } while (rc == -ASYNC_REQUEST_COMPLETE);
}


/// Schedule (queue for execution) a generic asynchronous request.
///
/// \param request An initialized and idle AsyncRequest
///
///
/// The request is queued for execution with all of the parameters provided
/// when the request was created.  It is considered an error to (re)schedule a
/// request that is currently scheduled, running, or has the callback queued or
/// in execution. Requests either need to be idle, or must be explicitly
/// cancelled or killed before thay can be (re)scheduled.  Because engine queue
/// interrupt handlers may run as critical interrupts, this routine operates in
/// a short \c SSX_CRITICAL critical section.
///
/// If the request is made to an otherwise empty queue then the async_handler()
/// must also be called immediately to begin the job. This will extend the
/// critical section slightly. This routine will not start a request on a
/// queue that has halted due to an error. The request will simply be enqueued
/// in that case.
///
/// \retval 0 Success
///
/// \retval -ASYNC_INVALID_OBJECT_SCHEDULE The \a request is NULL (0).
///
/// \retval -ASYNC_REQUEST_NOT_IDLE The \a request is (still) active in some
/// way at entry.
///
/// \retval -ASYNC_REQUEST_NOT_COMPLETE This code is returned for requests that
/// do not complete successfully, but only if ASYNC_REQUEST_BLOCKING is 
/// specified. The caller will need to examine the request state if necessary 
/// to determine whether the request failed, was cancelled or timed out.
///
/// \todo Consider making the critical section priority depend on the device
/// queue priority here, by adding a priority field to the queue. Without this
/// we always have to run the async_handler() in an SSX_CRITICAL critical
/// section.

int
async_request_schedule(AsyncRequest *request)
{
    SsxMachineContext ctx = SSX_THREAD_MACHINE_CONTEXT_DEFAULT; // For GCC
    AsyncQueue *queue;
    int rc;

    if (SSX_ERROR_CHECK_API) {
        SSX_ERROR_IF(request == 0, ASYNC_INVALID_OBJECT_SCHEDULE);
    }

    rc = 0;
    ssx_critical_section_enter(SSX_CRITICAL, &ctx);

    do {

        // Check to insure the request is idle (which check must be done in
        // the critical section), then start any required timeout.

        if (SSX_ERROR_CHECK_API) {
            if (!async_request_is_idle(request)) {
                rc = -ASYNC_REQUEST_NOT_IDLE;
                break;
            }
        }

        if (request->timeout != SSX_WAIT_FOREVER) {
            rc = ssx_timer_schedule(&(request->timer), request->timeout, 0);
            if (rc) break;
        }
    

        // Enqueue the request and initialize the request state.

        queue = request->queue;

        if (request->options & ASYNC_REQUEST_PRIORITY) {
            ssx_deque_push_front(&(queue->deque), &(request->deque));
        } else {
            ssx_deque_push_back(&(queue->deque), &(request->deque));
        }

        request->state = ASYNC_REQUEST_STATE_QUEUED;
        request->completion_state = ASYNC_REQUEST_STATE_COMPLETE;
        request->callback_rc = 0;


        //  If the queue is idle, call the async_handler() to start the job.
        //  Then block the calling thread if required.

        if (queue->state == ASYNC_QUEUE_STATE_IDLE) {
            async_handler(queue);
        }

        if (request->options & ASYNC_REQUEST_BLOCKING) {
            rc = ssx_semaphore_pend(&(request->sem), SSX_WAIT_FOREVER);
            if (rc) break;

            if (!async_request_completed(request)) {
                rc = -ASYNC_REQUEST_NOT_COMPLETE;
                break;
            }
        }

    } while (0);

    ssx_critical_section_exit(&ctx);

    return rc;
}


// The generic error handler
//
// This is a generic handler called in response to an error interrupt or
// timeout.  This handler calls a request-specific error method to stop the
// hardware device, collect FFDC and make the hardware device runnable again
// if possible.  Then the current request is marked as having failed, and the
// generic async_handler() is called which will start the next job (if any)
// and take care of the callbacks for the failed request.  If the error
// method returns a non-0 return code then the error is non-recoverable and
// the queue is marked with the error state.

void
async_error_handler(AsyncQueue* queue, uint8_t completion_state)
{
    AsyncRequest *finished;

    finished = (AsyncRequest *)(queue->current);

    if (SSX_ERROR_CHECK_KERNEL && (finished == 0)) {
        SSX_PANIC(ASYNC_PHANTOM_ERROR);
    }

    if (finished->error_method) {
        if (finished->error_method(finished)) {
            queue->state = ASYNC_QUEUE_STATE_ERROR;
        }
    }
    finished->abort_state = finished->state;
    finished->completion_state = completion_state;
    async_handler(queue);
}


////////////////////////////////////////////////////////////////////////////
// AsyncRequest
////////////////////////////////////////////////////////////////////////////

// Dequeue a queued AsyncRequest
//
// This is an internal API, always called from an SSX_CRITICAL critical
// section. The request is known to be queued in one of the async queues.  It
// is removed from the queue and its state is updated.

static void
async_request_dequeue(AsyncRequest* request, uint8_t completion_state)
{
    ssx_deque_delete((SsxDeque*)request);
    request->abort_state = request->state;
    request->completion_state = completion_state;
    async_request_complete(request);
}


// Time out an AsyncRequest
//
// This is an internal API, the timer callback used to time out long-running
// AsyncRequest. 
//
// This timer callback must run in an SSX_CRITICAL critical section to
// guarantee atomic access to the AsyncRequest object.

static void
async_timeout(void* arg)
{
    AsyncRequest *request = (AsyncRequest*)arg;
    SsxMachineContext ctx;

    ssx_critical_section_enter(SSX_CRITICAL, &ctx);

    // The behavior at timeout depends on the mode.  We'll handle the cases
    // from easiest to hardest.

    if (request->state & ASYNC_REQUEST_CALLBACK_GROUP) {

        // If the request has already queued or is running the callback (which
        // could happen due to interrupt interleaving) then the request is
        // already finished, and it will eventually make a (redundant) call to
        // ssx_timer_cancel() to cancel the timer.  So there's nothing to do.


    } else if (request->state & ASYNC_REQUEST_IDLE_GROUP) {

        // If the request is idle we panic - This can't happen as it would
        // indicate that the timer was not cancelled when the request
        // finished.  

        SSX_PANIC(ASYNC_TIMEOUT_BUG);


    } else if (request->state & ASYNC_REQUEST_QUEUED_GROUP) {

        // If the request is still in the queue then we can simply cancel it,
        // which includes running the callback.

        async_request_dequeue(request, ASYNC_REQUEST_STATE_TIMED_OUT); 


    } else if (request->state & ASYNC_REQUEST_RUNNING_GROUP) {

        // If the request is running on the hardware, then we need to call
        // the async_error_handler() to remove the job and restart the
        // hardware, which includes running the callback.

        async_error_handler(request->queue, ASYNC_REQUEST_STATE_TIMED_OUT);


    } else {

        SSX_PANIC(ASYNC_INVALID_STATE);

    }

    ssx_critical_section_exit(&ctx);
}


// Create (initialize) a generic AsyncRequest
//
// This is an internal API used to initialize generic requests. However this
// API does some error checking, and any errors will be returned by the
// higher-level call.
//
// \retval -ASYNC_INVALID_OBJECT_REQUEST The \a request or \a queue was 
/// null (0).
//
// \retval -ASYNC_INVALID_OPTIONS The \a options argument contains invalid
// options, or more than one callback protocol was selected.
//
// \retval -ASYNC_INVALID_ARGUMENT The \a run_method was null.
//
// \retval -ASYNC_CALLBACK_PROTOCOL_UNSPECIFIED The request includes a
// non-NULL callback, but no protocol for running the callback was specified.

int
async_request_create(AsyncRequest *request,
                     AsyncQueue *queue,
                     AsyncRunMethod run_method,
                     AsyncErrorMethod error_method,
                     SsxInterval timeout,
                     AsyncRequestCallback callback,
                     void *arg,
                     int options)
{
    if (SSX_ERROR_CHECK_API) {
        SSX_ERROR_IF((request == 0) || (queue == 0), 
                      ASYNC_INVALID_OBJECT_REQUEST);
        SSX_ERROR_IF((options & ~ASYNC_GENERIC_OPTIONS) ||
                     (__builtin_popcount(options & ASYNC_CALLBACK_OPTIONS) > 1),
                     ASYNC_INVALID_OPTIONS);
        SSX_ERROR_IF(run_method == 0, ASYNC_INVALID_ARGUMENT);
        SSX_ERROR_IF((callback != 0) && 
                     ((options & ASYNC_CALLBACK_OPTIONS) == 0),
                     ASYNC_CALLBACK_PROTOCOL_UNSPECIFIED);
    }

    ssx_deque_element_create(&(request->deque));
    request->queue = queue;
    request->run_method = run_method;
    request->error_method = error_method;
    request->timeout = timeout;
    request->state = ASYNC_REQUEST_STATE_INITIALIZED;
    request->callback = callback;
    request->arg = arg;
    request->options = options;
    
    if (request->timeout != SSX_WAIT_FOREVER) {
        ssx_timer_create(&(request->timer), async_timeout, (void*)request);
    }

    if (options & ASYNC_REQUEST_BLOCKING) {
        ssx_semaphore_create(&(request->sem), 0, 1);
    }


    return 0;
}


/// Get timestamps from an AsyncRequest object
///
/// \param request A pointer to an AsyncRequest
///
/// \param start_time A pointer to a location to get the \a start_time of the
/// request, or NULL (0) if this data is not required.
///
/// \param end_time A pointer to a location to get the \a end_time of the
/// request, or NULL (0) is this data is not required.
///
/// \retval 0 The request contains valid timestamps and they have been
/// returned. 
///
/// \retval -ASYNC_INVALID_TIMESTAMPS The caller's timestamps have been
/// updated, but the timestamps are fully or partially invalid.  This could be
/// due to several reasons:
///
/// - The request has never been scheduled
/// - The request has been scheduled but has not completed on the device
/// - For space/time reasons, timestamps are not supported

int
async_request_timestamps_get(AsyncRequest* request,
                             SsxTimebase* start_time,
                             SsxTimebase* end_time)
{
    int rc;
    SsxMachineContext ctx;

    ssx_critical_section_enter(SSX_CRITICAL, &ctx);

    if (start_time) {
        *start_time = request->start_time;
    }
    if (end_time) {
        *end_time = request->end_time;
    }

    if ((request->state & ASYNC_REQUEST_IDLE_GROUP) &&
        (request->state != ASYNC_REQUEST_STATE_INITIALIZED)) {
        rc = 0;
    } else {
        rc = -ASYNC_INVALID_TIMESTAMPS;
    }

    ssx_critical_section_exit(&ctx);

    return rc;
}


/// Compute the latency of an AsyncRequest
///
/// \param request A pointer to an AsyncRequest
///
/// \param latency A pointer to a location to receive the latency (end time -
/// start time) computed from the timestamps of \a request.
///
/// \retval 0 The request contains valid timestamps and they have been
/// returned. 
///
/// \retval -ASYNC_INVALID_TIMESTAMPS The latancy has been computed but may be
/// invalid.  This could be due to several reasons:
///
/// - The request has never been scheduled
/// - The request has been scheduled but has not completed on the device
/// - For space/time reasons, timestamps are not supported

int
async_request_latency(AsyncRequest* request, SsxTimebase* latency)
{
    int rc;
    SsxTimebase start, end;

    rc = async_request_timestamps_get(request, &start, &end);
    *latency = end - start;
    return rc;
}


// Dump an AsyncRequest

void
async_request_printk(AsyncRequest *request)
{
    printk("----------------------------------------\n");
    printk("-- AsyncRequest @@ %p\n", request);
    printk("--   deque            = %p\n", &(request->deque));
    printk("--   start_time       = 0x%016llx\n", request->start_time);
    printk("--   end_time         = 0x%016llx\n", request->end_time);
    printk("--   sem              = %p\n", &(request->sem));
    printk("--   queue            = %p\n", request->queue);
    printk("--   run_method       = %p\n", request->run_method);
    printk("--   error_method     = %p\n", request->error_method);
    printk("--   callback         = %p\n", request->callback);
    printk("--   arg              = %p\n", request->arg);
    printk("--   state            = 0x%02x\n", request->state);
    printk("--   completion_state = 0x%02x\n", request->completion_state);
    printk("--   options          = 0x%04x\n", request->options);
    printk("----------------------------------------\n");
}


////////////////////////////////////////////////////////////////////////////
// Callback Queue
////////////////////////////////////////////////////////////////////////////

SSX_IRQ_FAST2FULL(async_callback_handler, async_callback_handler_full);

// The handler for the asynchronous callback queue
//
// This is a full-mode noncritical interrupt handler.  It is activated to run
// 1) all deferred callbacks, and 2) noncritical callbacks invoked from
// critical interrupt handlers, and 3) Thread-unblock and/or timeout-cancel
// requests that needed to be deferred to a noncritical context. The callback
// is enqueued in the SsxDeque passed as the private argument, and an IPI is
// used to activate this handler.
//
// This handler runs each callback in order.  Since the callback queue may be
// managed by critical interrupt handlers we need to disable critical
// interrupts when popping the next element from the queue.
//
// Deferred callbacks are run with noncritical interrupts \e enabled, similar
// to how timer callbacks are run.
//
// Noncritical callbacks that were deferred here (by being invoked from a
// critical handler) run with interrupts \e disabled, to be consistent with
// the expected environment. Interrupts are then renabled briefly for
// interrupt latency mitigation.
//
// Note that NULL callbacks may be enqueued here but only in the state
// ASYNC_REQUEST_STATE_POSTPROCESSING.

// The handler runs with its own IRQ disabled to avoid infinite interrupt
// loops caused by enabling interrupt preemption. The IRQ status can only be
// cleared inside an SSX_CRITICAL critical section.  For efficiency we only do
// this when we know that no more callbacks are queued.

// Final request completion has been broken out into a generic routine,
// async_request_finalize().  This routine is also called by the PTS
// completion queue handler, which handles its requests ion a slightly
// different way from the other async handlers.

void
async_request_finalize(AsyncRequest* request)
{
    if (request->state == ASYNC_REQUEST_STATE_CALLBACK_QUEUED) {

        request->state = ASYNC_REQUEST_STATE_CALLBACK_RUNNING;

        if (request->options & ASYNC_CALLBACK_DEFERRED) {

            ssx_interrupt_preemption_enable();
            request->callback_rc = request->callback(request->arg);

        } else {

            request->callback_rc = request->callback(request->arg);
            ssx_interrupt_preemption_enable();
        }
        ssx_interrupt_preemption_disable();
    }

    request->state = request->completion_state;

    if (request->timeout != SSX_WAIT_FOREVER) {
        ssx_timer_cancel(&(request->timer));
    }

    if (request->options & ASYNC_REQUEST_BLOCKING) {
        ssx_semaphore_post(&(request->sem));
    }
}


void
async_callback_handler_full(void *arg, SsxIrqId irq, int priority)
{
    SsxMachineContext ctx;
    SsxDeque *queue = (SsxDeque *)arg;
    AsyncRequest *request;

    ssx_irq_disable(irq);

    do {

        ssx_critical_section_enter(SSX_CRITICAL, &ctx);

        request = (AsyncRequest *)ssx_deque_pop_front(queue);
        if (!request) {
            ssx_irq_status_clear(irq);
            break;
        }

        ssx_critical_section_exit(&ctx);

        async_request_finalize(request);

    } while (1);

    ssx_critical_section_exit(&ctx);
    ssx_irq_enable(irq);
}


////////////////////////////////////////////////////////////////////////////
// Initialization
////////////////////////////////////////////////////////////////////////////

// These initialization routines are for boot-time initialization, or
// test-mode reinitialization of interface parameters when the interface is
// known to be idle. They are not robust enough for mid-application
// reset/reprogramming of the asynchronous interfaces in the event of errors.

// For the interrupt setup, whether or not the interrupt is enabled at the
// exit of setup depends on the particular driver being initialized.

void
async_edge_handler_setup(SsxIrqHandler handler,
                         void *arg,
                         SsxIrqId irq,
                         int priority)
{
    ssx_irq_disable(irq);
    ssx_irq_setup(irq,
                  SSX_IRQ_POLARITY_ACTIVE_HIGH,
                  SSX_IRQ_TRIGGER_EDGE_SENSITIVE);
    ssx_irq_handler_set(irq, handler, arg, priority);
    ssx_irq_status_clear(irq);
}


void
async_level_handler_setup(SsxIrqHandler handler,
                          void *arg,
                          SsxIrqId irq,
                          int priority,
                          int polarity)
{
    ssx_irq_disable(irq);
    ssx_irq_setup(irq,
                  polarity,
                  SSX_IRQ_TRIGGER_LEVEL_SENSITIVE);
    ssx_irq_handler_set(irq, handler, arg, priority);
}


void
async_callbacks_initialize(SsxDeque *queue, SsxIrqId irq)
{
    ssx_deque_sentinel_create(queue);
    async_edge_handler_setup(async_callback_handler,
                             (void *)queue,
                             irq, SSX_NONCRITICAL);
    ssx_irq_enable(irq);
}


/// Create all of the PgP asynchronous request structures and install and
/// activate the interrupt handlers.

void
async_initialize()
{
    // This is the callback queue used e.g. when critical interrupts need to
    // run non-critical callbacks.

    async_callbacks_initialize(&G_async_callback_queue, PGP_IRQ_ASYNC_IPI);

    // PORE

    async_pore_initialize(&G_pore_gpe0_queue, ASYNC_ENGINE_PORE_GPE0);
    async_pore_initialize(&G_pore_gpe1_queue, ASYNC_ENGINE_PORE_GPE1);


#if CONFIGURE_PTS
    // PTS

    async_pts_initialize(&G_pts_gpe0_queue, ASYNC_ENGINE_PORE_GPE0);
    async_pts_initialize(&G_pts_gpe1_queue, ASYNC_ENGINE_PORE_GPE1);
#endif

    // BCE

    async_bce_initialize(&G_pba_bcde_queue,
                         ASYNC_ENGINE_BCDE,
                         PGP_IRQ_PBA_BCDE_ATTN);

    async_bce_initialize(&G_pba_bcue_queue,
                         ASYNC_ENGINE_BCUE,
                         PGP_IRQ_PBA_BCUE_ATTN);

    // OCB

    async_ocb_initialize(&(G_ocb_read_queue[0]),
                         ASYNC_ENGINE_OCB_PUSH0,
                         G_ocb_read0_buffer,
                         OCB_READ0_LENGTH,
                         OCB_READ0_PROTOCOL);

    async_ocb_initialize(&(G_ocb_read_queue[1]),
                         ASYNC_ENGINE_OCB_PUSH1,
                         G_ocb_read1_buffer,
                         OCB_READ1_LENGTH,
                         OCB_READ1_PROTOCOL);

    async_ocb_initialize(&(G_ocb_read_queue[2]),
                         ASYNC_ENGINE_OCB_PUSH2,
                         G_ocb_read2_buffer,
                         OCB_READ2_LENGTH,
                         OCB_READ2_PROTOCOL);

    async_ocb_initialize(&(G_ocb_write_queue[0]),
                         ASYNC_ENGINE_OCB_PULL0,
                         G_ocb_write0_buffer,
                         OCB_WRITE0_LENGTH,
                         OCB_WRITE0_PROTOCOL);

    async_ocb_initialize(&(G_ocb_write_queue[1]),
                         ASYNC_ENGINE_OCB_PULL1,
                         G_ocb_write1_buffer,
                         OCB_WRITE1_LENGTH,
                         OCB_WRITE1_PROTOCOL);

    async_ocb_initialize(&(G_ocb_write_queue[2]),
                         ASYNC_ENGINE_OCB_PULL2,
                         G_ocb_write2_buffer,
                         OCB_WRITE2_LENGTH,
                         OCB_WRITE2_PROTOCOL);

    // PBAX

    async_pbax_initialize(&G_pbax_read_queue[0],
                          ASYNC_ENGINE_PBAX_PUSH0,
                          PGP_IRQ_PBA_OCC_PUSH0,
                          G_pbax_read0_buffer,
                          PBAX_READ0_LENGTH,
                          PBAX_READ0_PROTOCOL);

    async_pbax_initialize(&G_pbax_read_queue[1],
                          ASYNC_ENGINE_PBAX_PUSH1,
                          PGP_IRQ_PBA_OCC_PUSH1,
                          G_pbax_read1_buffer,
                          PBAX_READ1_LENGTH,
                          PBAX_READ1_PROTOCOL);
}
OpenPOWER on IntegriCloud