PEL: Notifier support for 'host full'

The host firmware may have a limited size for its staging area
before it passes the PELs through to the OS, and this area may
fill up with PELs it can't send if there are too many or the OS
isn't up yet.

In this case, it will send down an 'Ack PEL' PLDM command with a
special response indicating this host full condition.  The PLDM
daemon will then call a method on this daemon to let it know.

This command handles the host full condition on the HostNotifier
class.  When this is set:

* The PEL that hit this condition will be put back on the queue
  to be sent again.
* No new PELs will be sent up, except as noted below
* A 60s timer will be started at the end of which another attempt
  will be made to send a PEL, in the hopes the condition went away.
  - If it didn't go away, this process will repeat.
  - If it did go away, a successful ack will be received and things
    will proceed as normal.

Signed-off-by: Matt Spinler <spinler@us.ibm.com>
Change-Id: Iaeb38f43f7acc595bcff234ba50cedf8188b3d9b

2 files changed, 113 insertions, 4 deletions

diff --git a/extensions/openpower-pels/host_notifier.cpp b/extensions/openpower-pels/host_notifier.cpp
index b4e92a8..ca30340 100644
--- a/extensions/openpower-pels/host_notifier.cpp
+++ b/extensions/openpower-pels/host_notifier.cpp
@@ -30,7 +30,10 @@ HostNotifier::HostNotifier(Repository& repo, DataInterfaceBase& dataIface,
     _repo(repo),
     _dataIface(dataIface), _hostIface(std::move(hostIface)),
     _retryTimer(_hostIface->getEvent(),
-                std::bind(std::mem_fn(&HostNotifier::retryTimerExpired), this))
+                std::bind(std::mem_fn(&HostNotifier::retryTimerExpired), this)),
+    _hostFullTimer(
+        _hostIface->getEvent(),
+        std::bind(std::mem_fn(&HostNotifier::hostFullTimerExpired), this))
 {
     // Subscribe to be told about new PELs.
     _repo.subscribeToAdds(subscriptionName,
@@ -155,7 +158,8 @@ void HostNotifier::newLogCallback(const PEL& pel)
 
     _pelQueue.push_back(pel.id());
 
-    if (!_dataIface.isHostUp())
+    // Notify shouldn't happen if host is down or full
+    if (!_dataIface.isHostUp() || _hostFull)
     {
         return;
     }
@@ -194,7 +198,8 @@ void HostNotifier::dispatch(sdeventplus::source::EventBase& source)
 
 void HostNotifier::doNewLogNotify()
 {
-    if (!_dataIface.isHostUp() || _retryTimer.isEnabled())
+    if (!_dataIface.isHostUp() || _retryTimer.isEnabled() ||
+        _hostFullTimer.isEnabled())
     {
         return;
     }
@@ -264,6 +269,7 @@ void HostNotifier::doNewLogNotify()
 void HostNotifier::hostStateChange(bool hostUp)
 {
     _retryCount = 0;
+    _hostFull = false;
 
     if (hostUp && !_pelQueue.empty())
     {
@@ -282,6 +288,11 @@ void HostNotifier::hostStateChange(bool hostUp)
         }
 
         _sentPELs.clear();
+
+        if (_hostFullTimer.isEnabled())
+        {
+            _hostFullTimer.setEnabled(false);
+        }
     }
 }
 
@@ -298,7 +309,8 @@ void HostNotifier::commandResponse(ResponseStatus status)
 
         _repo.setPELHostTransState(id, TransmissionState::sent);
 
-        if (!_pelQueue.empty())
+        // If the host is full, don't send off the next PEL
+        if (!_hostFull && !_pelQueue.empty())
         {
             doNewLogNotify();
         }
@@ -324,6 +336,11 @@ void HostNotifier::retryTimerExpired()
     }
 }
 
+void HostNotifier::hostFullTimerExpired()
+{
+    doNewLogNotify();
+}
+
 void HostNotifier::stopCommand()
 {
     _retryCount = 0;
@@ -355,6 +372,51 @@ void HostNotifier::ackPEL(uint32_t id)
     {
         _sentPELs.erase(sent);
     }
+
+    // An ack means the host is no longer full
+    if (_hostFullTimer.isEnabled())
+    {
+        _hostFullTimer.setEnabled(false);
+    }
+
+    if (_hostFull)
+    {
+        _hostFull = false;
+
+        // Start sending PELs again, from the event loop
+        if (!_pelQueue.empty())
+        {
+            scheduleDispatch();
+        }
+    }
+}
+
+void HostNotifier::setHostFull(uint32_t id)
+{
+    log<level::INFO>("Received Host full indication", entry("PEL_ID=0x%X", id));
+
+    _hostFull = true;
+
+    // This PEL needs to get re-sent
+    auto sent = std::find(_sentPELs.begin(), _sentPELs.end(), id);
+    if (sent != _sentPELs.end())
+    {
+        _sentPELs.erase(sent);
+        _repo.setPELHostTransState(id, TransmissionState::newPEL);
+
+        if (std::find(_pelQueue.begin(), _pelQueue.end(), id) ==
+            _pelQueue.end())
+        {
+            _pelQueue.push_front(id);
+        }
+    }
+
+    // The only PELs that will be sent when the
+    // host is full is from this timer callback.
+    if (!_hostFullTimer.isEnabled())
+    {
+        _hostFullTimer.restartOnce(_hostIface->getHostFullRetryDelay());
+    }
 }
 
 } // namespace openpower::pels
diff --git a/extensions/openpower-pels/host_notifier.hpp b/extensions/openpower-pels/host_notifier.hpp
index d160dd4..21bd072 100644
--- a/extensions/openpower-pels/host_notifier.hpp
+++ b/extensions/openpower-pels/host_notifier.hpp
@@ -87,10 +87,36 @@ class HostNotifier
      *
      * This means the PEL never needs to be sent up again.
      *
+     * If the host was previously full, it is also an indication
+     * it no longer is.
+     *
      * @param[in] id - The PEL ID
      */
     void ackPEL(uint32_t id);
 
+    /**
+     * @brief Called when the host does not have room for more
+     *        PELs at this time.
+     *
+     * This can happen when an OS isn't running yet, and the
+     * staging area to hold the PELs before sending them up
+     * to the OS is full.  This will stop future PEls from being
+     * sent up, as explained below.
+     *
+     * The PEL with this ID will need to be sent again, so its
+     * state is set back to 'new', and it is removed from the list
+     * of already sent PELs.
+     *
+     * A timer will be started, if it isn't already running, to
+     * issue another send in the hopes that space has been freed
+     * up by then (Receiving an ackPEL response is also an
+     * indication of this if there happened to have been other
+     * PELs in flight).
+     *
+     * @param[in] id - The PEL ID
+     */
+    void setHostFull(uint32_t id);
+
   private:
     /**
      * @brief This function gets called by the Repository class
@@ -171,6 +197,15 @@ class HostNotifier
     void retryTimerExpired();
 
     /**
+     * @brief The function called when the 'host full' retry timer
+     *        expires.
+     *
+     * This will re-issue a command to try again with the PEL at
+     * the front of the queue.
+     */
+    void hostFullTimerExpired();
+
+    /**
      * @brief Stops an in progress command
      *
      * In progress meaning after the send but before the response.
@@ -217,11 +252,23 @@ class HostNotifier
     size_t _retryCount = 0;
 
     /**
+     * @brief Indicates if the host has said it is full and does not
+     *        currently have the space for more PELs.
+     */
+    bool _hostFull = false;
+
+    /**
      * @brief The command retry timer.
      */
     sdeventplus::utility::Timer<sdeventplus::ClockId::Monotonic> _retryTimer;
 
     /**
+     * @brief The host full timer, used to retry sending a PEL if the host
+     *        said it is full.
+     */
+    sdeventplus::utility::Timer<sdeventplus::ClockId::Monotonic> _hostFullTimer;
+
+    /**
      * @brief The object used to dispatch a new PEL send from the
      *        event loop, so the calling function can be returned from
      *        first.