ape: Improve reset handling for better FreeBSD compatibility. (#223)

- Ensure the APE/BMC does not go offline when an unrelated port is reconfigured.
- Reduce latency before reconfiguring a port after the host is powered on or off.
- Increase global reset delay for inproved FreeBSD compatibility.

3 files changed, 131 insertions, 81 deletions

diff --git a/ape/main.c b/ape/main.c
index a72e22d..0b238b2 100644
--- a/ape/main.c
+++ b/ape/main.c
@@ -66,12 +66,22 @@
 
 #define RMU_WATCHDOG_TIMEOUT_MS (10)
 #define RX_CPU_RESET_TIMEOUT_MS (1000) /* Wait up to 1 second for each RX CPU to start */
-#define GRC_RESET_TIMEOUT_MS (150)     /* Wait 150ms for the GRC reset to settle */
+#define GRC_RESET_TIMEOUT_MS (250)     /* Wait 250ms for the GRC reset to settle */
 
 static NetworkPort_t *gPort;
 static uint32_t gResetTime;
+static uint32_t gResetDelay;
+static bool gPortReset;
 
-void handleDriverEvent(volatile SHM_t *shm)
+void triggerPendingReset(uint32_t delay)
+{
+    gPortReset = true;
+    gResetDelay = delay;
+
+    gResetTime = Timer_getCurrentTime1KHz();
+}
+
+void handleDriverEvent(volatile SHM_t *shm, int port)
 {
     RegSHMEventStatus_t event = shm->EventStatus;
 
@@ -79,10 +89,26 @@ void handleDriverEvent(volatile SHM_t *shm)
     {
         APE_aquireMemLock();
         // TODO: Handle the event from the driver.
+        switch (event.bits.Command)
+        {
+            case SHM_EVENT_STATUS_COMMAND_STATE_CHANGE:
+                printf("Driver State %d: %x\n", port, event.r32);
+                break;
+
+            case SHM_EVENT_STATUS_COMMAND_SCRATCHPAD_READ:
+            case SHM_EVENT_STATUS_COMMAND_SCRATCHPAD_WRITE:
+                // Unimplemented.
+                break;
+
+            default:
+                // Unknown command.
+                printf("Unknown APE Event %d: %x\n", port, event.r32);
+                break;
+        }
+
         shm->EventStatus.r32 = 0;
-        APE_releaseMemLock();
 
-        printf("APE Event: %x\n", event.r32);
+        APE_releaseMemLock();
     }
 }
 
@@ -281,11 +307,18 @@ void __attribute__((interrupt)) IRQ_VoltageSource()
     }
 
     // Ensure we reinitialize hardware as needed.
-    gResetTime = Timer_getCurrentTime1KHz();
-    if (!gResetTime)
+    triggerPendingReset(GRC_RESET_TIMEOUT_MS);
+}
+
+bool portResetInProgress(RegAPEStatus_t status, RegAPEStatus2_t status2, const NetworkPort_t *port)
+{
+    if ((status.r32 & port->APEStatus.r32) || (status2.r32 & port->APEStatus2.r32))
+    {
+        return true;
+    }
+    else
     {
-        // We use 0 to mean that no reset has happend. Make sure this value is never 0.
-        gResetTime--;
+        return false;
     }
 }
 
@@ -314,22 +347,16 @@ void __attribute__((interrupt)) IRQ_PowerStatusChanged(void)
 
     printf("PowerStateChanged.\n");
 
-    if (!gResetTime)
+    if (resetInProgress(status, status2))
     {
-
-        if (resetInProgress(status, status2))
+        if (portResetInProgress(status, status2, gPort))
         {
             printf("GRC Reset.\n");
-            gResetTime = Timer_getCurrentTime1KHz();
-            if (!gResetTime)
-            {
-                // We use 0 to mean that no reset has happend. Make sure this value is never 0.
-                gResetTime--;
-            }
-
-            // Disable the interrupt so that we can exit the interrupt handler
-            NVIC.InterruptClearEnable.r32 = NVIC_INTERRUPT_SET_ENABLE_SETENA_GENERAL_RESET;
+            triggerPendingReset(GRC_RESET_TIMEOUT_MS);
         }
+
+        // Disable the interrupt so that we can exit the interrupt handler
+        NVIC.InterruptClearEnable.r32 = NVIC_INTERRUPT_SET_ENABLE_SETENA_GENERAL_RESET;
     }
 }
 
@@ -358,9 +385,6 @@ void initSHM(volatile SHM_t *shm)
 
 void __attribute__((noreturn)) loaderLoop(void)
 {
-    uint32_t host_state = SHM.HostDriverState.bits.State;
-    bool reset_allowed = host_state == SHM_HOST_DRIVER_STATE_STATE_START;
-
     // Update SHM.Sig to signal ready.
     initSHM(&SHM);
     initSHM(&SHM1);
@@ -372,8 +396,9 @@ void __attribute__((noreturn)) loaderLoop(void)
 
     for (;;)
     {
-        if (gResetTime)
+        if (NVIC_INTERRUPT_SET_ENABLE_SETENA_GENERAL_RESET != (NVIC.InterruptSetEnable.r32 & NVIC_INTERRUPT_SET_ENABLE_SETENA_GENERAL_RESET))
         {
+            // A Global Reset occured, wait for it to settle.
             RegAPEStatus_t status = APE.Status;
             RegAPEStatus2_t status2 = APE.Status2;
 
@@ -385,83 +410,60 @@ void __attribute__((noreturn)) loaderLoop(void)
                 APE.Status2.r32 = status2.r32;
 
                 // Initialize timer for reset.
-                gResetTime = Timer_getCurrentTime1KHz();
-                if (!gResetTime)
+                if (portResetInProgress(status, status2, gPort))
                 {
-                    // We use 0 to mean that no reset has happend. Make sure this value is never 0.
-                    gResetTime--;
+                    if (!gPortReset)
+                    {
+                        printf("GRC Reset.\n");
+                    }
+
+                    triggerPendingReset(GRC_RESET_TIMEOUT_MS);
                 }
             }
-            else if (Timer_didTimeElapsed1KHz(gResetTime, GRC_RESET_TIMEOUT_MS))
+            else
             {
                 // We may still have an interrupt pending since we disabled the interrupt. Clear it so we don't get an extra trigger.
                 NVIC.InterruptClearPending.r32 = NVIC_INTERRUPT_CLEAR_PENDING_CLRPEND_GENERAL_RESET;
-                gResetTime = 0;
-
-                printf("Handling reset...\n");
-
-                // Perform TX reinit as the PHY / MII was also probably reset.
-                wait_for_all_rx();
-                NCSI_reload(AS_NEEDED);
 
                 // Reset complete, re-enable interrupt handler.
                 NVIC.InterruptSetEnable.r32 = NVIC_INTERRUPT_SET_ENABLE_SETENA_GENERAL_RESET;
-            }
-
-            handleBMCPacket(false);
-        }
-        else
-        {
-            Network_checkPortState(gPort);
-
-            handleBMCPacket(true);
-            NCSI_handlePassthrough();
 
-            if (host_state != SHM.HostDriverState.bits.State)
-            {
-                host_state = SHM.HostDriverState.bits.State;
-
-                if (SHM_HOST_DRIVER_STATE_STATE_START == host_state)
+                if (!Network_checkEnableState(gPort))
                 {
-                    printf("host started\n");
-
-                    reset_allowed = true;
-                }
-                else
-                {
-                    if (SHM_HOST_DRIVER_STATE_STATE_UNLOAD == host_state)
-                    {
-                        printf("host unloaded.\n");
-                    }
-                    else
-                    {
-                        printf("wol?\n");
-                    }
-
-                    reset_allowed = false;
+                    // If an unrelated port is unloaded, the APE mode may be cleared. Set it again.
+                    wait_for_all_rx();
+                    Network_setEnableState(gPort);
                 }
             }
-            else if (reset_allowed && !Network_checkEnableState(gPort) && !gResetTime)
-            {
-                printf("APE mode change, resetting.\n");
-                wait_for_all_rx();
-                NCSI_reload(AS_NEEDED);
+        }
+
+        if (gPortReset && Timer_didTimeElapsed1KHz(gResetTime, gResetDelay))
+        {
+            gPortReset = false;
 
-                // Update host state to make sure we don't reset twice if it's changed.
-                host_state = SHM.HostDriverState.bits.State;
+            printf("Handling reset...\n");
 
-                reset_allowed = false;
-            }
+            // Perform TX reinit as the PHY / MII was also probably reset.
+            wait_for_all_rx();
+            NCSI_reload(AS_NEEDED);
         }
 
+        if (!gPortReset)
+        {
+            Network_checkPortState(gPort);
+        }
+
+        handleBMCPacket((bool)!gPortReset);
+        NCSI_handlePassthrough();
+
         handleCommand(&SHM);
         handleCommand(&SHM1);
         handleCommand(&SHM2);
         handleCommand(&SHM3);
-        handleDriverEvent(&SHM);
-        handleDriverEvent(&SHM1);
-        handleDriverEvent(&SHM2);
-        handleDriverEvent(&SHM3);
+        handleDriverEvent(&SHM, 0);
+        handleDriverEvent(&SHM1, 1);
+        handleDriverEvent(&SHM2, 2);
+        handleDriverEvent(&SHM3, 3);
     }
 }
 
@@ -556,7 +558,7 @@ void __attribute__((noreturn)) __start()
     else
     {
         printf("APE Reload.\n");
-        NCSI_reload(SHM_HOST_DRIVER_STATE_STATE_START != SHM.HostDriverState.bits.State ? AS_NEEDED : NEVER_RESET);
+        NCSI_reload(SHM_HOST_DRIVER_STATE_STATE_START != gPort->shm->HostDriverState.bits.State ? AS_NEEDED : NEVER_RESET);
     }
 
     loaderLoop();
diff --git a/libs/Network/include/Network.h b/libs/Network/include/Network.h
index dadc2e8..7d5dde1 100644
--- a/libs/Network/include/Network.h
+++ b/libs/Network/include/Network.h
@@ -51,6 +51,7 @@
 #include <APE_FILTERS0.h>
 #include <APE_RX_PORT0.h>
 #include <APE_TX_PORT0.h>
+#include <APE_SHM.h>
 #include <APE_SHM_CHANNEL0.h>
 #include <types.h>
 
@@ -60,6 +61,7 @@ typedef struct
     VOLATILE DEVICE_t *device;
     VOLATILE FILTERS_t *filters;
     VOLATILE SHM_CHANNEL_t* shm_channel;
+    VOLATILE SHM_t* shm;
 
     /* TX Registers */
     VOLATILE TX_PORT_t *tx_port;
@@ -78,6 +80,8 @@ typedef struct
     /* APE Registers */
     VOLATILE RegAPEMode_t APEModeEnable;
     VOLATILE RegAPEMode2_t APEMode2Enable;
+    VOLATILE RegAPEStatus_t APEStatus;
+    VOLATILE RegAPEStatus2_t APEStatus2;
 
     /* State Trackking */
     bool link_state_printed;
diff --git a/libs/Network/ports.c b/libs/Network/ports.c
index cde3f18..bafc9bd 100644
--- a/libs/Network/ports.c
+++ b/libs/Network/ports.c
@@ -52,6 +52,10 @@
 #include <APE_RX_PORT1.h>
 #include <APE_RX_PORT2.h>
 #include <APE_RX_PORT3.h>
+#include <APE_SHM.h>
+#include <APE_SHM1.h>
+#include <APE_SHM2.h>
+#include <APE_SHM3.h>
 #include <APE_SHM_CHANNEL0.h>
 #include <APE_SHM_CHANNEL1.h>
 #include <APE_SHM_CHANNEL2.h>
@@ -72,6 +76,7 @@ NetworkPort_t gPort0 = {
     .device = &DEVICE,
     .filters = &FILTERS0,
     .shm_channel = &SHM_CHANNEL0,
+    .shm = &SHM,
 
     .tx_port = &TX_PORT0,
     .tx_allocator = &APE.TxToNetBufferAllocator0,
@@ -92,6 +97,12 @@ NetworkPort_t gPort0 = {
     .APEMode2Enable = {
         .r32 = 0,
     },
+    .APEStatus = {
+        .r32 = APE_STATUS_PORT_0_GRC_RESET_MASK,
+    },
+    .APEStatus2 = {
+        .r32 = 0,
+    },
 #endif
 };
 
@@ -99,6 +110,11 @@ NetworkPort_t gPort1 = {
     .device = &DEVICE1,
     .filters = &FILTERS1,
     .shm_channel = &SHM_CHANNEL1,
+#ifndef CXX_SIMULATOR
+    .shm = &SHM1,
+#else
+    .shm = &SHM,
+#endif
 
     .tx_port = &TX_PORT1,
     .tx_allocator = &APE.TxToNetBufferAllocator1,
@@ -119,6 +135,12 @@ NetworkPort_t gPort1 = {
     .APEMode2Enable = {
         .r32 = 0,
     },
+    .APEStatus = {
+        .r32 = APE_STATUS_PORT_1_GRC_RESET_MASK,
+    },
+    .APEStatus2 = {
+        .r32 = 0,
+    },
 #endif
 };
 
@@ -126,6 +148,11 @@ NetworkPort_t gPort2 = {
     .device = &DEVICE2,
     .filters = &FILTERS2,
     .shm_channel = &SHM_CHANNEL2,
+#ifndef CXX_SIMULATOR
+    .shm = &SHM2,
+#else
+    .shm = &SHM,
+#endif
 
     .tx_port = &TX_PORT2,
     .tx_allocator = &APE.TxToNetBufferAllocator2,
@@ -146,6 +173,12 @@ NetworkPort_t gPort2 = {
     .APEMode2Enable = {
         .r32 = APE_MODE_2_CHANNEL_0_ENABLE_MASK | APE_MODE_2_CHANNEL_2_ENABLE_MASK,
     },
+    .APEStatus = {
+        .r32 = 0,
+    },
+    .APEStatus2 = {
+        .r32 = APE_STATUS_2_PORT_2_GRC_RESET_MASK,
+    },
 #endif
 };
 
@@ -153,6 +186,11 @@ NetworkPort_t gPort3 = {
     .device = &DEVICE3,
     .filters = &FILTERS3,
     .shm_channel = &SHM_CHANNEL3,
+#ifndef CXX_SIMULATOR
+    .shm = &SHM3,
+#else
+    .shm = &SHM,
+#endif
 
     .tx_port = &TX_PORT3,
     .tx_allocator = &APE.TxToNetBufferAllocator3,
@@ -173,6 +211,12 @@ NetworkPort_t gPort3 = {
     .APEMode2Enable = {
         .r32 = APE_MODE_2_CHANNEL_1_ENABLE_MASK | APE_MODE_2_CHANNEL_3_ENABLE_MASK,
     },
+    .APEStatus = {
+        .r32 = 0,
+    },
+    .APEStatus2 = {
+        .r32 = APE_STATUS_2_PORT_3_GRC_RESET_MASK,
+    },
 #endif
 };