Check fans for failure

Add check the fan fail. If detect fan fail then into
the fail safe mode.

Signed-off-by: Will Liang <will.liang@quantatw.com>
Change-Id: I6ef9d42e131500c1b38a708e1c6fda15dc712f60

2 files changed, 141 insertions, 1 deletions

diff --git a/pid/zone.cpp b/pid/zone.cpp
index eef4fde..e206e17 100644
--- a/pid/zone.cpp
+++ b/pid/zone.cpp
@@ -217,9 +217,9 @@ void PIDZone::updateFanTelemetry(void)
      * is disabled?  I think it's a waste to try and log things even if the
      * data is just being dropped though.
      */
+    tstamp now = std::chrono::high_resolution_clock::now();
     if (loggingEnabled)
     {
-        tstamp now = std::chrono::high_resolution_clock::now();
         _log << std::chrono::duration_cast<std::chrono::milliseconds>(
                     now.time_since_epoch())
                     .count();
@@ -231,7 +231,13 @@ void PIDZone::updateFanTelemetry(void)
         auto sensor = _mgr.getSensor(f);
         ReadReturn r = sensor->read();
         _cachedValuesByName[f] = r.value;
+        int64_t timeout = sensor->getTimeout();
+        tstamp then = r.updated;
 
+        auto duration =
+            std::chrono::duration_cast<std::chrono::seconds>(now - then)
+                .count();
+        auto period = std::chrono::seconds(timeout).count();
         /*
          * TODO(venture): We should check when these were last read.
          * However, these are the fans, so if I'm not getting updated values
@@ -241,6 +247,25 @@ void PIDZone::updateFanTelemetry(void)
         {
             _log << "," << r.value;
         }
+
+        // check if fan fail.
+        if (sensor->getFailed())
+        {
+            _failSafeSensors.insert(f);
+        }
+        else if (timeout != 0 && duration >= period)
+        {
+            _failSafeSensors.insert(f);
+        }
+        else
+        {
+            // Check if it's in there: remove it.
+            auto kt = _failSafeSensors.find(f);
+            if (kt != _failSafeSensors.end())
+            {
+                _failSafeSensors.erase(kt);
+            }
+        }
     }
 
     if (loggingEnabled)
@@ -300,6 +325,9 @@ void PIDZone::initializeCache(void)
     for (const auto& f : _fanInputs)
     {
         _cachedValuesByName[f] = 0;
+
+        // Start all fans in fail-safe mode.
+        _failSafeSensors.insert(f);
     }
 
     for (const auto& t : _thermalInputs)
diff --git a/test/pid_zone_unittest.cpp b/test/pid_zone_unittest.cpp
index 11ba4c1..269b3f8 100644
--- a/test/pid_zone_unittest.cpp
+++ b/test/pid_zone_unittest.cpp
@@ -328,6 +328,118 @@ TEST_F(PidZoneTest, ThermalInput_ValueTimeoutEntersFailSafeMode)
     EXPECT_TRUE(zone->getFailSafeMode());
 }
 
+TEST_F(PidZoneTest, FanInputTest_FailsafeToValid_ReadsSensors)
+{
+    // This will add a couple fan inputs, and verify the values are cached.
+
+    std::string name1 = "fan1";
+    int64_t timeout = 2;
+
+    std::unique_ptr<Sensor> sensor1 =
+        std::make_unique<SensorMock>(name1, timeout);
+    SensorMock* sensor_ptr1 = reinterpret_cast<SensorMock*>(sensor1.get());
+
+    std::string name2 = "fan2";
+    std::unique_ptr<Sensor> sensor2 =
+        std::make_unique<SensorMock>(name2, timeout);
+    SensorMock* sensor_ptr2 = reinterpret_cast<SensorMock*>(sensor2.get());
+
+    std::string type = "unchecked";
+    mgr.addSensor(type, name1, std::move(sensor1));
+    EXPECT_EQ(mgr.getSensor(name1), sensor_ptr1);
+    mgr.addSensor(type, name2, std::move(sensor2));
+    EXPECT_EQ(mgr.getSensor(name2), sensor_ptr2);
+
+    // Now that the sensors exist, add them to the zone.
+    zone->addFanInput(name1);
+    zone->addFanInput(name2);
+
+    // Initialize Zone
+    zone->initializeCache();
+
+    // Verify now in failsafe mode.
+    EXPECT_TRUE(zone->getFailSafeMode());
+
+    ReadReturn r1;
+    r1.value = 10.0;
+    r1.updated = std::chrono::high_resolution_clock::now();
+    EXPECT_CALL(*sensor_ptr1, read()).WillOnce(Return(r1));
+
+    ReadReturn r2;
+    r2.value = 11.0;
+    r2.updated = std::chrono::high_resolution_clock::now();
+    EXPECT_CALL(*sensor_ptr2, read()).WillOnce(Return(r2));
+
+    // Method under test will read through each fan sensor for the zone and
+    // cache the values.
+    zone->updateFanTelemetry();
+
+    // We should no longer be in failsafe mode.
+    EXPECT_FALSE(zone->getFailSafeMode());
+
+    EXPECT_EQ(r1.value, zone->getCachedValue(name1));
+    EXPECT_EQ(r2.value, zone->getCachedValue(name2));
+}
+
+TEST_F(PidZoneTest, FanInputTest_ValueTimeoutEntersFailSafeMode)
+{
+    // This will add a couple fan inputs, and verify the values are cached.
+
+    std::string name1 = "fan1";
+    int64_t timeout = 2;
+
+    std::unique_ptr<Sensor> sensor1 =
+        std::make_unique<SensorMock>(name1, timeout);
+    SensorMock* sensor_ptr1 = reinterpret_cast<SensorMock*>(sensor1.get());
+
+    std::string name2 = "fan2";
+    std::unique_ptr<Sensor> sensor2 =
+        std::make_unique<SensorMock>(name2, timeout);
+    SensorMock* sensor_ptr2 = reinterpret_cast<SensorMock*>(sensor2.get());
+
+    std::string type = "unchecked";
+    mgr.addSensor(type, name1, std::move(sensor1));
+    EXPECT_EQ(mgr.getSensor(name1), sensor_ptr1);
+    mgr.addSensor(type, name2, std::move(sensor2));
+    EXPECT_EQ(mgr.getSensor(name2), sensor_ptr2);
+
+    // Now that the sensors exist, add them to the zone.
+    zone->addFanInput(name1);
+    zone->addFanInput(name2);
+
+    // Initialize Zone
+    zone->initializeCache();
+
+    // Verify now in failsafe mode.
+    EXPECT_TRUE(zone->getFailSafeMode());
+
+    ReadReturn r1;
+    r1.value = 10.0;
+    r1.updated = std::chrono::high_resolution_clock::now();
+    EXPECT_CALL(*sensor_ptr1, read()).WillOnce(Return(r1));
+
+    ReadReturn r2;
+    r2.value = 11.0;
+    r2.updated = std::chrono::high_resolution_clock::now();
+    EXPECT_CALL(*sensor_ptr2, read()).WillOnce(Return(r2));
+
+    // Method under test will read through each fan sensor for the zone and
+    // cache the values.
+    zone->updateFanTelemetry();
+
+    // We should no longer be in failsafe mode.
+    EXPECT_FALSE(zone->getFailSafeMode());
+
+    r1.updated -= std::chrono::seconds(3);
+    r2.updated = std::chrono::high_resolution_clock::now();
+
+    EXPECT_CALL(*sensor_ptr1, read()).WillOnce(Return(r1));
+    EXPECT_CALL(*sensor_ptr2, read()).WillOnce(Return(r2));
+
+    zone->updateFanTelemetry();
+    EXPECT_TRUE(zone->getFailSafeMode());
+}
+
 TEST_F(PidZoneTest, GetSensorTest_ReturnsExpected)
 {
     // One can grab a sensor from the manager through the zone.