Backport IPL observer and LED handler changes04-13-2018

2 files changed, 159 insertions, 146 deletions

diff --git a/pyiplledmonitor/ipl_status_led_monitor.py b/pyiplledmonitor/ipl_status_led_monitor.py
index be4b784..6474832 100644
--- a/pyiplledmonitor/ipl_status_led_monitor.py
+++ b/pyiplledmonitor/ipl_status_led_monitor.py
@@ -21,6 +21,7 @@ import obmc.system
 
 DBUS_NAME = 'org.openbmc.status.IPL'
 OBJ_NAME = u'/org/openbmc/status/IPL'
+LEDOBJ_NAME = u'/org/openbmc/status/IPLLED'
 
 class IPLStatusLEDMonitor(DbusProperties, DbusObjectManager):
     def __init__(self, bus, obj_name):
@@ -221,10 +222,9 @@ class IPLStatusLEDMonitor(DbusProperties, DbusObjectManager):
 if __name__ == '__main__':
     dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
     bus = get_dbus()
-    obj = IPLStatusLEDMonitor(bus, OBJ_NAME)
+    obj = IPLStatusLEDMonitor(bus, LEDOBJ_NAME)
     mainloop = gobject.MainLoop()
     obj.unmask_signals()
-    name = dbus.service.BusName(DBUS_NAME, bus)
 
     print "Running IPLStatusLEDMonitor"
     mainloop.run()
diff --git a/pyiplobserver/ipl_status_observer.py b/pyiplobserver/ipl_status_observer.py
index cc12511..095a3e7 100644..100755
--- a/pyiplobserver/ipl_status_observer.py
+++ b/pyiplobserver/ipl_status_observer.py
@@ -12,9 +12,14 @@ import gobject
 import dbus
 import dbus.service
 import dbus.mainloop.glib
+import re
 import os
 import subprocess
+import time
+import subprocess
 import traceback
+from fcntl import fcntl, F_GETFL, F_SETFL
+from os import O_NONBLOCK, read
 import obmc.dbuslib.propertycacher as PropertyCacher
 from obmc.dbuslib.bindings import DbusProperties, DbusObjectManager, get_dbus
 import obmc.enums
@@ -31,6 +36,9 @@ IPL_FLAG_FILE = 'host@0-ipl-complete'
 
 HOST_STATUS_RUNNING = "xyz.openbmc_project.State.Host.HostState.Running"
 
+IPL_SBE_START_ISTEP_MAJOR = 0
+IPL_SBE_START_ISTEP_MINOR = 1
+
 IPL_SBE_DONE_ISTEP_MAJOR = 5
 IPL_SBE_DONE_ISTEP_MINOR = 2
 
@@ -55,10 +63,15 @@ class IPLStatus(DbusProperties, DbusObjectManager):
             conn=bus,
             object_path=obj_name)
         self.bus = bus
-        self.prev_access_fail_count = 0;
+        self.prev_access_fail_count = 0
         self.activateMonitoring = False
-        self.last_status = "";
-        self.last_istep = "";
+        self.activateMonitoringPrev = False
+        self.last_status = ""
+        self.last_istep = ""
+        self.current_istep_major = 0
+        self.current_istep_minor = 0
+        self.consoleProcess = None
+        self.consoleReadBuffer = ""
 
         if not os.path.exists(IPL_FLAG_DIR):
             os.makedirs(IPL_FLAG_DIR)
@@ -68,6 +81,11 @@ class IPLStatus(DbusProperties, DbusObjectManager):
         host_state_value = host_state_iface.Get("xyz.openbmc_project.State.Host", "CurrentHostState")
         if host_state_value is not None:
             if (host_state_value == HOST_STATUS_RUNNING):
+                skiboot_flags = self.getSkibootBootFlags()
+                if (skiboot_flags != 0):
+                    self.current_istep_major = IPL_COMPLETE_ISTEP_MAJOR
+                    self.current_istep_minor = IPL_COMPLETE_ISTEP_MINOR
+                self.activateMonitoringPrev = True
                 self.activateMonitoring = True
 
         self.Set(DBUS_NAME, "current_istep", "")
@@ -114,14 +132,8 @@ class IPLStatus(DbusProperties, DbusObjectManager):
             # Set up defaults
             current_status = "OFFLINE"
             current_istep = ""
-            sbe_fetch_failed = False
-            hostboot_fetch_failed = False
             sbe_istep_major = 0
             sbe_istep_minor = 0
-            hostboot_istep_major = 0
-            hostboot_istep_minor = 0
-            current_istep_major = 0
-            current_istep_minor = 0
 
             if (self.activateMonitoring == False):
                 # Publish offline status
@@ -136,153 +148,140 @@ class IPLStatus(DbusProperties, DbusObjectManager):
                     self.last_istep = current_istep
                     self.setSkibootBootFlags(0)
 
+                self.activateMonitoringPrev = self.activateMonitoring
                 return True;
 
-            # Get SBE status
-            proc = subprocess.Popen(["pdbg", "-b", "kernel", "-p0", "getcfam", "0x2809"], stdout=subprocess.PIPE, stderr=open(os.devnull, 'wb'), shell=False)
-            (out, err) = proc.communicate()
-            status_data = out.split("=")
-            if (len(status_data) > 1):
-                try:
-                    binary_status_data = bin(int(status_data[1], 16))
-                except ValueError:
-                    sbe_fetch_failed = True
-
-                if (sbe_fetch_failed == False):
+            if (self.activateMonitoring == True) and (self.activateMonitoringPrev == False):
+                if (self.consoleProcess is None):
+                    self.consoleProcess = subprocess.Popen(["/usr/bin/obmc-console-client"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
+                    # set the O_NONBLOCK flag of self.consoleProcess.stdout file descriptor:
+                    stdout_flags = fcntl(self.consoleProcess.stdout, F_GETFL) # get current p.stdout flags
+                    fcntl(self.consoleProcess.stdout, F_SETFL, stdout_flags | O_NONBLOCK)
+                current_status = "IPL_RUNNING"
+                self.current_istep_major = IPL_SBE_START_ISTEP_MAJOR
+                self.current_istep_minor = IPL_SBE_START_ISTEP_MINOR
+                current_istep = str(self.current_istep_major) + "," + str(self.current_istep_minor)
+                self.Set(DBUS_NAME, "current_istep", current_istep)
+                self.IPLStepChanged(current_istep)
+                self.last_istep = current_istep
+            elif (self.activateMonitoring == False) and (self.activateMonitoringPrev == True):
+                if (self.consoleProcess is not None):
+                    self.consoleProcess.kill()
+                    self.consoleProcess = None
+                    self.consoleReadBuffer = ""
+                    self.current_istep_major = 0
+                    self.current_istep_minor = 0
+
+            # Check for output
+            if (self.consoleProcess is not None):
+                all_lines_processed = False
+                while all_lines_processed == False:
+                    candidate_character = ''
                     try:
-                        sbe_istep_major = int(binary_status_data[14:22], 2)
-                        sbe_istep_minor = int(binary_status_data[22:28], 2)
-                        sbe_istep = str(sbe_istep_major) + "," + str(sbe_istep_minor)
-                    except ValueError:
-                        sbe_fetch_failed = True
-            else:
-                sbe_fetch_failed = True
-
-            # Determine if hostboot should be checked for activity
-            hostboot_active = False
-            if (sbe_fetch_failed):
-                # Check if hostboot is running
-                hostboot_active = True
-            if ((sbe_istep_major == IPL_SBE_DONE_ISTEP_MAJOR) and (sbe_istep_minor == IPL_SBE_DONE_ISTEP_MINOR)):
-                # Hostboot should definitely be running
-                hostboot_active = True
-
-            # Get hostboot status
-            if (hostboot_active == False):
-                if (sbe_fetch_failed == False):
-                    hostboot_istep_major = sbe_istep_major
-                    hostboot_istep_minor = sbe_istep_minor
-                    hostboot_istep = sbe_istep
-            else:
-                proc = subprocess.Popen(["pdbg", "-b", "kernel", "-p0", "getcfam", "0x283c"], stdout=subprocess.PIPE, stderr=open(os.devnull, 'wb'), shell=False)
-                (out, err) = proc.communicate()
-                status_data = out.split("=")
-                if (len(status_data) > 1):
-                    try:
-                        binary_status_data = bin(int(status_data[1], 16))
-                    except ValueError:
-                        hostboot_fetch_failed = True
-
-                    if (hostboot_fetch_failed == False):
+                        while True:
+                            candidate_character = self.consoleProcess.stdout.read(1)
+                            self.consoleReadBuffer += candidate_character
+                            if (candidate_character == '\n'):
+                                break;
+                    except:
+                        # ENODATA is expected
+                        all_lines_processed = True
+                        break;
+
+                    self.consoleReadBuffer += candidate_character
+                    if (candidate_character != '\n'):
+                        return True
+
+                    candidate_line = self.consoleReadBuffer.decode('utf-8', 'ignore').rstrip().lstrip()
+                    self.consoleReadBuffer = ""
+
+                    if ("|ISTEP " in candidate_line):
+                        candidate_line = candidate_line.replace(" ", "")
                         try:
-                            hostboot_istep_magic = int(binary_status_data[2:10], 2)
-                            hostboot_istep_major = int(binary_status_data[18:26], 2)
-                            hostboot_istep_minor = int(binary_status_data[26:34], 2)
-                        except ValueError:
-                            hostboot_fetch_failed = True
-                        if (hostboot_fetch_failed == False):
-                            if (hostboot_istep_magic != 170):
-                                hostboot_istep_major = 0
-                                hostboot_istep_minor = 0
-                            hostboot_istep = str(hostboot_istep_major) + "," + str(hostboot_istep_minor)
-                else:
-                    hostboot_fetch_failed = True
-
-            if ((sbe_fetch_failed == False) and (hostboot_fetch_failed == False)):
-                # Figure out system status
-                current_istep_major = sbe_istep_major
-                current_istep_minor = sbe_istep_minor
-                if (hostboot_istep_major > current_istep_major):
-                    current_istep_major = hostboot_istep_major
-                if (hostboot_istep_minor > current_istep_minor):
-                    current_istep_minor = hostboot_istep_minor
-                current_istep = str(current_istep_major) + "," + str(current_istep_minor)
-
-                # Asseble status strings
-                current_status = "OFFLINE"
-                if ((current_istep_major > 0) or (current_istep_minor > 0 )):
-                    current_status = "IPL_RUNNING"
-                if (current_istep_major >= IPL_COMPLETE_ISTEP_MAJOR):
-                    skiboot_flags = self.getSkibootBootFlags()
-                    if (skiboot_flags == 0):
-                        current_status = "IPL_RUNNING"
-                        current_istep_major = SKIBOOT_RUNNING_ISTEP_MAJOR
-                        current_istep_minor = SKIBOOT_RUNNING_ISTEP_MINOR
-                        current_istep = str(current_istep_major) + "," + str(current_istep_minor)
-                    elif (skiboot_flags == 1):
-                        current_status = "IPL_RUNNING"
-                        current_istep_major = KERNEL_RUNNING_ISTEP_MAJOR
-                        current_istep_minor = KERNEL_RUNNING_ISTEP_MINOR
-                        current_istep = str(current_istep_major) + "," + str(current_istep_minor)
+                            (timestamp_seconds, timestamp_fraction, istep_major, istep_minor, istep_name) = [t(s) for t, s in zip((int, int, int, int, str), re.search('^(\d+).(\d+)\|ISTEP(\d+).(\d+)-(\S+)$', candidate_line).groups())]
+                        except:
+                            self.activateMonitoringPrev = self.activateMonitoring
+                            return True
                     else:
-                        current_status = "IPL_COMPLETE"
-                        current_istep_major = KERNEL_COMPLETE_ISTEP_MAJOR
-                        current_istep_minor = KERNEL_COMPLETE_ISTEP_MINOR
-                        current_istep = str(current_istep_major) + "," + str(current_istep_minor)
-
-                # Reset failure counter
-                self.prev_access_fail_count = 0
-            else:
-                # Increment failure counter
-                if (self.prev_access_fail_count < 255):
-                    self.prev_access_fail_count += 1
-
-            # Since it is extremely unlikely that both fetches will fail unless the host is offline, simply check that both fetches passed or failed before continuing
-            if (sbe_fetch_failed == hostboot_fetch_failed):
-                # Check error results multiple times before signalling unexpected offline status
-                if ((current_status == "OFFLINE") and (self.prev_access_fail_count < 10)):
-                    return True
+                        self.activateMonitoringPrev = self.activateMonitoring
+                        return True
 
-                # Check for status changes
-                if (self.last_status != current_status):
-                    if (current_status == "IPL_COMPLETE"):
-                        # Set skiboot completion flag
-                        self.setSkibootBootFlags(2)
+                    # Figure out system status
+                    self.current_istep_major = istep_major
+                    self.current_istep_minor = istep_minor
 
-                        # Signal systemd that IPL is complete
-                        try:
-                            with open(IPL_FLAG_DIR + IPL_FLAG_FILE, 'a'):
-                                os.utime(IPL_FLAG_DIR + IPL_FLAG_FILE, None)
-
-                            systemd_emit_dev = self.bus.get_object("org.freedesktop.systemd1", u"/org/freedesktop/systemd1")
-                            systemd_emit_iface = dbus.Interface(systemd_emit_dev, 'org.freedesktop.systemd1.Manager')
-                            systemd_emit_function = getattr(systemd_emit_iface, 'StartUnit')
-                            systemd_emit_function.call_async('obmc-host-ipl-complete@0.target', 'replace')
-                        except Exception as e:
-                            pass
-                    else:
-                        try:
-                            os.remove(IPL_FLAG_DIR + IPL_FLAG_FILE)
-                        except:
-                            pass
-                if (current_status == "IPL_RUNNING"):
-                    if (current_istep_major < IPL_COMPLETE_ISTEP_MAJOR):
-                        self.setSkibootBootFlags(0)
-
-                # Publish status
-                if (self.last_status != current_status):
-                    self.Set(DBUS_NAME, "current_status", current_status)
-                    self.IPLStatusChanged(current_status)
-                    self.last_status = current_status
-                if (self.last_istep != current_istep):
-                    self.Set(DBUS_NAME, "current_istep", current_istep)
-                    self.IPLStepChanged(current_istep)
-                    self.last_istep = current_istep
+            # Assemble status strings
+            current_istep = str(self.current_istep_major) + "," + str(self.current_istep_minor)
+            current_status = "OFFLINE"
+            if ((self.current_istep_major > 0) or (self.current_istep_minor > 0 )):
+                current_status = "IPL_RUNNING"
+            if (self.current_istep_major >= IPL_COMPLETE_ISTEP_MAJOR):
+                 # Disconnect console if still active
+                if (self.consoleProcess is not None):
+                    self.consoleProcess.kill()
+                    self.consoleProcess = None
+                    self.consoleReadBuffer = ""
+
+                # Monitor remainder of IPL via status flags
+                skiboot_flags = self.getSkibootBootFlags()
+                if (skiboot_flags == 0):
+                    current_status = "IPL_RUNNING"
+                    self.current_istep_major = SKIBOOT_RUNNING_ISTEP_MAJOR
+                    self.current_istep_minor = SKIBOOT_RUNNING_ISTEP_MINOR
+                    current_istep = str(self.current_istep_major) + "," + str(self.current_istep_minor)
+                elif (skiboot_flags == 1):
+                    current_status = "IPL_RUNNING"
+                    self.current_istep_major = KERNEL_RUNNING_ISTEP_MAJOR
+                    self.current_istep_minor = KERNEL_RUNNING_ISTEP_MINOR
+                    current_istep = str(self.current_istep_major) + "," + str(self.current_istep_minor)
+                else:
+                    current_status = "IPL_COMPLETE"
+                    self.current_istep_major = KERNEL_COMPLETE_ISTEP_MAJOR
+                    self.current_istep_minor = KERNEL_COMPLETE_ISTEP_MINOR
+                    current_istep = str(self.current_istep_major) + "," + str(self.current_istep_minor)
+
+            # Check for status changes
+            if (self.last_status != current_status):
+                if (current_status == "IPL_COMPLETE"):
+                    # Set skiboot completion flag
+                    self.setSkibootBootFlags(2)
+
+                    # Signal systemd that IPL is complete
+                    try:
+                        with open(IPL_FLAG_DIR + IPL_FLAG_FILE, 'a'):
+                            os.utime(IPL_FLAG_DIR + IPL_FLAG_FILE, None)
+
+                        systemd_emit_dev = self.bus.get_object("org.freedesktop.systemd1", u"/org/freedesktop/systemd1")
+                        systemd_emit_iface = dbus.Interface(systemd_emit_dev, 'org.freedesktop.systemd1.Manager')
+                        systemd_emit_function = getattr(systemd_emit_iface, 'StartUnit')
+                        systemd_emit_function.call_async('obmc-host-ipl-complete@0.target', 'replace')
+                    except Exception as e:
+                        pass
+                else:
+                    try:
+                        os.remove(IPL_FLAG_DIR + IPL_FLAG_FILE)
+                    except:
+                        pass
+            if (current_status == "IPL_RUNNING"):
+                if (self.current_istep_major < IPL_COMPLETE_ISTEP_MAJOR):
+                    self.setSkibootBootFlags(0)
 
+            # Publish status
+            if (self.last_status != current_status):
+                self.Set(DBUS_NAME, "current_status", current_status)
+                self.IPLStatusChanged(current_status)
+                self.last_status = current_status
+            if (self.last_istep != current_istep):
+                self.Set(DBUS_NAME, "current_istep", current_istep)
+                self.IPLStepChanged(current_istep)
+                self.last_istep = current_istep
+
+            self.activateMonitoringPrev = self.activateMonitoring
             return True
         except Exception as e:
             print(e)
             traceback.print_exc()
+            self.activateMonitoringPrev = self.activateMonitoring
             return True
 
     def HostStatusChangeHandler(self, interface_name, changed_properties,
@@ -292,6 +291,20 @@ class IPLStatus(DbusProperties, DbusObjectManager):
             if (value == HOST_STATUS_RUNNING):
                 self.activateMonitoring = True
             else:
+                # Disconnect console if still active
+                if (self.consoleProcess is not None):
+                    self.consoleProcess.kill()
+                    self.consoleProcess = None
+                    self.consoleReadBuffer = ""
+                    self.current_istep_major = 0
+                    self.current_istep_minor = 0
+
+                # Remove IPL complete flag if present
+                try:
+                    os.remove(IPL_FLAG_DIR + IPL_FLAG_FILE)
+                except:
+                    pass
+
                 self.setSkibootBootFlags(0)
                 self.activateMonitoring = False