Cache objdump output for build performance

Hostboot uses the objdump tool to disassemble modules and the genlist
program to create the listing files (*.list.bz2) for each binary blob
generated by the build process (hbibl, hbicore, etc). The hundreds of
objdump invocations account for a majority of the build time when
recompiling hostboot after a small number of files have been changed
because all the shared objects for a binary must be dumped again
regardless of whether they have been modified.

This commit causes the genlist program and the makefiles to cache the
(compressed) disassembly when the BUILD_FAST flag is set so that only
modified ELF files need to be redumped. Also several binaries include
the same modules, which means that a given module may be dumped
multiple times needlessly.

Benchmarks for recompilation after modifying small numbers of files
show at least a 5x speedup as compared with the original build system.

Using BUILD_FAST comes with a cost of about 35 MB of disk space. When
compiling all of hostboot (i.e. all libraries must be dumped), using
BUILD_FAST is not slower than a vanilla build. The cached objdump
output is stored in the img/objdump/ folder.

Example:

make BUILD_FAST=1 -j32

Change-Id: I65be7b29cec953950bfa664a5ee04e3a711007b5
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/80115
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Luis P Fernandez <luis.fernandez@ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Nicholas E Bofferding <bofferdn@us.ibm.com>
Reviewed-by: Daniel M Crowell <dcrowell@us.ibm.com>

2 files changed, 96 insertions, 14 deletions

diff --git a/src/build/linker/genlist.C b/src/build/linker/genlist.C
index 053f7aaa2..d89fbfaa6 100644
--- a/src/build/linker/genlist.C
+++ b/src/build/linker/genlist.C
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2013,2015                        */
+/* Contributors Listed Below - COPYRIGHT 2013,2019                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -95,6 +95,19 @@ string g_imageName;
      *  call binutils tools. */
 char* g_crossPrefix = NULL;
 
+    /** Whether we should use cached objdump files (true) or redump the
+        ELF objects (false). */
+bool use_cached_objdump = false;
+
+    /** Print status messages */
+bool verbose = false;
+
+#define VFPRINTF(...)                           \
+    do {                                        \
+        if (verbose)                             \
+            fprintf(__VA_ARGS__);               \
+    } while (0)
+
 int main(int argc, char** argv)
 {
     // Only parameter allowed is the name of the base image.
@@ -103,6 +116,16 @@ int main(int argc, char** argv)
         print_usage();
     }
 
+    verbose = getenv("BUILD_VERBOSE") != NULL;
+
+    use_cached_objdump = getenv("BUILD_FAST") != NULL;
+
+    if (use_cached_objdump) {
+        VFPRINTF(stderr, "Using cached OBJDUMP output\n");
+    } else {
+        VFPRINTF(stderr, "Using fresh OBJDUMP output\n");
+    }
+
     // Get base image name from parameters.
     g_imageName = argv[1];
     add_image_subdir(g_imageName);
@@ -310,18 +333,42 @@ void* read_module_content(void* input)
     // Assumes they are in the same subdirectory.
     string module_path = g_imageName.substr(0, g_imageName.rfind('/') + 1) +
                          module;
+    const string objdump_path =
+        g_imageName.substr(0, g_imageName.rfind('/') + 1) + "/objdump/" + module;
+
+    FILE* pipe = NULL;
 
-    // Create the 'objdump' command for finding all the symbols and start as
-    // a sub-process.
-    //    -d - Disassemble sections containing code.
-    //    -C - Intersparse C code.
-    //    -S - Demangle symbol names.
-    //    -j .text, .data, .rodata - Only dump those 3 sections.
-    string command = string(g_crossPrefix) +
-                     string("objdump -dCS -j .text -j .data -j .rodata ") +
-                     module_path;
-    FILE* pipe = popen(command.c_str(), "r");
-    if (NULL == pipe) return NULL;
+    VFPRINTF(stderr, "GENLIST: Processing %s\n", module_path.c_str());
+
+    if (use_cached_objdump) {
+        string bzcat_command = "bzcat " + objdump_path + ".objdump";
+        pipe = popen(bzcat_command.c_str(), "r");
+
+        if (NULL == pipe) {
+            VFPRINTF(stderr, "GENLIST: Failed to run %s, falling back to live dump\n",
+                     bzcat_command.c_str());
+        }
+    }
+
+    if (pipe == NULL) {
+        // Create the 'objdump' command for finding all the symbols and start as
+        // a sub-process.
+        //    -d - Disassemble sections containing code.
+        //    -C - Intersparse C code.
+        //    -S - Demangle symbol names.
+        //    -j .text, .data, .rodata - Only dump those 3 sections.
+        string objdump_command = string(g_crossPrefix) +
+            string("objdump -dCS -j .text -j .data -j .rodata ") +
+            module_path;
+
+        pipe = popen(objdump_command.c_str(), "r");
+
+        if (NULL == pipe) {
+            fprintf(stderr, "GENLIST: Failed to open pipe for objdump: %s\n",
+                    objdump_command.c_str());
+            return NULL;
+        }
+    }
 
     // Start result string and add module start header.
     string result;
diff --git a/src/build/mkrules/images.rules.mk b/src/build/mkrules/images.rules.mk
index 2a611b030..c6b863d89 100644
--- a/src/build/mkrules/images.rules.mk
+++ b/src/build/mkrules/images.rules.mk
@@ -5,7 +5,7 @@
 #
 # OpenPOWER HostBoot Project
 #
-# Contributors Listed Below - COPYRIGHT 2013,2017
+# Contributors Listed Below - COPYRIGHT 2013,2019
 # [+] International Business Machines Corp.
 #
 #
@@ -27,14 +27,49 @@
 # Description:
 #     Rules for linking the Hostboot binary images using the custom linker.
 
+# Folder to store *.objdump files in
+OBJDUMP_FOLDER := $(IMGDIR)/objdump
+
+# Clean up after ourselves
+clean: clean-objdump
+
+.PHONY: clean-objdump
+clean-objdump:
+	$(C2) "    MAKE       objdump CLEAN"
+	$(C1)rm -rf $(OBJDUMP_FOLDER)
+
 ifdef IMGS
 _IMGS = $(addprefix $(IMGDIR)/, $(IMGS))
 IMAGES += $(addsuffix .bin, $(_IMGS)) $(addsuffix .elf, $(_IMGS))
 
+ifdef BUILD_FAST
+OBJDUMP_MODULES := $(sort $(foreach img, $(IMGS), $($(img)_MODULES) $($(img)_EXTENDED_MODULES)))
+
+OBJDUMP_BIN_TARGETS := $(addsuffix .elf.objdump, $(IMGS))
+OBJDUMP_LIB_TARGETS := $(addsuffix .so.objdump, $(OBJDUMP_MODULES))
+
+OBJDUMP_TARGETS := $(addprefix $(OBJDUMP_FOLDER)/lib, $(OBJDUMP_LIB_TARGETS))
+OBJDUMP_TARGETS += $(addprefix $(OBJDUMP_FOLDER)/, $(OBJDUMP_BIN_TARGETS))
+
+# Tell make not to delete our objdumps
+.SECONDARY: $(OBJDUMP_TARGETS)
+endif
+
 IMAGE_PASS_POST += $(addsuffix .list.bz2, $(_IMGS)) $(addsuffix .syms, $(_IMGS))
 CLEAN_TARGETS += $(addsuffix .list.bz2, $(_IMGS)) $(addsuffix .syms, $(_IMGS))
 CLEAN_TARGETS += $(addsuffix .lnkout.bz2, $(addprefix $(IMGDIR)/., $(IMGS)))
 
+$(OBJDUMP_FOLDER):
+	mkdir -p $@
+
+$(OBJDUMP_FOLDER)/%.so.objdump: $(IMGDIR)/%.so $(OBJDUMP_FOLDER)
+	$(C2) "    OBJDUMP    $(notdir $*)"
+	$(C1)$(OBJDUMP) -dCS -j .text -j .data -j .rodata $< | bzip2 >$@
+
+$(OBJDUMP_FOLDER)/%.elf.objdump: $(IMGDIR)/%.elf $(OBJDUMP_FOLDER)
+	$(C2) "    OBJDUMP    $(notdir $*)"
+	$(C1)$(OBJDUMP) -dCS -j .text -j .data -j .rodata $< | bzip2 >$@
+
 define ELF_template
 $$(IMGDIR)/$(1).elf: $$(addprefix $$(OBJDIR)/, $$($(1)_OBJECTS)) \
                      $$(ROOTPATH)/src/$$($(1)_LDFILE)
@@ -63,7 +98,7 @@ $(IMGDIR)/%.bin: $(IMGDIR)/%.elf \
         | bzip2 -zc > $(IMGDIR)/.$*.lnkout.bz2'
 	$(C1)$(ROOTPATH)/src/build/tools/addimgid $@ $<
 
-$(IMGDIR)/%.list.bz2 $(IMGDIR)/%.syms: $(IMGDIR)/%.bin
+$(IMGDIR)/%.list.bz2 $(IMGDIR)/%.syms: $(IMGDIR)/%.bin $(OBJDUMP_TARGETS)
 	$(C2) "    GENLIST    $(notdir $*)"
 	$(C1)(cd $(ROOTPATH)&& \
               src/build/linker/gensyms $*.bin \