Developer Improvement: Get code coverage tool working with Hostboot

This commit fixes GCOV code coverage for P9 with GCC 4.9.2

Change-Id: Ie1e7c35f67414531dbd6e7a771ac1529a9ebd59d
RTC: 208351
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/76812
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Reviewed-by: Ilya Smirnov <ismirno@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

20 files changed, 840 insertions, 207 deletions

diff --git a/.gitignore b/.gitignore
index 52c97e3b2..dbd0c53e8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,5 @@ hbsandboxrc
 *.__afs*
 *~
 _SYNCAPP/
+gcov_report
+lcov_data
diff --git a/makefile b/makefile
index 16217fd5d..fc344b76c 100644
--- a/makefile
+++ b/makefile
@@ -56,14 +56,28 @@ docs: src/build/doxygen/doxygen.conf
 citest:
 	src/build/citest/cxxtest-start.sh
 
+gcov: HOSTBOOT_PROFILE := 1
+
+export HOSTBOOT_PROFILE
+
 .PHONY: gcov
 gcov:
-	rm -rf obj/gcov/*
-	$(MAKE) gcov_pass
-	find obj/gcov/ -size 0c | xargs rm # Delete empty files.
-	genhtml obj/gcov/*.lcov -o obj/gcov/html --prefix `pwd` \
-	    --title `git describe --dirty`
-	@echo "View GCOV results with: firefox obj/gcov/html/index.html"
+	@echo Building Hostboot with profiling enabled.
+	$(MAKE)
+	@echo Run simics and execute the hb-Gcov command at the end of the simulation to extract gcov data.
+	@echo Then you can "make lcov" to generate the coverage report.
+
+.PHONY: lcov
+lcov:
+	rm -f obj/lcov_data
+	lcov -c --dir . -o obj/lcov_data --gcov-tool $(GCOV)
+	rm -rf obj/gcov_report
+	genhtml obj/lcov_data -o obj/gcov_report --ignore-errors source
+	@echo Coverage report now available in obj/gcov_report
+
+.PHONY: gcda_clean
+gcda_clean:
+	find -name '*.gcda' -exec rm -f {} \;
 
 $(GENDIR)/hwp_id.html :
 	$(ROOTPATH)/src/build/tools/hwp_id.pl -i -l > $@
diff --git a/src/bootloader/makefile b/src/bootloader/makefile
index 1ff44222c..6d49d2bc1 100644
--- a/src/bootloader/makefile
+++ b/src/bootloader/makefile
@@ -5,7 +5,7 @@
 #
 # OpenPOWER HostBoot Project
 #
-# Contributors Listed Below - COPYRIGHT 2015,2017
+# Contributors Listed Below - COPYRIGHT 2015,2019
 # [+] International Business Machines Corp.
 #
 #
@@ -24,6 +24,9 @@
 # IBM_PROLOG_END_TAG
 ROOTPATH = ../..
 
+# Bootloader omitted from profiling to keep size down
+HOSTBOOT_PROFILE=
+
 EXTRAINCDIR += ${ROOTPATH}/src/usr/pnor/
 EXTRAINCDIR += ${ROOTPATH}/src/usr/lpc/
 EXTRAINCDIR += ${ROOTPATH}/src/include/usr/
diff --git a/src/build/buildpnor/PnorUtils.pm b/src/build/buildpnor/PnorUtils.pm
index 9cae4360b..d6faac42d 100644
--- a/src/build/buildpnor/PnorUtils.pm
+++ b/src/build/buildpnor/PnorUtils.pm
@@ -374,10 +374,11 @@ sub checkSpaceConstraints
         {
             # If this is a test run increase HBI size by PAGE_SIZE until all test
             # cases fit
-            if ($testRun && $eyeCatch eq "HBI")
+            if ($eyeCatch eq "HBI")
             {
-                print "$this_func: Adjusting HBI size - ran out of space for test cases\n";
-                adjustHbiPhysSize(\%sectionHash, $layoutKey, $filesize);
+                print "Adjusting HBI size - ran out of space for test cases\n";
+                my $stopKey = findLayoutKeyByEyeCatch("TEST", \%$i_pnorLayoutRef);
+                adjustSecPhysSize(\%sectionHash, $layoutKey, $filesize, $stopKey);
             }
             else
             {
@@ -390,39 +391,35 @@ sub checkSpaceConstraints
 
 
 ###############################################################################
-# adjustHbiPhysSize - Adjust HBI physical size when running test cases and fix
-#                     up physical offsets of partitions after it
+# adjustSecPhysSize - Adjust section physical size when running test cases
+#                     and fix up physical offsets between partitions
+#                     (for example HBI and TEST)
 ################################################################################
-sub adjustHbiPhysSize
+sub adjustSecPhysSize
 {
-    my ($i_sectionHashRef, $i_hbiKey, $i_filesize) = @_;
-    my $this_func = (caller(0))[3];
+    my ($i_sectionHashRef, $i_secKey, $i_filesize, $i_stopKey) = @_;
 
     my %sectionHash = %$i_sectionHashRef;
 
     # Increment HBI physical size by PAGE_SIZE until the HBI file can fit
-    my $hbi_old = $sectionHash{$i_hbiKey}{physicalRegionSize};
-    while ($i_filesize > $sectionHash{$i_hbiKey}{physicalRegionSize})
+    my $sec_old = $sectionHash{$i_secKey}{physicalRegionSize};
+    while ($i_filesize > $sectionHash{$i_secKey}{physicalRegionSize})
     {
-        $sectionHash{$i_hbiKey}{physicalRegionSize} += PAGE_SIZE;
+        $sectionHash{$i_secKey}{physicalRegionSize} += PAGE_SIZE;
     }
-    my $hbi_move = $sectionHash{$i_hbiKey}{physicalRegionSize} - $hbi_old;
-    my $hbi_end = $sectionHash{$i_hbiKey}{physicalRegionSize} + $hbi_move;
+    my $sec_move = $sectionHash{$i_secKey}{physicalRegionSize} - $sec_old;
+    my $sec_end = $sectionHash{$i_secKey}{physicalRegionSize} + $sec_move;
 
     # Fix up physical offset affected by HBI size change
     foreach my $section (keys %sectionHash)
     {
-        # Only fix partitions after HBI
-        if ( $sectionHash{$section}{physicalOffset} >
-             $sectionHash{$i_hbiKey}{physicalOffset} )
-        {
-            my $origoffset = $sectionHash{$section}{physicalOffset};
-            $sectionHash{$section}{physicalOffset} += $hbi_move;
-            trace(3, "$this_func: Section $sectionHash{$section}{eyeCatch} : " . sprintf("%X",$origoffset) . " --> " . sprintf("%X",$sectionHash{$section}{physicalOffset}));
-        }
-        else
+        # Only fix partitions after HBI and before $i_stopKey
+        if ($sectionHash{$section}{physicalOffset} > $sectionHash{$i_secKey}{physicalOffset}
+            && $sectionHash{$section}{physicalOffset} < $sectionHash{$i_stopKey}{physicalOffset})
         {
-            printf "$this_func: Section $sectionHash{$section}{eyeCatch} : unchanged";
+            my $new_location = $sectionHash{$section}{physicalOffset} + $sec_move;
+            print "Moving section $sectionHash{$section}{eyeCatch} forward by $sec_move bytes to $new_location\n";
+            $sectionHash{$section}{physicalOffset} = $new_location;
         }
     }
 }
diff --git a/src/build/buildpnor/defaultPnorLayout.xml b/src/build/buildpnor/defaultPnorLayout.xml
index 5a61c7397..9e7419f70 100644
--- a/src/build/buildpnor/defaultPnorLayout.xml
+++ b/src/build/buildpnor/defaultPnorLayout.xml
@@ -180,7 +180,7 @@ Layout Description
         <description>Payload (19.875MB)</description>
         <eyeCatch>PAYLOAD</eyeCatch>
         <physicalOffset>0x1D2D000</physicalOffset>
-        <physicalRegionSize>0x13E0000</physicalRegionSize>
+        <physicalRegionSize>0x0600000</physicalRegionSize>
         <sha512Version/>
         <side>sideless</side>
         <ecc/>
diff --git a/src/build/debug/Hostboot/Gcov.pm b/src/build/debug/Hostboot/Gcov.pm
index 84df2f9f9..48e71d66b 100755
--- a/src/build/debug/Hostboot/Gcov.pm
+++ b/src/build/debug/Hostboot/Gcov.pm
@@ -1,4 +1,3 @@
-#!/usr/bin/perl
 # IBM_PROLOG_BEGIN_TAG
 # This is an automatically generated prolog.
 #
@@ -6,7 +5,7 @@
 #
 # OpenPOWER HostBoot Project
 #
-# Contributors Listed Below - COPYRIGHT 2012,2015
+# Contributors Listed Below - COPYRIGHT 2012,2019
 # [+] International Business Machines Corp.
 #
 #
@@ -23,15 +22,18 @@
 # permissions and limitations under the License.
 #
 # IBM_PROLOG_END_TAG
+#!/usr/bin/perl
 use strict;
+use warnings;
 use File::Path;
 use File::Basename;
+use IO::Handle;
 
 package Hostboot::Gcov;
 use Hostboot::_DebugFrameworkVMM qw(NotFound NotPresent getPhysicalAddr);
 
 use Exporter;
-our @EXPORT_OK = ('main');
+our @EXPORT_OK = ('init', 'main', 'parseGcovInfo');
 
 # NOTE:
 #
@@ -50,26 +52,37 @@ our @EXPORT_OK = ('main');
 
 use constant GCOV_EXTENDED_IMAGE_ADDRESS => (1024 * 1024 * 1024);
 use constant GCOV_INFO_HEAD_SYMBOLNAME => "_gcov_info_head";
+use constant GCOV_INFO_MAGIC_SYMBOLNAME => "_gcov_info_magic";
+use constant GCOV_MAGIC_IDENTIFIER => 0xbeefb055;
 
-use constant GCOV_INFO_VERSION_OFFSET => 0;
-use constant GCOV_INFO_NEXT_OFFSET => GCOV_INFO_VERSION_OFFSET + 8;
-use constant GCOV_INFO_TIMESTAMP_OFFSET => GCOV_INFO_NEXT_OFFSET + 8;
-use constant GCOV_INFO_FILENAME_OFFSET => GCOV_INFO_TIMESTAMP_OFFSET + 8;
-use constant GCOV_INFO_NFUNCTIONS_OFFSET => GCOV_INFO_FILENAME_OFFSET + 8;
-use constant GCOV_INFO_FUNCTIONS_OFFSET => GCOV_INFO_NFUNCTIONS_OFFSET + 8;
-use constant GCOV_INFO_CTRMASK_OFFSET => GCOV_INFO_FUNCTIONS_OFFSET + 8;
-use constant GCOV_INFO_COUNTS_OFFSET => GCOV_INFO_CTRMASK_OFFSET + 8;
-
-use constant GCOV_FNINFO_IDENT_OFFSET => 0;
-use constant GCOV_FNINFO_CHECKSUM_OFFSET => GCOV_FNINFO_IDENT_OFFSET + 4;
-use constant GCOV_FNINFO_NCTRS_OFFSET => GCOV_FNINFO_CHECKSUM_OFFSET + 4;
+use constant GCOV_COUNTERS_492 => 9;
+use constant SIZEOF_PTR => 8;
+use constant SIZEOF_UINT64 => 8;
 
-use constant GCOV_CTRINFO_COUNT_OFFSET => 0;
-use constant GCOV_CTRINFO_VALUEPTR_OFFSET => GCOV_CTRINFO_COUNT_OFFSET + 8;
-
-use constant GCOV_GCDA_MAGIC_VALUE => 0x67636461;
 use constant GCOV_FUNCTION_TAG => 0x01000000;
 use constant GCOV_COUNTERS_TAG => 0x01a10000;
+use constant GCOV_PROGRAM_SUMMARY_TAG => 0xa3000000;
+
+use constant GCOV_GCDA_MAGIC_VALUE => 0x67636461;
+
+# See gcov.h for the structs (gcov_info, gcov_fn_info) from which
+# these offsets derive
+
+use constant GCOV_INFO_VERSION_OFFSET_492 => 0;
+use constant GCOV_INFO_NEXT_OFFSET_492 => 8;
+use constant GCOV_INFO_TIMESTAMP_OFFSET_492 => 16;
+use constant GCOV_INFO_FILENAME_OFFSET_492 => 24;
+use constant GCOV_INFO_MERGE_OFFSET_492 => 32;
+use constant GCOV_INFO_N_FUNCTIONS_OFFSET_492 => 32 + (9 * 8);
+use constant GCOV_INFO_FUNCTIONS_OFFSET_492 => GCOV_INFO_N_FUNCTIONS_OFFSET_492 + 8;
+
+use constant GCOV_FN_INFO_IDENT_OFFSET_492 => 8;
+use constant GCOV_FN_INFO_LINENO_CHECKSUM_OFFSET_492 => 12;
+use constant GCOV_FN_INFO_CFG_CHECKSUM_OFFSET_492 => 16;
+use constant GCOV_FN_INFO_CTR_INFO_OFFSET_492 => 24;
+
+use constant GCOV_CTR_INFO_NUM_OFFSET_492 => 0;
+use constant GCOV_CTR_INFO_VALUES_OFFSET_492 => 8;
 
 # In memory format:
 #       GCC creates a 'gcov_info' structure for each .o file.  The info
@@ -134,20 +147,43 @@ use constant GCOV_COUNTERS_TAG => 0x01a10000;
 #       uint64_ts, containing instrumented counts, for the preceeding function.
 
 # Global of where we want the output to go.
-our $output_dir;
 our $debug_mode;
+our $hbicore_extended_bin_file;
+our $hbicore_extended_bin_file_size;
 BEGIN
 {
     $debug_mode = 0;
-    $output_dir = "";
 }
-return 1;
+
+sub init
+{
+    # TODO: We need to figure out how to handle reading data from
+    # HBB/HBRT for when those are instrumented. One hurdle is being
+    # able to determine from an address what module it belongs to,
+    # because HBB/HBI/HBRT are not necessarily laid out in memory as
+    # they are in PNOR or anywhere else.
+
+    my $hbicore_extended_bin_fname = "$ENV{SANDBOXROOT}/$ENV{SANDBOXNAME}/src/hbfw/img/hostboot_extended.bin";
+
+    userDebug("Opening " . $hbicore_extended_bin_fname . " for HBI\n");
+
+    unless (open($hbicore_extended_bin_file, "< $hbicore_extended_bin_fname")) {
+        ::userDisplay "Failed to open $hbicore_extended_bin_fname, exiting\n";
+        return 0;
+    }
+
+    binmode($hbicore_extended_bin_file);
+
+    $hbicore_extended_bin_file_size = -s $hbicore_extended_bin_fname;
+
+    return 1;
+}
 
 sub main
 {
-    # Pick a new output directory based on the time.
-    $output_dir = sprintf "gcov.output.%d/", time;
-    File::Path::mkpath($output_dir);
+    if (!init()) {
+        return;
+    }
 
     # Find all the hostboot modules.
     my @modules = getModules();
@@ -160,7 +196,10 @@ sub main
 
     my $pwd = `pwd`;
     chomp $pwd;
-    ::userDisplay "GCOV output written to: $pwd/$output_dir\n";
+
+    close $hbicore_extended_bin_file or die;
+
+    ::userDisplay("GCOV info extraction complete.\n");
 }
 
 sub parseModuleGcov
@@ -168,8 +207,23 @@ sub parseModuleGcov
     my $module = shift;
     ::userDisplay "Extracting GCOV info for ".$module."\n";
 
+    # Search for magic symbol.
+    my ($gcov_magic, $unused) =
+        ::findSymbolAddress($module.GCOV_INFO_MAGIC_SYMBOLNAME);
+
+    if (!defined($gcov_magic))
+    {
+        $gcov_magic = 0;
+    }
+
+    if ($gcov_magic == 0 || read32($gcov_magic, 1) != GCOV_MAGIC_IDENTIFIER)
+    {
+        ::userDisplay "\tgcov_magic at address " . (sprintf "0x%x", $gcov_magic) . " is incorrect.  Skipped.\n";
+        return;
+    }
+
     # Search for gcov_info chain symbol.
-    my ($gcov_info, $unused) =
+    my ($gcov_info, $unused2) =
         ::findSymbolAddress($module.GCOV_INFO_HEAD_SYMBOLNAME);
 
     userDebug("\tFound info at 0x" . (sprintf "%x", $gcov_info) . "\n");
@@ -181,7 +235,7 @@ sub parseModuleGcov
 
         if (($gcov_info eq NotFound) || ($gcov_info eq NotPresent))
         {
-            ::userDisplay "\tModule data is not present.\n";
+            ::userDisplay "\tModule data is not present, module might have been unloaded, skipping.\n";
             return;
         }
     }
@@ -189,7 +243,7 @@ sub parseModuleGcov
     # Check that we found the gcov_info chain.
     if ($gcov_info == 0)
     {
-        ::userDisplay "\tUnable to find gcov_info chain.  Skipped.\n";
+        ::userDisplay "\tUnable to find gcov_info chain, module might hvae been unloaded. Skipping.\n";
         return;
     }
 
@@ -202,39 +256,44 @@ sub parseGcovInfo
     my $info_ptr = shift;
     return if (0 eq $info_ptr);
 
-    my $filename = readStr(read64($info_ptr + GCOV_INFO_FILENAME_OFFSET));
-    userDebug("\tFile = ".$filename."\n");
+    userDebug("\tReading filename pointer from offset " . (sprintf "0x%x", ($info_ptr + GCOV_INFO_FILENAME_OFFSET_492)) . "\n");
 
-    my $version = read32($info_ptr + GCOV_INFO_VERSION_OFFSET);
-    my $stamp = read32($info_ptr + GCOV_INFO_TIMESTAMP_OFFSET);
+    my $filename_addr = read64($info_ptr + GCOV_INFO_FILENAME_OFFSET_492);
 
-    my $func_count = read32($info_ptr + GCOV_INFO_NFUNCTIONS_OFFSET);
-    userDebug("\tFunction Count = ".$func_count."\n");
+    userDebug("\tReading filename from offset " . (sprintf "0x%x", $filename_addr) . "\n");
 
-    my $funcs = read64($info_ptr + GCOV_INFO_FUNCTIONS_OFFSET);
-    userDebug("\tFunc Address = ".(sprintf "%x", $funcs)."\n");
+    my $filename = readStr($filename_addr);
 
-    my $ctrmask = read32($info_ptr + GCOV_INFO_CTRMASK_OFFSET);
-    if ($ctrmask % 2) # Check that COUNTER_ARCS is turned on.
-    {
-        # COUNTER_ARCS is on.  Create file, find arc-values array,
-        # parse functions.
+    if ($filename) {
+        ::userDisplay("\tFile = ".$filename."\n");
 
-        my $fd = createGcovFile($filename, $version, $stamp);
+        my $version = read32($info_ptr + GCOV_INFO_VERSION_OFFSET_492);
+        my $stamp = read32($info_ptr + GCOV_INFO_TIMESTAMP_OFFSET_492);
 
-        my $arcs_ptr = read64($info_ptr + GCOV_INFO_COUNTS_OFFSET +
-                                GCOV_CTRINFO_VALUEPTR_OFFSET);
-        parseGcovFuncs($fd, $funcs, $func_count, $ctrmask, $arcs_ptr);
+        my $func_count = read32($info_ptr + GCOV_INFO_N_FUNCTIONS_OFFSET_492);
+        userDebug("\tFunction Count = ".$func_count."\n");
 
-        close $fd;
-    }
-    else
-    {
-        userDebug("COUNTER_ARCS is missing!\n");
+        my $funcs = read64($info_ptr + GCOV_INFO_FUNCTIONS_OFFSET_492, 1);
+
+        if ($funcs ne NotFound && $funcs ne NotPresent) {
+            userDebug("\tFunc Address = ".(sprintf "0x%x", $funcs)."\n");
+
+            if ($version ne NotFound && $stamp ne NotFound && $func_count ne NotFound) {
+                my $fd = createGcovFile($filename, $version, $stamp);
+
+                parseGcovFuncs($fd, $funcs, $func_count);
+
+                close $fd or die $!;
+            }
+        } else {
+            userDebug("\tFunc Address is NULL, skipping\n");
+        }
+    } else {
+        userDebug("\tCannot read filename, skipping\n");
     }
 
     # Look for next .o in gcov_info chain, parse.
-    my $next = read64($info_ptr + GCOV_INFO_NEXT_OFFSET);
+    my $next = read64($info_ptr + GCOV_INFO_NEXT_OFFSET_492);
     parseGcovInfo($next);
 }
 
@@ -243,62 +302,95 @@ sub parseGcovFuncs
     my $fd = shift;
     my $func_ptr = shift;
     my $func_count = shift;
-    my $mask = shift;
-    my $val_ptr = shift;
 
-    my $fn_offset = 0;
+    my $GCOV_COUNTERS_SUMMABLE_492 = 1;
 
-    # Need to calculate the number of counters based on the bits on in
-    # the 'mask'.  This is used to determine the size of the function
-    # descriptor object.
-    my $counters = 0;
-    {
-        my $_mask = $mask;
+    print $fd pack('l', GCOV_PROGRAM_SUMMARY_TAG); # data.program.header.tag
 
-        while (0 != $_mask)
-        {
-            $counters++;
-            $_mask = ($_mask >> 1);
-        }
-    }
+    # for each GCOV_COUNTERS_SUMMABLE we have ten int32 (num, runs, and bitvector{8})
+    # plus three int64 (sum, max, sum_max) i.e. 10 + 3*2
+    # data.unit.header.length is the number of int32's we have following.
+    print $fd pack('l', 1 + $GCOV_COUNTERS_SUMMABLE_492 * (10 + 3 * 2)); # data.unit.header.length;
 
-    userDebug("\tCounters = ".$counters."\n");
+    print $fd pack('l', 0); # data.summary:object.checksum (must be 0 according to docs)
 
-    # Round up the counter count to the nearest two for alignment of the
-    # function descriptor object.
-    if ($counters % 2)
-    {
-        $counters++;
-    }
-    my $func_size = GCOV_FNINFO_CHECKSUM_OFFSET + 4 * $counters;
+    for (my $i = 0; $i < $GCOV_COUNTERS_SUMMABLE_492;  $i++) {
+        print $fd pack('l', 0); # data.summary:object.count-summary.num
+        print $fd pack('l', 0); # data.summary:object.count-summary.runs
+        print $fd pack('l', 0); # data.summary:object.count-summary.sum@lo
+        print $fd pack('l', 0); # data.summary:object.count-summary.sum@hi
+        print $fd pack('l', 0); # data.summary:object.count-summary.max@lo
+        print $fd pack('l', 0); # data.summary:object.count-summary.max@hi
+        print $fd pack('l', 0); # data.summary:object.count-summary.sum_max@lo
+        print $fd pack('l', 0); # data.summary:object.count-summary.sum_max@hi
 
-    userDebug("\tFunction size = ".$func_size."\n");
+        print $fd pack('l8', (0) x 8);   # data.summary:object.count-summary.histogram.bitvector{8}
+    }
 
     # Iterate through the functions and parse.
     for(my $function = 0; $function < $func_count; $function++)
     {
-        my $func_off = ($func_ptr + $func_size * $function);
-        my $ident = read32($func_off + GCOV_FNINFO_IDENT_OFFSET);
-        my $chksum = read32($func_off + GCOV_FNINFO_CHECKSUM_OFFSET);
+        userDebug("\tFunction $function of $func_count\n");
+
+        my $fn_info_ptr = read64($func_ptr + SIZEOF_PTR*$function, 1);
+
+        if (($fn_info_ptr eq NotFound) || ($fn_info_ptr eq NotPresent))
+        {
+            userDebug("\tCannot read function info pointer, skipping\n");
+            next;
+        }
+
+        userDebug("\tfn_info_ptr = " . (sprintf "%x", $fn_info_ptr) . "\n");
+
+        my $ident = read32($fn_info_ptr + GCOV_FN_INFO_IDENT_OFFSET_492, 1);
+        my $lineno_chksum = read32($fn_info_ptr + GCOV_FN_INFO_LINENO_CHECKSUM_OFFSET_492, 1);
+        my $cfg_chksum = read32($fn_info_ptr + GCOV_FN_INFO_CFG_CHECKSUM_OFFSET_492, 1);
+        my $ctr_info_ptr = $fn_info_ptr + GCOV_FN_INFO_CTR_INFO_OFFSET_492;
+
+        if ($ident eq NotFound
+            || $lineno_chksum eq NotFound
+            || $cfg_chksum eq NotFound)
+        {
+            userDebug("Skipping because fn_info structure members are not readable\n");
+            next;
+        }
+
+        my $num_ctrs = read32($ctr_info_ptr + GCOV_CTR_INFO_NUM_OFFSET_492, 1);
+        my $ctrs_ptr = read64($ctr_info_ptr + GCOV_CTR_INFO_VALUES_OFFSET_492, 1);
+
+        if ($ctrs_ptr eq NotFound || $num_ctrs eq NotFound)
+        {
+            userDebug("Skipping because counters length isn't mapped\n");
+            next;
+        }
 
-        userDebug("Ident = ".(sprintf "%x", $ident)."\n");
-        userDebug("Chksum = ".(sprintf "%x", $chksum)."\n");
+        my $counters = readData($ctrs_ptr, SIZEOF_UINT64 * $num_ctrs);
+
+        userDebug("Ident = ".(sprintf "0x%x", $ident)."\n");
+        userDebug("lineno Chksum = ".(sprintf "0x%x", $lineno_chksum)."\n");
+        userDebug("cfg Chksum = ".(sprintf "0x%x", $cfg_chksum)."\n");
+        userDebug("Num counters = ".(sprintf "%d", $num_ctrs)."\n");
+        userDebug("ctrs_ptr = ".(sprintf "0x%x", $ctrs_ptr)."\n");
+
+        if (($counters eq NotFound) || ($counters eq NotPresent))
+        {
+            userDebug("Skipping because counter data not resident in memory\n");
+            next;
+        }
 
         print $fd pack('l', GCOV_FUNCTION_TAG);  # Write function tag.
-        print $fd pack('l', 2); # Write size = 2.
+        print $fd pack('l', 3); # Write size = 3.
         print $fd pack('l', $ident); # Write ident.
-        print $fd pack('l', $chksum); # Write checksum.
-
-        my $nctr_val = read32($func_off + GCOV_FNINFO_NCTRS_OFFSET);
-        userDebug("N-Counters = ".$nctr_val."\n");
+        print $fd pack('l', $lineno_chksum); # Write checksum.
+        print $fd pack('l', $cfg_chksum); # Write checksum.
 
         print $fd pack('l', GCOV_COUNTERS_TAG); # Write counter tag.
-        print $fd pack('l', $nctr_val * 2); # Write counter length.
+        print $fd pack('l', $num_ctrs * 2); # Write counter length.
 
         # Read each counter value, output.
         #       Read as one big block for performance reasons.
-        my $counters = readData($val_ptr + 8*($fn_offset), 8 * $nctr_val);
-        for(my $v_idx = 0; $v_idx < $nctr_val; $v_idx++)
+
+        for(my $v_idx = 0; $v_idx < $num_ctrs; $v_idx++)
         {
             my $val = substr $counters, 0, 8;
             $counters = substr $counters, 8;
@@ -306,15 +398,34 @@ sub parseGcovFuncs
             $val = unpack("Q", $val);
             userDebug("\tValue[$v_idx] = ".$val."\n");
 
+            my $preex_read_low = read($fd, my $low_word, 4);
+            my $preex_read_high = read($fd, my $high_word, 4);
+
+            if (!defined($preex_read_low) or !(defined($preex_read_high))) {
+                die;
+            }
+            my $preex_read = $preex_read_low + $preex_read_high;
+
+            if ($preex_read == 8)
+            {
+                my $preex_val = (unpack("l", $high_word) << 32) | unpack("l", $low_word);
+
+                $val += $preex_val;
+            }
+
+            if ($preex_read > 0)
+            {
+                seek $fd, -$preex_read, 1;
+            }
+            else
+            {
+                seek $fd, 0, 2;
+            }
+
             print $fd pack('l', $val & 0xFFFFFFFF);  # Write lower word.
             print $fd pack('l', $val >> 32) ; # Write upper word.
         }
-
-        # We used up a number of counters, so move the offset forward for
-        # the next function.
-        $fn_offset += $nctr_val;
     }
-
 }
 
 # The *.gcda filename found in the gcov_info struct is an absolute path to
@@ -330,16 +441,25 @@ sub createGcovFile
     my $version = shift;
     my $stamp = shift;
 
-    # Change *./../obj/ into obj/, prepend output_dir.
-    $name =~ s/.*\/obj\//obj\//;
-    $name = $output_dir.$name;
-
-    # Make sure everything after 'obj/' exists (create subdirs).
-    my $dir = File::Basename::dirname($name);
-    File::Path::mkpath($dir);
+    # if the file exists then we update it, if not we create it
+    my $GCOVFILE;
+    if (-e $name)
+    {
+        if (!open($GCOVFILE, "+<$name"))
+        {
+            ::userDisplay("Failed to open $name for reading/writing gcov information\n");
+            die;
+        }
+    }
+    else
+    {
+        if (!open($GCOVFILE, "+>$name"))
+        {
+            ::userDisplay("Failed to open $name for writing gcov information\n");
+            die;
+        }
+    }
 
-    # Create file.
-    open(my $GCOVFILE, "> $name");
     binmode($GCOVFILE);
 
     # Write out header.
@@ -377,6 +497,22 @@ sub isVirtualAddress
     return ($addr >= GCOV_EXTENDED_IMAGE_ADDRESS);
 }
 
+sub readExtImage
+{
+    my $addr = shift;
+    my $amount = shift;
+
+    if ($addr + $amount >= $hbicore_extended_bin_file_size) {
+        return NotFound;
+    }
+
+    seek $hbicore_extended_bin_file, $addr, 0;
+
+    read $hbicore_extended_bin_file, my ($contents), $amount;
+
+    return $contents;
+}
+
 # Utility to read a block of data from eithr memory or using the extended
 # image file as a fallback if not present in memory.
 use constant PAGESIZE => 4096;
@@ -401,8 +537,13 @@ sub readData
             my $paddr = getPhysicalAddr($addr);
             if ((NotFound eq $paddr) || (NotPresent eq $paddr))
             {
-                $paddr = $addr - GCOV_EXTENDED_IMAGE_ADDRESS;
-                $result = $result.::readExtImage($paddr, $amount);
+                my $tmpdata = readExtImage($addr - GCOV_EXTENDED_IMAGE_ADDRESS, $amount);
+
+                if ($tmpdata eq NotFound) {
+                    return NotFound;
+                }
+
+                $result = $result . $tmpdata;
             }
             else
             {
@@ -423,14 +564,27 @@ sub readData
 sub read64
 {
     my $addr = shift;
+    my $fallback = shift;
+
     my $old_addr = $addr;
     if (isVirtualAddress($addr))
     {
         $addr = getPhysicalAddr($addr);
         if ((NotFound eq $addr) || (NotPresent eq $addr))
         {
+            userDebug((sprintf "0x%x", $old_addr). " not translatable 2\n");
+
+            if (!$fallback) {
+                return NotFound;
+            }
+
             $addr = $old_addr - GCOV_EXTENDED_IMAGE_ADDRESS;
-            my $result = ::readExtImage($addr, 8);
+            my $result = readExtImage($addr, 8);
+
+            if ($result eq NotFound) {
+                return NotFound;
+            }
+
             if (::littleendian()) { $result = reverse($result); }
             return unpack("Q", $result);
         }
@@ -444,14 +598,25 @@ sub read64
 sub read32
 {
     my $addr = shift;
+    my $fallback = shift;
+
     my $old_addr = $addr;
     if (isVirtualAddress($addr))
     {
         $addr = getPhysicalAddr($addr);
         if ((NotFound eq $addr) || (NotPresent eq $addr))
         {
+            userDebug((sprintf "0x%x", $old_addr). "not translatable 3\n");
+
+            if (!$fallback) {
+                return NotFound;
+            }
+
             $addr = $old_addr - GCOV_EXTENDED_IMAGE_ADDRESS;
-            my $result = ::readExtImage($addr, 4);
+            my $result = readExtImage($addr, 4);
+            if ($result eq NotFound) {
+                return NotFound;
+            }
             if (::littleendian()) { $result = reverse($result); }
             return unpack("L", $result);
         }
@@ -471,8 +636,10 @@ sub read8
         $addr = getPhysicalAddr($addr);
         if ((NotFound eq $addr) || (NotPresent eq $addr))
         {
+            userDebug((sprintf "0x%x", $addr). "not translatable 4\n");
+
             $addr = $old_addr - GCOV_EXTENDED_IMAGE_ADDRESS;
-            my $result = ::readExtImage($addr, 1);
+            my $result = readExtImage($addr, 1);
             return unpack("C", $result);
         }
     }
@@ -486,31 +653,82 @@ sub readStr
 {
     my $addr = shift;
     my $old_addr = $addr;
+
     if (isVirtualAddress($addr))
     {
-        $addr = $addr - GCOV_EXTENDED_IMAGE_ADDRESS;
+        userDebug("it is a virtual address, addr is " . (sprintf "%x", $addr) . "\n");
+        my $phys_addr = getPhysicalAddr($addr);
 
-        # Virtual address, so need to read 1 byte at a time from the file.
-        my $string = "";
-        my $byte = 0;
-
-        do
+        if ((NotFound eq $phys_addr) || (NotPresent eq $phys_addr))
         {
-            $byte = ::readExtImage($addr,1);
-            $addr = $addr + 1;
+            userDebug("Translation not found, reading from pnor\n");
+            # Virtual address, so need to read 1 byte at a time from the file.
+            my $string = "";
+            my $byte = 0;
 
-            if (unpack("C",$byte) eq 0)
+            do
             {
-                return $string;
-            }
+                $byte = readExtImage($addr - GCOV_EXTENDED_IMAGE_ADDRESS, 1);
 
-            $string = $string.$byte;
+                if ($byte eq NotFound)
+                {
+                    return "";
+                }
 
-        } while (1)
+                $addr = $addr + 1;
+
+                if (unpack("C",$byte) eq 0)
+                {
+                    return $string;
+                }
+
+                $string = $string.$byte;
+            } while (1);
+        }
+        else
+        {
+            my $string = "";
+            my $byte = 0;
+
+            do
+            {
+                if (($addr & 0xfff) == 0)
+                {
+                    # we have to recalculate the physical address
+                    # whenever we cross a page boundary
+                    $phys_addr = getPhysicalAddr($addr);
+
+                    if ((NotFound eq $phys_addr) || (NotPresent eq $phys_addr))
+                    {
+                        userDebug((sprintf "0x%x", $addr). "not translatable 10\n");
+                        return "";
+                    }
+                }
+
+                $byte = read8($phys_addr);
+
+                if ($byte eq NotFound)
+                {
+                    userDebug("Cannot read byte from physical address\n");
+                    return "";
+                }
+
+                $addr += 1;
+                $phys_addr += 1;
+
+                if ($byte != 0)
+                {
+                    $string = $string . pack("C", $byte);
+                }
+            } while ($byte != 0);
+
+            return $string;
+        }
     }
     else
     {
-        ::readStr($addr);
+        userDebug("it is NOT a virtual address\n");
+        return ::readStr($addr);
     }
 }
 
@@ -531,3 +749,5 @@ sub helpInfo
         intro => [ "Extracts the GCOV information."],
     );
 }
+
+1; # Last expression in a perl module must be truthy.
diff --git a/src/build/debug/Hostboot/GcovModuleUnload.pm b/src/build/debug/Hostboot/GcovModuleUnload.pm
new file mode 100644
index 000000000..1a411cc3c
--- /dev/null
+++ b/src/build/debug/Hostboot/GcovModuleUnload.pm
@@ -0,0 +1,69 @@
+# IBM_PROLOG_BEGIN_TAG
+# This is an automatically generated prolog.
+#
+# $Source: src/build/debug/Hostboot/GcovModuleUnload.pm $
+#
+# OpenPOWER HostBoot Project
+#
+# Contributors Listed Below - COPYRIGHT 2019
+# [+] International Business Machines Corp.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# IBM_PROLOG_END_TAG
+#!/usr/bin/perl
+use strict;
+use File::Path;
+use File::Basename;
+
+package Hostboot::GcovModuleUnload;
+use Hostboot::Gcov qw(parseGcovInfo init);
+
+use Exporter;
+our @EXPORT_OK = ('main', 'parseGcovInfo');
+
+sub main
+{
+    my ($packName, $args) = @_;
+
+    my $gcov_info_address = $args->{"address"};
+
+    ::userDisplay("Dumping gcov module info from " . (sprintf "0x%x", $gcov_info_address) . "\n");
+
+    if ($gcov_info_address <= 0) {
+        ::userDisplay("Can't dump from NULL\n");
+        return -1;
+    }
+
+    if (!Hostboot::Gcov::init()) {
+        return -2;
+    }
+
+    Hostboot::Gcov::parseGcovInfo($gcov_info_address);
+
+    ::userDisplay("Done.\n");
+
+    return 0;
+}
+
+# Debug tool help info.
+sub helpInfo
+{
+    my %info = (
+        name => "GcovModuleUnload",
+        intro => [ "Extracts the GCOV information from modules as they are being unloaded."],
+    );
+}
+
+1; # Last expression in a perl module must be truthy.
diff --git a/src/build/debug/simics-debug-framework.py b/src/build/debug/simics-debug-framework.py
index 3176f8cba..69465da3f 100644
--- a/src/build/debug/simics-debug-framework.py
+++ b/src/build/debug/simics-debug-framework.py
@@ -745,7 +745,9 @@ def magic_instruction_callback(user_arg, cpu, arg):
         #file = open("hb_trace_debug.dat", "a")
         #file.write("%s\n" % (saveCommand))
         #file.close()
-
+    if arg == 7056:   # MAGIC_GCOV_DUMP_NOW
+        print('Gcov dumping chain from 0x%x' % (cpu.r3,))
+        SIM_run_alone(run_command, 'hb-GcovModuleUnload "address=%d"' % (cpu.r3,))
 
 # Continuous trace: Clear these files.
 rc = os.system( "rm -f hbTracMERG" )
@@ -769,4 +771,3 @@ SIM_hap_add_callback_range( "Core_Magic_Instruction", magic_instruction_callback
 
 # Run the registration automatically whenever this script is loaded.
 register_hb_debug_framework_tools()
-
diff --git a/src/build/linker/linker.C b/src/build/linker/linker.C
index a9a9d0f5e..e5e53ad8a 100644
--- a/src/build/linker/linker.C
+++ b/src/build/linker/linker.C
@@ -439,6 +439,8 @@ int main(int argc, char** argv)
             //     A contained member value might be something like
             //     _ZZ3fooE3bar.
             string sym_name = string((i->c_str())+1);
+            const char* gcovstr = "__gcov";
+            size_t gcovstrlen = strlen(gcovstr);
 
             cout << "Checking weak symbol: " << *i << endl;
 
@@ -451,6 +453,12 @@ int main(int argc, char** argv)
                         == j->find("traceData_codeInfo"))
                     && (*i != *j))
                 {
+                    if (strncmp((*j).c_str(),
+                        gcovstr,
+                        gcovstrlen)==0)
+                    {
+                        continue;
+                    }
                     cout << "\tDuplicate member found: " << *j << endl;
                     throw std::runtime_error(
                                 string("Duplicate weak symbol with contained "
diff --git a/src/build/mkrules/cc.rules.mk b/src/build/mkrules/cc.rules.mk
index c89995f82..584b11db4 100644
--- a/src/build/mkrules/cc.rules.mk
+++ b/src/build/mkrules/cc.rules.mk
@@ -5,7 +5,7 @@
 #
 # OpenPOWER HostBoot Project
 #
-# Contributors Listed Below - COPYRIGHT 2013,2017
+# Contributors Listed Below - COPYRIGHT 2013,2019
 # [+] International Business Machines Corp.
 #
 #
@@ -31,11 +31,27 @@ $(OBJDIR)/%.list : $(OBJDIR)/%.o
 	$(C2) "    OBJDUMP    $(notdir $<)"
 	$(C1)$(OBJDUMP) -rdlCS $< > $@
 
+# SOURCE_FILE and INCLUDE_DIRS are variables that are either absolute
+# paths to the .C file being compiled and the include directories if
+# we're building with gcov, or else they are relative paths
+# otherwise. The key thing to remember is that they are lazily
+# expanded, so they're relevant to whatever rule they're used in. We
+# don't want to always have absolute paths because of build
+# performance and because it causes the build output with
+# BUILD_VERBOSE to be larger and less readable.
+ifdef HOSTBOOT_PROFILE
+SOURCE_FILE=$(shell readlink -f $<)
+INCLUDE_DIRS=$(shell $(ROOTPATH)/src/build/tools/cflags.sh $(INCFLAGS))
+else
+SOURCE_FILE=$<
+INCLUDE_DIRS=$(INCFLAGS)
+endif
+
 $(OBJDIR)/%.o : %.C
 	@mkdir -p $(OBJDIR)
 	$(C2) "    CXX        $(notdir $<)"
-	$(C1)$(CXX) -c $(call FLAGS_FILTER, $(CXXFLAGS), $<) $< \
-	            -o $@.trace $(INCFLAGS) -iquote .
+	$(C1)$(CXX) -c $(call FLAGS_FILTER, $(CXXFLAGS), $<) $(SOURCE_FILE) \
+	            -o $@.trace $(INCLUDE_DIRS) -iquote .
 	$(C1)$(TRACE_HASHER) $@ $(TRACE_FLAGS)
 	@rm $@.trace
 
@@ -43,7 +59,8 @@ $(OBJDIR)/%.o : %.C
 $(OBJDIR)/%.o : %.cc
 	@mkdir -p $(OBJDIR)
 	$(C2) "    CXX        $(notdir $<)"
-	$(C1)$(CXX) -c $(CXXFLAGS) $< -o $@.trace $(INCFLAGS) -iquote .
+	$(C1)$(CXX) -c $(CXXFLAGS) $(SOURCE_FILE) -o $@.trace \
+	               $(INCLUDE_DIRS) -iquote .
 	$(C1)$(TRACE_HASHER) $@ $(TRACE_FLAGS)
 	@rm $@.trace
 
@@ -54,12 +71,12 @@ $(OBJDIR)/%.o : %.c
 # CC_OVERRIDE is set in the makefile of the component
 ifndef CC_OVERRIDE
 	$(C2) "    CC         $(notdir $<)"
-	$(C1)$(CC) -c $(call FLAGS_FILTER, $(CFLAGS), $<) $< \
-	           -o $@.trace $(INCFLAGS) -iquote .
+	$(C1)$(CC) -c $(call FLAGS_FILTER, $(CFLAGS), $<) $(SOURCE_FILE) \
+	           -o $@.trace $(INCLUDE_DIRS) -iquote .
 else
 	$(C2) "    CXX        $(notdir $<)"
-	$(C1)$(CXX) -c $(call FLAGS_FILTER, $(CXXFLAGS), $<) $< \
-	            -o $@.trace $(INCFLAGS) -iquote .
+	$(C1)$(CXX) -c $(call FLAGS_FILTER, $(CXXFLAGS), $<) $(SOURCE_FILE) \
+	            -o $@.trace $(INCLUDE_DIRS) -iquote .
 endif
 	$(C1)$(TRACE_HASHER) $@ $(TRACE_FLAGS)
 	@rm $@.trace
diff --git a/src/build/mkrules/gcov.env.mk b/src/build/mkrules/gcov.env.mk
index 8ddcd3ef6..edce52a24 100644
--- a/src/build/mkrules/gcov.env.mk
+++ b/src/build/mkrules/gcov.env.mk
@@ -5,7 +5,9 @@
 #
 # OpenPOWER HostBoot Project
 #
-# COPYRIGHT International Business Machines Corp. 2013,2014
+# Contributors Listed Below - COPYRIGHT 2013,2019
+# [+] International Business Machines Corp.
+#
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,21 +28,44 @@
 #     Configuration of the GCOV settings.
 
 GCOVDIR = $(ROOTPATH)/obj/gcov
+vpath %.C $(ROOTPATH)/src/sys/prof
 
 ifdef MODULE
-GCOVNAME = $(MODULE).lcov
+
+# Don't profile HBRT modules to keep size down
+ifdef HOSTBOOT_RUNTIME
+HOSTBOOT_PROFILE=
+endif
+
+## We don't want certain modules to be profiled (HBB, HBRT).
+
+# This is replacing spaces with colons so that we can get an exact
+# match on the module name in the list of unprofilable modules with
+# findstring, which otherwise would find matches on "partial"
+# substrings (i.e. we don't want to deprofile module ABC just because
+# module ABCD is blacklisted, so we create a blacklist of modules
+# separated by colons and search for :ABC:).
+null :=
+MODULE_PROFILE_BLACKLIST:=:$(subst ${null} ${null},:,$(BASE_MODULES_GCOV_BLACKLIST) $(RUNTIME_MODULES_GCOV_BLACKLIST)):
+
+ifneq (,$(findstring :$(MODULE):,$(MODULE_PROFILE_BLACKLIST)))
+HOSTBOOT_PROFILE=
+endif
+
+GCOVNAME := $(MODULE).lcov
+
 ifndef TESTS
 ifdef HOSTBOOT_PROFILE
-vpath %.C $(ROOTPATH)/src/sys/prof
 OBJS := gcov.o $(OBJS)
 endif
 endif
 else
-GCOVNAME = $(notdir $(shell pwd)).lcov
+GCOVNAME := $(notdir $(shell pwd)).lcov
 endif
 
-## Disable coverage on test cases, any directory that sets
-## HOSTBOOT_PROFILE_NO_INSTRUMENT or any file that has 'gcov' in the name.
+## Disable coverage on test cases or any directory that sets
+## HOSTBOOT_PROFILE_NO_INSTRUMENT
+
 ifndef TESTS
 ifdef HOSTBOOT_PROFILE
 ifndef HOSTBOOT_PROFILE_NO_INSTRUMENT
@@ -59,5 +84,16 @@ endif
 ## Reduce the optimization level when profiling is enabled to ensure the
 ## base image fits in 512k still.
 ifdef HOSTBOOT_PROFILE
-OPT_LEVEL = -Os
+# We're not doing this right now because it causes linker errors in
+# various parts of hostboot with or without gcov (i.e. functions that
+# the code relies on being inlined are not; some const statics that
+# aren't defined in a compilation unit have linker references, etc.)
+# and it also causes the image to be generated incorrectly (symbol
+# names are wrong, calls to functions named things like __savegpr_rXX
+# are emitted but the functions themselves aren't and so you end up
+# jumping into an area of zeroes, ...). We should come back to this
+# later and fix, we might be able to profile more of hostboot with the
+# space savings that -Os could give.
+
+#OPT_LEVEL = -Os
 endif
diff --git a/src/build/tools/cflags.sh b/src/build/tools/cflags.sh
new file mode 100755
index 000000000..09483f3eb
--- /dev/null
+++ b/src/build/tools/cflags.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+# IBM_PROLOG_BEGIN_TAG
+# This is an automatically generated prolog.
+#
+# $Source: src/build/tools/cflags.sh $
+#
+# OpenPOWER HostBoot Project
+#
+# Contributors Listed Below - COPYRIGHT 2019
+# [+] International Business Machines Corp.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# IBM_PROLOG_END_TAG
+
+# The purpose of this script is to convert a series of GCC compiler
+# flags into an equivalent series of flags, but which use absolute
+# paths for include directories (i.e. the "-I" flag) instead of
+# relative paths. We have to do this when GCOV is in use, because it
+# stupidly puts data files in one path but stores paths inside the
+# data files that are relative to some other path. When GCOV then
+# tries to read the source files via these relative paths, it can't
+# find the files. We correct this by converting all paths to source
+# and header files to absolute paths.
+
+if [ ! "$HOSTBOOT_PROFILE" ] ; then
+    echo "$@"
+    exit
+fi
+
+make_path_abs () {
+    local ABSPATH=$(readlink -f "$1")
+
+    if [ $? -ne 0 ] ; then
+        ABSPATH="$1"
+    fi
+
+    echo -n "$ABSPATH"
+}
+
+while [ "$#" -gt 0 ] ; do
+    FLAG="$1"
+
+    if [ "$FLAG" = "-I" ] ; then
+        shift
+        echo -n "-I $(make_path_abs "$1") "
+    elif [ "${FLAG:0:2}" = "-I" ] ; then
+        echo -n "-I $(make_path_abs "${FLAG:2}") "
+    else
+        echo -n "$FLAG "
+    fi
+
+    shift
+done
+
+echo
diff --git a/src/include/arch/ppc.H b/src/include/arch/ppc.H
index 81c49eb4f..b0c076d0a 100644
--- a/src/include/arch/ppc.H
+++ b/src/include/arch/ppc.H
@@ -514,6 +514,7 @@ enum
     MAGIC_TOGGLE_OUTPUT       =  7023, // Enable simic log capture
 
     MAGIC_CONTINUOUS_TRACE    =  7055, // extract mixed trace buffer
+    MAGIC_GCOV_MODULE_UNLOAD  =  7056, // extract gcov info
 
 
     // 8000-8999 are defined by the Simics CEC team
diff --git a/src/include/usr/gcov.h b/src/include/usr/gcov.h
index ede5e547b..8b506a935 100644
--- a/src/include/usr/gcov.h
+++ b/src/include/usr/gcov.h
@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2012,2014              */
+/* Contributors Listed Below - COPYRIGHT 2012,2019                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -39,6 +41,7 @@
 #include <stddef.h>
 #include <stdint.h>
 #include <string.h>
+#include <arch/ppc.H>
 
 /** @struct gcov_info
  *  @brief Structure generated by gcc.  Do not use.
@@ -56,23 +59,109 @@
  *  aligned on a 64-bit boundary.  The unusedN fields are to ensure proper
  *  alignment.
  */
+
+// Inferred settings
+#define BITS_PER_UNIT 8
+#define LONG_LONG_TYPE_SIZE 64
+
+#define _STRINGIFY(X) #X
+#define STRINGIFY(X) _STRINGIFY(X)
+
+static_assert(sizeof(long long) * 8 == LONG_LONG_TYPE_SIZE,
+              "sizeof(long long) * 8 must be LONG_LONG_TYPE_SIZE ("
+              STRINGIFY(LONG_LONG_TYPE_SIZE)
+              ")");
+
+/* The following typedefs and structures were taken from:
+ * https://github.com/gcc-mirror/gcc/blob/gcc-4_9-branch/libgcc/libgcov.h
+ */
+
+#if BITS_PER_UNIT == 8
+typedef unsigned gcov_unsigned_t __attribute__ ((mode (SI)));
+typedef unsigned gcov_position_t __attribute__ ((mode (SI)));
+#if LONG_LONG_TYPE_SIZE > 32
+typedef signed gcov_type __attribute__ ((mode (DI)));
+typedef unsigned gcov_type_unsigned __attribute__ ((mode (DI)));
+#else
+typedef signed gcov_type __attribute__ ((mode (SI)));
+typedef unsigned gcov_type_unsigned __attribute__ ((mode (SI)));
+#endif
+#else
+#if BITS_PER_UNIT == 16
+typedef unsigned gcov_unsigned_t __attribute__ ((mode (HI)));
+typedef unsigned gcov_position_t __attribute__ ((mode (HI)));
+#if LONG_LONG_TYPE_SIZE > 32
+typedef signed gcov_type __attribute__ ((mode (SI)));
+typedef unsigned gcov_type_unsigned __attribute__ ((mode (SI)));
+#else
+typedef signed gcov_type __attribute__ ((mode (HI)));
+typedef unsigned gcov_type_unsigned __attribute__ ((mode (HI)));
+#endif
+#else
+typedef unsigned gcov_unsigned_t __attribute__ ((mode (QI)));
+typedef unsigned gcov_position_t __attribute__ ((mode (QI)));
+#if LONG_LONG_TYPE_SIZE > 32
+typedef signed gcov_type __attribute__ ((mode (HI)));
+typedef unsigned gcov_type_unsigned __attribute__ ((mode (HI)));
+#else
+typedef signed gcov_type __attribute__ ((mode (QI)));
+typedef unsigned gcov_type_unsigned __attribute__ ((mode (QI)));
+#endif
+#endif
+#endif
+
+static_assert(sizeof(gcov_type) == 8, "gcov_type isn't 64 bits for some reason");
+
+#if __GNUC__ == 4 && __GNUC_MINOR__ == 9
+#define GCOV_COUNTERS 9
+#else
+#error We dont support this compiler yet
+#endif
+
+/* Structures embedded in coveraged program.  The structures generated
+   by write_profile must match these.  */
+
+/* Information about counters for a single function.  */
+struct gcov_ctr_info
+{
+    gcov_unsigned_t num;    /* number of counters.  */
+    gcov_type *values;      /* their values.  */
+};
+
+/* Information about a single function.  This uses the trailing array
+   idiom. The number of counters is determined from the merge pointer
+   array in gcov_info.  The key is used to detect which of a set of
+   comdat functions was selected -- it points to the gcov_info object
+   of the object file containing the selected comdat function.  */
+
+struct gcov_fn_info
+{
+    const struct gcov_info *key;        /* comdat key */
+    gcov_unsigned_t ident;              /* unique ident of function */
+    gcov_unsigned_t lineno_checksum;    /* function lineno_checksum */
+    gcov_unsigned_t cfg_checksum;       /* function cfg checksum */
+    struct gcov_ctr_info ctrs[0];       /* instrumented counters */
+};
+
+/* Type of function used to merge counters.  */
+typedef void (*gcov_merge_fn) (gcov_type *, gcov_unsigned_t);
+
+/* Information about a single object file.  */
 struct gcov_info
 {
-    uint32_t version;
-    uint32_t unused0;
-    gcov_info* next;
-    uint32_t timestamp;
-    uint32_t unused1;
-    char* filename;
-    uint32_t n_functions;
-    uint32_t unused2;
-    void* functions;
-    uint32_t counter_mask;
-    uint32_t unused3;
-    uint32_t n_counters;
-    uint32_t unused4;
-    uint64_t* counters;
-} PACKED;
+    gcov_unsigned_t version;/* expected version number */
+    struct gcov_info *next; /* link to next, used by libgcov */
+
+    gcov_unsigned_t stamp;  /* uniquifying time stamp */
+    const char *filename;   /* output file name */
+
+    gcov_merge_fn merge[GCOV_COUNTERS]; /* merge functions (null for
+                                           unused) */
+
+    unsigned n_functions;                        /* number of functions */
+    const struct gcov_fn_info *const *functions; /* pointer to pointers
+                                                    to function information */
+};
 
 // Preprocessor magic to create a variable name based off the module name.
 // GCOV_INFO_OBJ() will create a post-processed name like
@@ -90,6 +179,9 @@ struct gcov_info
 #define __GCOV_INFO_OBJ(X,Y) X ## Y
 #define _GCOV_INFO_OBJ(X,Y) __GCOV_INFO_OBJ(X,Y)
 #define GCOV_INFO_OBJ() _GCOV_INFO_OBJ(__GCOV_PREFIX, _gcov_info_head)
+#define GCOV_INFO_MAGIC() _GCOV_INFO_OBJ(__GCOV_PREFIX, _gcov_info_magic)
+
+uint32_t GCOV_INFO_MAGIC() = 0xbeefb055;
 
 /** Pointer to the beginning of the gcov_info chain for this module. */
 gcov_info* GCOV_INFO_OBJ() = NULL;
@@ -121,7 +213,7 @@ void __gcov_init(gcov_info* i_info)
 //
 #ifdef __HOSTBOOT_MODULE
 // Forward declaration of __gcov_module_copychain for modules.
-extern "C" void __gcov_module_copychain(gcov_info* chain);
+extern "C" void __gcov_module_copychain(gcov_info** chain);
 
 /** Function called by module unloading to move the module's gcov_info
  *  instances to the global chain.
@@ -129,7 +221,7 @@ extern "C" void __gcov_module_copychain(gcov_info* chain);
 extern "C"
 void __gcov_module_unload(void* unused)
 {
-    __gcov_module_copychain(GCOV_INFO_OBJ());
+    __gcov_module_copychain(&GCOV_INFO_OBJ());
 }
     // Register __gcov_module_unload with __cxa_atexit.
 extern void* __dso_handle;
@@ -137,27 +229,93 @@ extern "C" int __cxa_atexit(void(*)(void*),void*,void*);
 int __unused_gcov_cxa_register =
                     __cxa_atexit(&__gcov_module_unload, NULL, __dso_handle);
 #else
+
+// This is set to 1 for now because it reduces memory pressure, but it
+// won't work on hardware. See the comment below about
+// MAGIC_GCOV_MODULE_UNLOAD.
+#define HOSTBOOT_GCOV_EAGER_DATA_EXTRACTION 1
+
 /** Function called by a module being unloaded (via __gcov_module_unload) to
  *  copy the module's gcov_info chain into the base gcov_info chain.
  */
 extern "C"
-void __gcov_module_copychain(gcov_info* chain)
+void __gcov_module_copychain(gcov_info** const chain_ptr)
 {
+    gcov_info* chain = *chain_ptr;
+
+#if HOSTBOOT_GCOV_EAGER_DATA_EXTRACTION
+    asm volatile("mr %%r3, %0"
+                 :
+                 : "r" (chain)
+                 : "%r3");
+
+    /* This magic instruction will cause simics to read the gcov_info
+     * pointer in r3 and use it to immediately extract this unloading
+     * module's data. We do this to reduce the max simultaneous memory
+     * pressure, otherwise we run out of memory having to preserve and
+     * store all the gcov info for all unloaded modules until the end
+     * of the run. This only works in simics, but the alternative
+     * won't work on hardware until we find other ways to reduce our
+     * memory footprint. */
+
+    MAGIC_INSTRUCTION(MAGIC_GCOV_MODULE_UNLOAD);
+#else
     while(chain != NULL)
     {
         // Copy old info.
-        gcov_info* new_info = new gcov_info();
-        memcpy(new_info, chain, sizeof(gcov_info));
+        gcov_info* const new_info = new gcov_info();
+
+        memcpy(new_info, chain, sizeof(*chain));
 
-        // Copy old counters.
-        uint64_t* new_counters = new uint64_t[chain->n_counters]();
-        memcpy(new_counters, chain->counters,
-               chain->n_counters*sizeof(uint64_t));
-        new_info->counters = new_counters;
+        char* const new_filename = strdup(chain->filename);
+        new_info->filename = new_filename;
+
+        struct gcov_fn_info** const new_functions
+            = new struct gcov_fn_info*[new_info->n_functions];
+
+        new_info->functions = new_functions;
+
+        unsigned int num_gcov_counters = 0;
+
+        for (unsigned int i = 0; i < GCOV_COUNTERS; ++i)
+        {
+            if (new_info->merge[i]) {
+                ++num_gcov_counters;
+            }
+        }
+
+        for (unsigned int i = 0; i < chain->n_functions; ++i)
+        {
+            // malloc(base structure size + trailing array size)
+            new_functions[i] =
+                ((struct gcov_fn_info*)
+                 (new char[sizeof(*new_functions[i])
+                           + (sizeof(new_functions[i]->ctrs[0])
+                              * num_gcov_counters)]));
+
+            struct gcov_fn_info* const new_info = new_functions[i];
+            const struct gcov_fn_info* const old_info = chain->functions[i];
+
+            for (unsigned int j = 0; j < num_gcov_counters; ++j)
+            {
+                const gcov_unsigned_t num_values = old_info->ctrs[j].num;
+
+                new_info->key = NULL;
+                new_info->ctrs[j].num = num_values;
+                new_info->ctrs[j].values = new gcov_type[num_values];
+
+                memcpy(new_info->ctrs[j].values,
+                       old_info->ctrs[j].values,
+                       sizeof(gcov_type) * num_values);
+            }
+        }
 
         // Atomically push new_info onto the core_gcov_info_head stack.
         do
         {
+            /* GCOV_INFO_OBJ() in this function is always
+               core_gcov_info_head because this function is only
+               defined when __HOSTBOOT_MODULE is not defined */
             new_info->next = GCOV_INFO_OBJ();
         } while (!__sync_bool_compare_and_swap(&GCOV_INFO_OBJ(),
                                                new_info->next, new_info));
@@ -165,6 +323,12 @@ void __gcov_module_copychain(gcov_info* chain)
         // Advance to next info in this modules chain.
         chain = chain->next;
     }
+#endif // #if HOSTBOOT_GCOV_EAGER_DATA_EXTRACTION
+
+    /* Then we set the module info pointer to NULL so that simics
+     * won't dump it twice. */
+
+    *chain_ptr = NULL;
 }
 #endif
 
diff --git a/src/kernel/makefile b/src/kernel/makefile
index 211a8b10e..c2f2e0451 100644
--- a/src/kernel/makefile
+++ b/src/kernel/makefile
@@ -5,7 +5,7 @@
 #
 # OpenPOWER HostBoot Project
 #
-# Contributors Listed Below - COPYRIGHT 2010,2018
+# Contributors Listed Below - COPYRIGHT 2010,2019
 # [+] International Business Machines Corp.
 #
 #
@@ -24,6 +24,9 @@
 # IBM_PROLOG_END_TAG
 ROOTPATH = ../..
 
+# we need HBB to be small
+HOSTBOOT_PROFILE_NO_INSTRUMENT=1
+
 OBJS += start.o
 OBJS += kernel.o
 OBJS += console.o
diff --git a/src/lib/makefile b/src/lib/makefile
index 406d5ead8..adcfb0c2b 100644
--- a/src/lib/makefile
+++ b/src/lib/makefile
@@ -24,6 +24,8 @@
 # IBM_PROLOG_END_TAG
 ROOTPATH = ../..
 
+VPATH += ${ROOTPATH}/src/sys/prof
+
 OBJS += string.o
 OBJS += string_ext.o
 OBJS += string_utils.o
@@ -52,6 +54,9 @@ OBJS += tls.o
 OBJS += errno.o
 OBJS += tlsrt.o
 
+# Don't care about instrumenting basic library functions
+HOSTBOOT_PROFILE=
+
 ifdef HOSTBOOT_MEMORY_LEAKS
 COMMONFLAGS += -DHOSTBOOT_MEMORY_LEAKS=1
 endif
diff --git a/src/libc++/makefile b/src/libc++/makefile
index 48ffe5be2..18fb2f852 100644
--- a/src/libc++/makefile
+++ b/src/libc++/makefile
@@ -5,7 +5,9 @@
 #
 # OpenPOWER HostBoot Project
 #
-# COPYRIGHT International Business Machines Corp. 2010,2014
+# Contributors Listed Below - COPYRIGHT 2010,2019
+# [+] International Business Machines Corp.
+#
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,6 +24,8 @@
 # IBM_PROLOG_END_TAG
 ROOTPATH = ../..
 
+HOSTBOOT_PROFILE=
+
 OBJS += builtins.o
 
 include ${ROOTPATH}/config.mk
diff --git a/src/makefile b/src/makefile
index 460485629..48f4b30be 100644
--- a/src/makefile
+++ b/src/makefile
@@ -59,10 +59,6 @@ BASE_OBJECTS += sprintf.o
 BASE_OBJECTS += crc32.o
 BASE_OBJECTS += utilmisc.o
 
-ifdef HOSTBOOT_PROFILE
-BASE_OBJECTS += gcov.o
-endif
-
 BL_BASE_OBJECTS += bl_builtins.o
 
 BOOTLDR_OBJECTS += bl_start.o
@@ -135,6 +131,9 @@ DIRECT_BOOT_OBJECTS += assert.o
 DIRECT_BOOT_OBJECTS += workitem.o
 DIRECT_BOOT_OBJECTS += bltohbdatamgr.o
 DIRECT_BOOT_OBJECTS += errno.o
+ifdef HOSTBOOT_PROFILE
+DIRECT_BOOT_OBJECTS += gcov.o
+endif
 
 BASE_MODULES += trace
 BASE_MODULES += errl
@@ -150,6 +149,11 @@ BASE_MODULES += vfs
 BASE_MODULES += $(if $(CONFIG_AST2400) || $(CONFIG_AST2500), sio)
 BASE_MODULES += $(if $(CONFIG_CONSOLE),console)
 
+# This is exported so that gcov knows the list of base modules to
+# exclude from instrumentation. We can't simply export BASE_MODULES or
+# else we get duplicate modules in the list which causes linker errors.
+export BASE_MODULES_GCOV_BLACKLIST:=$(BASE_MODULES)
+
 EXTENDED_MODULES += istep06
 EXTENDED_MODULES += istep07
 EXTENDED_MODULES += istep08
@@ -283,7 +287,12 @@ RUNTIME_OBJECTS += rt_assert.o
 RUNTIME_OBJECTS += rt_vfs.o
 RUNTIME_OBJECTS += rt_task.o
 RUNTIME_OBJECTS += rt_time.o
-RUNTIME_OBJECTS += ${RUNTIME_COMMON_OBJS}
+RUNTIME_OBJECTS += runtime_utils.o
+ifdef HOSTBOOT_PROFILE
+# we don't instrument runtime because we don't have space, but we
+# still link this in because it uses some object files that need it
+RUNTIME_OBJECTS += gcov.o
+endif
 
 RUNTIME_MODULES += trace_rt
 RUNTIME_MODULES += errl_rt
@@ -314,6 +323,11 @@ RUNTIME_MODULES += $(if $(CONFIG_NVDIMM),nvdimm_rt)
 RUNTIME_MODULES += mss_rt
 RUNTIME_MODULES += expaccess_rt
 
+# This is exported so that gcov knows the list of runtime modules to
+# exclude from instrumentation. We can't simply export RUNTIME_MODULES or
+# else we get duplicate modules in the list which causes linker errors.
+export RUNTIME_MODULES_GCOV_BLACKLIST:=$(RUNTIME_MODULES)
+
 RUNTIME_DATA_MODULES +=
 
 RUNTIME_TESTCASE_MODULES += cxxtest_rt
@@ -410,10 +424,10 @@ $(IMGDIR)/hbotStringFile : $(IMAGES)
 #20K for bootloader code and data)
 #PROCESS: get size of hbibl.bin, sort with respect to 32k (32768),
 #then see if last word is 32k.  If not, the bootloader image is too big.
+
 MAX_BASE_SIZE = 925696
 MAX_BTLDR_SIZE = 32768
+
 imgsizecheck: ${IMGDIR}/hbicore.bin ${IMGDIR}/hbibl.bin
 	$(if $(findstring $(shell (stat -c%s ${IMGDIR}/hbicore.bin; echo $(MAX_BASE_SIZE)) | sort -n | tail -n1), $(MAX_BASE_SIZE)),true, @echo ERROR: ${IMGDIR}/hbicore.bin too large.  Max allowed size is $(MAX_BASE_SIZE); false)
 	$(if $(findstring $(shell (stat -c%s ${IMGDIR}/hbibl.bin; echo $(MAX_BTLDR_SIZE)) | sort -n | tail -n1), $(MAX_BTLDR_SIZE)),true, @echo ERROR: ${IMGDIR}/hbibl.bin too large.  Max allowed size is $(MAX_BTLDR_SIZE); false)
-
-
diff --git a/src/runtime/makefile b/src/runtime/makefile
index 981e00499..407c58a7f 100644
--- a/src/runtime/makefile
+++ b/src/runtime/makefile
@@ -5,7 +5,7 @@
 #
 # OpenPOWER HostBoot Project
 #
-# Contributors Listed Below - COPYRIGHT 2013,2017
+# Contributors Listed Below - COPYRIGHT 2013,2019
 # [+] International Business Machines Corp.
 #
 #
@@ -23,6 +23,12 @@
 #
 # IBM_PROLOG_END_TAG
 HOSTBOOT_RUNTIME = 1
+
+# Profiling the runtime makes it too big, but we should revisit this
+# later after we reduce our memory footprint because it would be good
+# to instrument this too.
+HOSTBOOT_PROFILE=
+
 ROOTPATH = ../..
 include ../usr/runtime/common/common.mk
 VPATH += ../usr/runtime/common
diff --git a/src/securerom/makefile b/src/securerom/makefile
index 295330217..925886294 100644
--- a/src/securerom/makefile
+++ b/src/securerom/makefile
@@ -5,7 +5,7 @@
 #
 # OpenPOWER HostBoot Project
 #
-# Contributors Listed Below - COPYRIGHT 2016,2017
+# Contributors Listed Below - COPYRIGHT 2016,2019
 # [+] International Business Machines Corp.
 #
 #
@@ -25,6 +25,8 @@
 
 ROOTPATH = ../..
 
+HOSTBOOT_PROFILE=
+
 COMMONFLAGS += -DBYTE_ORDER=BIG_ENDIAN
 COMMONFLAGS += -DBN_POWER64_MUL
 COMMONFLAGS += -DBN_POWER64_CMP