Tools for managing binary files in git.

Change-Id: I420803b36707e877a4ba35f6f484c74c9ed4eb43
Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/711
Tested-by: Jenkins Server
Reviewed-by: Terry J. Opie <opiet@us.ibm.com>
Reviewed-by: Thi N. Tran <thi@us.ibm.com>
Reviewed-by: A. Patrick Williams III <iawillia@us.ibm.com>

6 files changed, 120 insertions, 9 deletions

diff --git a/config.mk b/config.mk
index 9c518b290..9f46a3274 100644
--- a/config.mk
+++ b/config.mk
@@ -97,6 +97,9 @@ GCOV = ppc64-mcp6-gcov
 APYFIPSHDR = apyfipshdr
 APYRUHHDR = apyruhhdr
 
+BINFILE_CACHE_LOCALDIR = ${ROOTPATH}/.git/hb_cache/data/
+BINFILE_CACHE_REMOTEDIR = /gsa/ausgsa/projects/h/hostboot/.binary_cache/data/
+
 BEAMVER = beam-3.5.2
 BEAMPATH = /afs/rch/projects/esw/beam/${BEAMVER}
 BEAMCMD = i686-mcp6-jail ${BEAMPATH}/bin/beam_compile
@@ -233,6 +236,42 @@ ${OBJDIR}/${MODULE}.C: ${TESTS}
 	${TESTGEN} --hostboot -o $@ ${TESTS}
 endif
 
+# Rules for BINARY_FILES directive.
+#
+#    The BINARY_FILES directives are used to include files out of the binary
+#    files cache (see 'hb cacheadd' command).  This cache exists to keep
+#    binary files outside of git, because they take a larger space in the git
+#    database, especially if they change frequently.
+#
+#    The BINARY_FILES variable is a set of <destination>:<hash_value> pairs.
+#    The destination is where the make system should put the file.  The hash
+#    value tells which version of a file to use and it comes from the 
+#    'hb cacheadd' tool when a version of the file is added to the binary
+#    files cache.
+#
+define __BINARY_CACHE_FILE
+_BINARYFILES += $(1)
+
+ifneq "$(wildcard $(addprefix ${BINFILE_CACHE_LOCALDIR},$(2)))" ""
+$(1) : $(addprefix $${BINFILE_CACHE_LOCALDIR},$(2))
+	echo "$(2) $$<" | sha1sum --check
+	cp $$< $$@
+else
+$(1) : $(addprefix $${BINFILE_CACHE_REMOTEDIR},$(2))
+	echo "$(2) $$<" | sha1sum --check
+	cp $$< $$@
+endif
+endef
+ifdef BINARY_FILES
+$(foreach file,$(BINARY_FILES), \
+	  $(eval $(call __BINARY_CACHE_FILE, \
+			    $(firstword $(subst :, ,$(file))), \
+			    $(lastword $(subst :, ,$(file))) \
+	  )) \
+)
+endif
+# end BINARY_FILES rules.
+
 define ELF_template
 $${IMGDIR}/$(1).elf: $$(addprefix $${OBJDIR}/, $$($(1)_OBJECTS)) \
 		    $${ROOTPATH}/src/kernel.ld
@@ -301,8 +340,8 @@ gen_pass:
 	mkdir -p ${GENDIR}
 	${MAKE} GEN_PASS
 
-_GENFILES = $(addprefix ${GENDIR}/, ${GENFILES})
-GEN_PASS: ${_GENFILES} ${SUBDIRS:.d=.gen_pass}
+_GENFILES = $(addprefix ${GENDIR}/, $(GENFILES))
+GEN_PASS: $(_GENFILES) $(_BINARYFILES) ${SUBDIRS:.d=.gen_pass}
 
 GENTARGET = $(addprefix %/, $(1))
 
@@ -350,7 +389,7 @@ clean: cleanud ${SUBDIRS:.d=.clean}
 	       ${IMAGES:.bin=.bin.modinfo} ${IMAGES:.ruhx=.lid} \
 	       ${IMAGES:.ruhx=.lidhdr} ${IMAGES:.bin=_extended.bin} \
 	       ${IMAGE_EXTRAS} ${EXTRA_LIDS_} \
-	       ${EXTRA_OBJS} ${_GENFILES} ${EXTRA_PARTS} ${EXTRA_CLEAN} )
+	       ${EXTRA_OBJS} ${_GENFILES} ${_BINARYFILES} ${EXTRA_PARTS} ${EXTRA_CLEAN} )
 
 cscope: ${SUBDIRS}
 	mkdir -p ${ROOTPATH}/obj/cscope
diff --git a/img/sbe_pnor.bin b/img/sbe_pnor.bin
deleted file mode 100755
index 1671aa81e..000000000
--- a/img/sbe_pnor.bin
+++ /dev/null
diff --git a/src/build/tools/hb b/src/build/tools/hb
index 9179966e9..0ea3cdad7 100755
--- a/src/build/tools/hb
+++ b/src/build/tools/hb
@@ -175,6 +175,26 @@ hb_helptext()
         echo "  Prepares a copy of FipS errl tool that is hostboot aware,"
         echo "  places it in simics directory."
         ;;
+    cachesync)
+        echo "  Topic 'cachesync'"
+        echo
+        echo "  Usage:"
+        echo "     hb cachesync"
+        echo
+        echo "  Synchronizes the repository's binary file cache with the"
+        echo "  project repository."
+        ;;
+    cacheadd)
+        echo "  Topic 'cacheadd'"
+        echo
+        echo "  Usage:"
+        echo "     hb cacheadd [--local] <file> \"<msg>\""
+        echo
+        echo "  Inserts a file into the project binary file cache."
+        echo
+        echo "  Options:"
+        echo "     --local: Indicates file should be put only in local cache."
+        ;;
     customrc)
         echo "  Topic 'customrc'"
         echo
@@ -205,7 +225,7 @@ hb_helptext()
         echo
         echo "  Available Commands:"
         echo "      workon, simsetup, prime, startsimics, rsync, objsizes,"
-        echo "      copyright_check, errlparser"
+        echo "      copyright_check, errlparser, cachesync, cacheadd"
         echo
         echo "  Additional Help Topics:"
         echo "      customrc"
@@ -474,10 +494,58 @@ hb_errlparser()
     ln -sf ../obj/x86.nfp/errl/nfp/tool/errl
 }
 
+hb_cachesync()
+{
+    [ -z "${HOSTBOOTROOT}" ] && echo "Missing HOSTBOOTROOT." && exit -1
+
+    rsync -av /gsa/ausgsa/projects/h/hostboot/.binary_cache/ \
+              ${HOSTBOOTROOT}/.git/hb_cache
+
+    failure=0
+    for files in ${HOSTBOOTROOT}/.git/hb_cache/data/*; do
+        echo "`basename $files`  $files" | sha1sum --check >> /dev/null
+        if [ $? -ne 0 ]; then
+            echo "Failed SHA1 verification! $files"
+            failure=-1
+        fi
+    done
+
+    exit $failure
+}
+
+hb_cacheadd()
+{
+    [ -z "${HOSTBOOTROOT}" ] && echo "Missing HOSTBOOTROOT." && exit -1    
+
+    CACHE_PATH=/gsa/ausgsa/projects/h/hostboot/.binary_cache/
+    FILE=$1
+    shift
 
+    if [ $FILE == "--local" ]; then
+        echo "Adding as local copy."
+        CACHE_PATH=${HOSTBOOTROOT}/.git/hb_cache/
+
+        FILE=$1
+        shift
+    fi
 
+    MESSAGE=$*
 
+    [ ! -e "${FILE}" ] && echo "File $FILE does not exist." && exit -1
+    [ -z "${MESSAGE}" ] && echo "No message given." && exit -1
 
+    FILE_BASE=`basename $FILE`
+    SHA1SUM=`sha1sum -b $FILE | sed 's/ .*//'`
+
+    echo $FILE $SHA1SUM
+
+    cp $FILE $CACHE_PATH/data/$SHA1SUM
+    echo $SHA1SUM : `whoami` : `date` : $MESSAGE \
+            >> $CACHE_PATH/files/$FILE_BASE
+
+    echo "Added $FILE_BASE as $SHA1SUM"
+
+}
 
 
 
@@ -514,6 +582,12 @@ copyright_check)
 errlparser)
     hb_errlparser $*
     ;;
+cachesync)
+    hb_cachesync $*
+    ;;
+cacheadd)
+    hb_cacheadd $*
+    ;;
 *)
     hb_helptext $*
     exit -1
diff --git a/src/usr/pore/makefile b/src/usr/pore/makefile
index c544e8efb..6097b74cb 100644
--- a/src/usr/pore/makefile
+++ b/src/usr/pore/makefile
@@ -24,4 +24,6 @@ ROOTPATH = ../../..
 
 SUBDIRS = fapiporeve.d poreve.d test.d
 
+BINARY_FILES = $(IMGDIR)/sbe_pnor.bin:76398b3859c8395b0e73c2df9a76ee28bad185fc
+
 include ${ROOTPATH}/config.mk
diff --git a/src/usr/spd/dimmspd.dat b/src/usr/spd/dimmspd.dat
deleted file mode 100644
index cb4f85c83..000000000
--- a/src/usr/spd/dimmspd.dat
+++ /dev/null
diff --git a/src/usr/spd/makefile b/src/usr/spd/makefile
index 9cb0b14cb..9c9f96d5b 100644
--- a/src/usr/spd/makefile
+++ b/src/usr/spd/makefile
@@ -27,10 +27,6 @@ OBJS = spd.o dimmPres.o
 
 SUBDIRS = test.d
 
-DIMM_SPD_DATA = dimmspd.dat
-EXTRA_PARTS = $(addprefix $(IMGDIR)/, $(DIMM_SPD_DATA))
+BINARY_FILES = $(IMGDIR)/dimmspd.dat:70ba15165f78487ed870333891ae983167afd546
 
 include ${ROOTPATH}/config.mk
-
-${EXTRA_PARTS}: ${IMGDIR}/% : ./%
-	cp -f $^ $@