From 7838a7409307f1106aaa6df42f1909e6824de197 Mon Sep 17 00:00:00 2001 From: Patrick Williams Date: Thu, 1 Mar 2012 13:52:34 -0600 Subject: Tools for managing binary files in git. Change-Id: I420803b36707e877a4ba35f6f484c74c9ed4eb43 Reviewed-on: http://gfw160.austin.ibm.com:8080/gerrit/711 Tested-by: Jenkins Server Reviewed-by: Terry J. Opie Reviewed-by: Thi N. Tran Reviewed-by: A. Patrick Williams III --- config.mk | 45 ++++++++++++++++++++++++++-- img/sbe_pnor.bin | Bin 31360 -> 0 bytes src/build/tools/hb | 76 +++++++++++++++++++++++++++++++++++++++++++++++- src/usr/pore/makefile | 2 ++ src/usr/spd/dimmspd.dat | Bin 257 -> 0 bytes src/usr/spd/makefile | 6 +--- 6 files changed, 120 insertions(+), 9 deletions(-) delete mode 100755 img/sbe_pnor.bin delete mode 100644 src/usr/spd/dimmspd.dat diff --git a/config.mk b/config.mk index 9c518b290..9f46a3274 100644 --- a/config.mk +++ b/config.mk @@ -97,6 +97,9 @@ GCOV = ppc64-mcp6-gcov APYFIPSHDR = apyfipshdr APYRUHHDR = apyruhhdr +BINFILE_CACHE_LOCALDIR = ${ROOTPATH}/.git/hb_cache/data/ +BINFILE_CACHE_REMOTEDIR = /gsa/ausgsa/projects/h/hostboot/.binary_cache/data/ + BEAMVER = beam-3.5.2 BEAMPATH = /afs/rch/projects/esw/beam/${BEAMVER} BEAMCMD = i686-mcp6-jail ${BEAMPATH}/bin/beam_compile @@ -233,6 +236,42 @@ ${OBJDIR}/${MODULE}.C: ${TESTS} ${TESTGEN} --hostboot -o $@ ${TESTS} endif +# Rules for BINARY_FILES directive. +# +# The BINARY_FILES directives are used to include files out of the binary +# files cache (see 'hb cacheadd' command). This cache exists to keep +# binary files outside of git, because they take a larger space in the git +# database, especially if they change frequently. +# +# The BINARY_FILES variable is a set of : pairs. +# The destination is where the make system should put the file. The hash +# value tells which version of a file to use and it comes from the +# 'hb cacheadd' tool when a version of the file is added to the binary +# files cache. +# +define __BINARY_CACHE_FILE +_BINARYFILES += $(1) + +ifneq "$(wildcard $(addprefix ${BINFILE_CACHE_LOCALDIR},$(2)))" "" +$(1) : $(addprefix $${BINFILE_CACHE_LOCALDIR},$(2)) + echo "$(2) $$<" | sha1sum --check + cp $$< $$@ +else +$(1) : $(addprefix $${BINFILE_CACHE_REMOTEDIR},$(2)) + echo "$(2) $$<" | sha1sum --check + cp $$< $$@ +endif +endef +ifdef BINARY_FILES +$(foreach file,$(BINARY_FILES), \ + $(eval $(call __BINARY_CACHE_FILE, \ + $(firstword $(subst :, ,$(file))), \ + $(lastword $(subst :, ,$(file))) \ + )) \ +) +endif +# end BINARY_FILES rules. + define ELF_template $${IMGDIR}/$(1).elf: $$(addprefix $${OBJDIR}/, $$($(1)_OBJECTS)) \ $${ROOTPATH}/src/kernel.ld @@ -301,8 +340,8 @@ gen_pass: mkdir -p ${GENDIR} ${MAKE} GEN_PASS -_GENFILES = $(addprefix ${GENDIR}/, ${GENFILES}) -GEN_PASS: ${_GENFILES} ${SUBDIRS:.d=.gen_pass} +_GENFILES = $(addprefix ${GENDIR}/, $(GENFILES)) +GEN_PASS: $(_GENFILES) $(_BINARYFILES) ${SUBDIRS:.d=.gen_pass} GENTARGET = $(addprefix %/, $(1)) @@ -350,7 +389,7 @@ clean: cleanud ${SUBDIRS:.d=.clean} ${IMAGES:.bin=.bin.modinfo} ${IMAGES:.ruhx=.lid} \ ${IMAGES:.ruhx=.lidhdr} ${IMAGES:.bin=_extended.bin} \ ${IMAGE_EXTRAS} ${EXTRA_LIDS_} \ - ${EXTRA_OBJS} ${_GENFILES} ${EXTRA_PARTS} ${EXTRA_CLEAN} ) + ${EXTRA_OBJS} ${_GENFILES} ${_BINARYFILES} ${EXTRA_PARTS} ${EXTRA_CLEAN} ) cscope: ${SUBDIRS} mkdir -p ${ROOTPATH}/obj/cscope diff --git a/img/sbe_pnor.bin b/img/sbe_pnor.bin deleted file mode 100755 index 1671aa81e..000000000 Binary files a/img/sbe_pnor.bin and /dev/null differ diff --git a/src/build/tools/hb b/src/build/tools/hb index 9179966e9..0ea3cdad7 100755 --- a/src/build/tools/hb +++ b/src/build/tools/hb @@ -175,6 +175,26 @@ hb_helptext() echo " Prepares a copy of FipS errl tool that is hostboot aware," echo " places it in simics directory." ;; + cachesync) + echo " Topic 'cachesync'" + echo + echo " Usage:" + echo " hb cachesync" + echo + echo " Synchronizes the repository's binary file cache with the" + echo " project repository." + ;; + cacheadd) + echo " Topic 'cacheadd'" + echo + echo " Usage:" + echo " hb cacheadd [--local] \"\"" + echo + echo " Inserts a file into the project binary file cache." + echo + echo " Options:" + echo " --local: Indicates file should be put only in local cache." + ;; customrc) echo " Topic 'customrc'" echo @@ -205,7 +225,7 @@ hb_helptext() echo echo " Available Commands:" echo " workon, simsetup, prime, startsimics, rsync, objsizes," - echo " copyright_check, errlparser" + echo " copyright_check, errlparser, cachesync, cacheadd" echo echo " Additional Help Topics:" echo " customrc" @@ -474,10 +494,58 @@ hb_errlparser() ln -sf ../obj/x86.nfp/errl/nfp/tool/errl } +hb_cachesync() +{ + [ -z "${HOSTBOOTROOT}" ] && echo "Missing HOSTBOOTROOT." && exit -1 + + rsync -av /gsa/ausgsa/projects/h/hostboot/.binary_cache/ \ + ${HOSTBOOTROOT}/.git/hb_cache + + failure=0 + for files in ${HOSTBOOTROOT}/.git/hb_cache/data/*; do + echo "`basename $files` $files" | sha1sum --check >> /dev/null + if [ $? -ne 0 ]; then + echo "Failed SHA1 verification! $files" + failure=-1 + fi + done + + exit $failure +} + +hb_cacheadd() +{ + [ -z "${HOSTBOOTROOT}" ] && echo "Missing HOSTBOOTROOT." && exit -1 + + CACHE_PATH=/gsa/ausgsa/projects/h/hostboot/.binary_cache/ + FILE=$1 + shift + if [ $FILE == "--local" ]; then + echo "Adding as local copy." + CACHE_PATH=${HOSTBOOTROOT}/.git/hb_cache/ + + FILE=$1 + shift + fi + MESSAGE=$* + [ ! -e "${FILE}" ] && echo "File $FILE does not exist." && exit -1 + [ -z "${MESSAGE}" ] && echo "No message given." && exit -1 + FILE_BASE=`basename $FILE` + SHA1SUM=`sha1sum -b $FILE | sed 's/ .*//'` + + echo $FILE $SHA1SUM + + cp $FILE $CACHE_PATH/data/$SHA1SUM + echo $SHA1SUM : `whoami` : `date` : $MESSAGE \ + >> $CACHE_PATH/files/$FILE_BASE + + echo "Added $FILE_BASE as $SHA1SUM" + +} @@ -514,6 +582,12 @@ copyright_check) errlparser) hb_errlparser $* ;; +cachesync) + hb_cachesync $* + ;; +cacheadd) + hb_cacheadd $* + ;; *) hb_helptext $* exit -1 diff --git a/src/usr/pore/makefile b/src/usr/pore/makefile index c544e8efb..6097b74cb 100644 --- a/src/usr/pore/makefile +++ b/src/usr/pore/makefile @@ -24,4 +24,6 @@ ROOTPATH = ../../.. SUBDIRS = fapiporeve.d poreve.d test.d +BINARY_FILES = $(IMGDIR)/sbe_pnor.bin:76398b3859c8395b0e73c2df9a76ee28bad185fc + include ${ROOTPATH}/config.mk diff --git a/src/usr/spd/dimmspd.dat b/src/usr/spd/dimmspd.dat deleted file mode 100644 index cb4f85c83..000000000 Binary files a/src/usr/spd/dimmspd.dat and /dev/null differ diff --git a/src/usr/spd/makefile b/src/usr/spd/makefile index 9cb0b14cb..9c9f96d5b 100644 --- a/src/usr/spd/makefile +++ b/src/usr/spd/makefile @@ -27,10 +27,6 @@ OBJS = spd.o dimmPres.o SUBDIRS = test.d -DIMM_SPD_DATA = dimmspd.dat -EXTRA_PARTS = $(addprefix $(IMGDIR)/, $(DIMM_SPD_DATA)) +BINARY_FILES = $(IMGDIR)/dimmspd.dat:70ba15165f78487ed870333891ae983167afd546 include ${ROOTPATH}/config.mk - -${EXTRA_PARTS}: ${IMGDIR}/% : ./% - cp -f $^ $@ -- cgit v1.2.1