diff options
Diffstat (limited to 'poky/meta/classes/reproducible_build.bbclass')
| -rw-r--r-- | poky/meta/classes/reproducible_build.bbclass | 180 |
1 files changed, 98 insertions, 82 deletions
diff --git a/poky/meta/classes/reproducible_build.bbclass b/poky/meta/classes/reproducible_build.bbclass index 2df805330..0eb696ac7 100644 --- a/poky/meta/classes/reproducible_build.bbclass +++ b/poky/meta/classes/reproducible_build.bbclass @@ -1,33 +1,35 @@ -# # reproducible_build.bbclass # -# This bbclass is mainly responsible to determine SOURCE_DATE_EPOCH on a per recipe base. -# We need to set a recipe specific SOURCE_DATE_EPOCH in each recipe environment for various tasks. -# One way would be to modify all recipes one-by-one to specify SOURCE_DATE_EPOCH explicitly, -# but that is not realistic as there are hundreds (probably thousands) of recipes in various meta-layers. -# Therefore we do it this class. -# After sources are unpacked but before they are patched, we try to determine the value for SOURCE_DATE_EPOCH. +# Sets SOURCE_DATE_EPOCH in each component's build environment. +# Upstream components (generally) respect this environment variable, +# using it in place of the "current" date and time. +# See https://reproducible-builds.org/specs/source-date-epoch/ +# +# After sources are unpacked but before they are patched, we set a reproducible value for SOURCE_DATE_EPOCH. +# This value should be reproducible for anyone who builds the same revision from the same sources. # -# There are 4 ways to determine SOURCE_DATE_EPOCH: +# There are 4 ways we determine SOURCE_DATE_EPOCH: # -# 1. Use value from __source_date_epoch.txt file if this file exists. -# This file was most likely created in the previous build by one of the following methods 2,3,4. -# In principle, it could actually provided by a recipe via SRC_URI +# 1. Use the value from __source_date_epoch.txt file if this file exists. +# This file was most likely created in the previous build by one of the following methods 2,3,4. +# Alternatively, it can be provided by a recipe via SRC_URI. # -# If the file does not exist, first try to determine the value for SOURCE_DATE_EPOCH: +# If the file does not exist: # -# 2. If we detected a folder .git, use .git last commit date timestamp, as git does not allow checking out -# files and preserving their timestamps. +# 2. If there is a git checkout, use the last git commit timestamp. +# Git does not preserve file timestamps on checkout. # # 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ... -# This will work fine for any well kept repository distributed via tarballs. +# This works for well-kept repositories distributed via tarball. +# +# 4. If the above steps fail, use the modification time of the youngest file in the source tree. # -# 4. If the above steps fail, we need to check all package source files and use the youngest file of the source tree. +# Once the value of SOURCE_DATE_EPOCH is determined, it is stored in the recipe's SDE_FILE. +# If none of these mechanisms are suitable, replace the do_deploy_source_date_epoch task +# with recipe-specific functionality to write the appropriate SOURCE_DATE_EPOCH into the SDE_FILE. # -# Once the value of SOURCE_DATE_EPOCH is determined, it is stored in the recipe ${WORKDIR}/source_date_epoch folder -# in a text file "__source_date_epoch.txt'. If this file is found by other recipe task, the value is exported in -# the SOURCE_DATE_EPOCH variable in the task environment. This is done in an anonymous python function, -# so SOURCE_DATE_EPOCH is guaranteed to exist for all tasks the may use it (do_configure, do_compile, do_package, ...) +# If this file is found by other tasks, the value is exported in the SOURCE_DATE_EPOCH variable. +# SOURCE_DATE_EPOCH is set for all tasks that might use it (do_configure, do_compile, do_package, ...) BUILD_REPRODUCIBLE_BINARIES ??= '1' inherit ${@oe.utils.ifelse(d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1', 'reproducible_build_simple', '')} @@ -50,86 +52,100 @@ do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DIR}" addtask do_deploy_source_date_epoch_setscene addtask do_deploy_source_date_epoch before do_configure after do_patch -def get_source_date_epoch_known_files(d, path): - source_date_epoch = 0 +def get_source_date_epoch_from_known_files(d, sourcedir): + source_date_epoch = None + newest_file = None known_files = set(["NEWS", "ChangeLog", "Changelog", "CHANGES"]) for file in known_files: - filepath = os.path.join(path,file) + filepath = os.path.join(sourcedir, file) if os.path.isfile(filepath): - mtime = int(os.path.getmtime(filepath)) + mtime = int(os.lstat(filepath).st_mtime) # There may be more than one "known_file" present, if so, use the youngest one - if mtime > source_date_epoch: + if not source_date_epoch or mtime > source_date_epoch: source_date_epoch = mtime + newest_file = filepath + if newest_file: + bb.debug(1, "SOURCE_DATE_EPOCH taken from: %s" % newest_file) return source_date_epoch -def find_git_folder(path): - exclude = set(["temp", "license-destdir", "patches", "recipe-sysroot-native", "recipe-sysroot", "pseudo", "build", "image", "sysroot-destdir"]) - for root, dirs, files in os.walk(path, topdown=True): +def find_git_folder(d, sourcedir): + # First guess: WORKDIR/git + # This is the default git fetcher unpack path + workdir = d.getVar('WORKDIR') + gitpath = os.path.join(workdir, "git/.git") + if os.path.isdir(gitpath): + return gitpath + + # Second guess: ${S} + gitpath = os.path.join(sourcedir, ".git") + if os.path.isdir(gitpath): + return gitpath + + # Perhaps there was a subpath or destsuffix specified. + # Go looking in the WORKDIR + exclude = set(["build", "image", "license-destdir", "patches", "pseudo", + "recipe-sysroot", "recipe-sysroot-native", "sysroot-destdir", "temp"]) + for root, dirs, files in os.walk(workdir, topdown=True): dirs[:] = [d for d in dirs if d not in exclude] if '.git' in dirs: - #bb.warn("found root:%s" % (str(root))) return root - -def get_source_date_epoch_git(d, path): - source_date_epoch = 0 + + bb.warn("Failed to find a git repository in WORKDIR: %s" % workdir) + return None + +def get_source_date_epoch_from_git(d, sourcedir): + source_date_epoch = None if "git://" in d.getVar('SRC_URI'): - gitpath = find_git_folder(d.getVar('WORKDIR')) - if gitpath != None: + gitpath = find_git_folder(d, sourcedir) + if gitpath: import subprocess - if os.path.isdir(os.path.join(gitpath,".git")): - try: - source_date_epoch = int(subprocess.check_output(['git','log','-1','--pretty=%ct'], cwd=path)) - #bb.warn("JB *** gitpath:%s sde: %d" % (gitpath,source_date_epoch)) - bb.debug(1, "git repo path:%s sde: %d" % (gitpath,source_date_epoch)) - except subprocess.CalledProcessError as grepexc: - #bb.warn( "Expected git repository not found, (path: %s) error:%d" % (gitpath, grepexc.returncode)) - bb.debug(1, "Expected git repository not found, (path: %s) error:%d" % (gitpath, grepexc.returncode)) - else: - bb.warn("Failed to find a git repository for path:%s" % (path)) + source_date_epoch = int(subprocess.check_output(['git','log','-1','--pretty=%ct'], cwd=gitpath)) + bb.debug(1, "git repository: %s" % gitpath) return source_date_epoch - -python do_create_source_date_epoch_stamp() { - path = d.getVar('S') - if not os.path.isdir(path): - bb.warn("Unable to determine source_date_epoch! path:%s" % path) - return +def get_source_date_epoch_from_youngest_file(d, sourcedir): + # Do it the hard way: check all files and find the youngest one... + source_date_epoch = None + newest_file = None + # Just in case S = WORKDIR + exclude = set(["build", "image", "license-destdir", "patches", "pseudo", + "recipe-sysroot", "recipe-sysroot-native", "sysroot-destdir", "temp"]) + for root, dirs, files in os.walk(sourcedir, topdown=True): + files = [f for f in files if not f[0] == '.'] + dirs[:] = [d for d in dirs if d not in exclude] + + for fname in files: + filename = os.path.join(root, fname) + try: + mtime = int(os.lstat(filename).st_mtime) + except ValueError: + mtime = 0 + if not source_date_epoch or mtime > source_date_epoch: + source_date_epoch = mtime + newest_file = filename + + if newest_file: + bb.debug(1, "Newest file found: %s" % newest_file) + return source_date_epoch + +python do_create_source_date_epoch_stamp() { epochfile = d.getVar('SDE_FILE') if os.path.isfile(epochfile): - bb.debug(1, " path: %s reusing __source_date_epoch.txt" % epochfile) + bb.debug(1, "Reusing SOURCE_DATE_EPOCH from: %s" % epochfile) return - - # Try to detect/find a git repository - source_date_epoch = get_source_date_epoch_git(d, path) - if source_date_epoch == 0: - source_date_epoch = get_source_date_epoch_known_files(d, path) + + sourcedir = d.getVar('S') + source_date_epoch = ( + get_source_date_epoch_from_git(d, sourcedir) or + get_source_date_epoch_from_known_files(d, sourcedir) or + get_source_date_epoch_from_youngest_file(d, sourcedir) or + 0 # Last resort + ) if source_date_epoch == 0: - # Do it the hard way: check all files and find the youngest one... - filename_dbg = None - exclude = set(["temp", "license-destdir", "patches", "recipe-sysroot-native", "recipe-sysroot", "pseudo", "build", "image", "sysroot-destdir"]) - for root, dirs, files in os.walk(path, topdown=True): - files = [f for f in files if not f[0] == '.'] - dirs[:] = [d for d in dirs if d not in exclude] - - for fname in files: - filename = os.path.join(root, fname) - try: - mtime = int(os.path.getmtime(filename)) - except ValueError: - mtime = 0 - if mtime > source_date_epoch: - source_date_epoch = mtime - filename_dbg = filename - - if filename_dbg != None: - bb.debug(1," SOURCE_DATE_EPOCH %d derived from: %s" % (source_date_epoch, filename_dbg)) - - if source_date_epoch == 0: - # empty folder, not a single file ... - # kernel source do_unpack is special cased - if not bb.data.inherits_class('kernel', d): - bb.debug(1, "Unable to determine source_date_epoch! path:%s" % path) + # empty folder, not a single file ... + bb.debug(1, "No files found to determine SOURCE_DATE_EPOCH") + bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch) bb.utils.mkdirhier(d.getVar('SDE_DIR')) with open(epochfile, 'w') as f: f.write(str(source_date_epoch)) @@ -145,6 +161,6 @@ python () { if os.path.isfile(epochfile): with open(epochfile, 'r') as f: source_date_epoch = f.read() - bb.debug(1, "source_date_epoch stamp found ---> stamp %s" % source_date_epoch) + bb.debug(1, "SOURCE_DATE_EPOCH: %s" % source_date_epoch) d.setVar('SOURCE_DATE_EPOCH', source_date_epoch) } |
