summaryrefslogtreecommitdiffstats
path: root/import-layers/yocto-poky/meta/classes/toaster.bbclass
diff options
context:
space:
mode:
Diffstat (limited to 'import-layers/yocto-poky/meta/classes/toaster.bbclass')
-rw-r--r--import-layers/yocto-poky/meta/classes/toaster.bbclass397
1 files changed, 397 insertions, 0 deletions
diff --git a/import-layers/yocto-poky/meta/classes/toaster.bbclass b/import-layers/yocto-poky/meta/classes/toaster.bbclass
new file mode 100644
index 000000000..1a70f14a9
--- /dev/null
+++ b/import-layers/yocto-poky/meta/classes/toaster.bbclass
@@ -0,0 +1,397 @@
+#
+# Toaster helper class
+#
+# Copyright (C) 2013 Intel Corporation
+#
+# Released under the MIT license (see COPYING.MIT)
+#
+# This bbclass is designed to extract data used by OE-Core during the build process,
+# for recording in the Toaster system.
+# The data access is synchronous, preserving the build data integrity across
+# different builds.
+#
+# The data is transferred through the event system, using the MetadataEvent objects.
+#
+# The model is to enable the datadump functions as postfuncs, and have the dump
+# executed after the real taskfunc has been executed. This prevents task signature changing
+# is toaster is enabled or not. Build performance is not affected if Toaster is not enabled.
+#
+# To enable, use INHERIT in local.conf:
+#
+# INHERIT += "toaster"
+#
+#
+#
+#
+
+# Find and dump layer info when we got the layers parsed
+
+
+
+python toaster_layerinfo_dumpdata() {
+ import subprocess
+
+ def _get_git_branch(layer_path):
+ branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0]
+ branch = branch.replace('refs/heads/', '').rstrip()
+ return branch
+
+ def _get_git_revision(layer_path):
+ revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip()
+ return revision
+
+ def _get_url_map_name(layer_name):
+ """ Some layers have a different name on openembedded.org site,
+ this method returns the correct name to use in the URL
+ """
+
+ url_name = layer_name
+ url_mapping = {'meta': 'openembedded-core'}
+
+ for key in url_mapping.keys():
+ if key == layer_name:
+ url_name = url_mapping[key]
+
+ return url_name
+
+ def _get_layer_version_information(layer_path):
+
+ layer_version_info = {}
+ layer_version_info['branch'] = _get_git_branch(layer_path)
+ layer_version_info['commit'] = _get_git_revision(layer_path)
+ layer_version_info['priority'] = 0
+
+ return layer_version_info
+
+
+ def _get_layer_dict(layer_path):
+
+ layer_info = {}
+ layer_name = layer_path.split('/')[-1]
+ layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/'
+ layer_url_name = _get_url_map_name(layer_name)
+
+ layer_info['name'] = layer_url_name
+ layer_info['local_path'] = layer_path
+ layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name)
+ layer_info['version'] = _get_layer_version_information(layer_path)
+
+ return layer_info
+
+
+ bblayers = e.data.getVar("BBLAYERS", True)
+
+ llayerinfo = {}
+
+ for layer in { l for l in bblayers.strip().split(" ") if len(l) }:
+ llayerinfo[layer] = _get_layer_dict(layer)
+
+
+ bb.event.fire(bb.event.MetadataEvent("LayerInfo", llayerinfo), e.data)
+}
+
+# Dump package file info data
+
+def _toaster_load_pkgdatafile(dirpath, filepath):
+ import json
+ import re
+ pkgdata = {}
+ with open(os.path.join(dirpath, filepath), "r") as fin:
+ for line in fin:
+ try:
+ kn, kv = line.strip().split(": ", 1)
+ m = re.match(r"^PKG_([^A-Z:]*)", kn)
+ if m:
+ pkgdata['OPKGN'] = m.group(1)
+ kn = "_".join([x for x in kn.split("_") if x.isupper()])
+ pkgdata[kn] = kv.strip()
+ if kn == 'FILES_INFO':
+ pkgdata[kn] = json.loads(kv)
+
+ except ValueError:
+ pass # ignore lines without valid key: value pairs
+ return pkgdata
+
+python toaster_package_dumpdata() {
+ """
+ Dumps the data about the packages created by a recipe
+ """
+
+ # No need to try and dumpdata if the recipe isn't generating packages
+ if not d.getVar('PACKAGES', True):
+ return
+
+ pkgdatadir = d.getVar('PKGDESTWORK', True)
+ lpkgdata = {}
+ datadir = os.path.join(pkgdatadir, 'runtime')
+
+ # scan and send data for each generated package
+ for datafile in os.listdir(datadir):
+ if not datafile.endswith('.packaged'):
+ lpkgdata = _toaster_load_pkgdatafile(datadir, datafile)
+ # Fire an event containing the pkg data
+ bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d)
+}
+
+# 2. Dump output image files information
+
+python toaster_image_dumpdata() {
+ """
+ Image filename for output images is not standardized.
+ image_types.bbclass will spell out IMAGE_CMD_xxx variables that actually
+ have hardcoded ways to create image file names in them.
+ So we look for files starting with the set name.
+
+ We also look for other files in the images/ directory which don't
+ match IMAGE_NAME, such as the kernel bzImage, modules tarball etc.
+ """
+
+ dir_to_walk = d.getVar('DEPLOY_DIR_IMAGE', True);
+ image_name = d.getVar('IMAGE_NAME', True);
+ image_info_data = {}
+ artifact_info_data = {}
+
+ # collect all images and artifacts in the images directory
+ for dirpath, dirnames, filenames in os.walk(dir_to_walk):
+ for filename in filenames:
+ full_path = os.path.join(dirpath, filename)
+ try:
+ if filename.startswith(image_name):
+ # image
+ image_info_data[full_path] = os.stat(full_path).st_size
+ else:
+ # other non-image artifact
+ if not os.path.islink(full_path):
+ artifact_info_data[full_path] = os.stat(full_path).st_size
+ except OSError as e:
+ bb.event.fire(bb.event.MetadataEvent("OSErrorException", e), d)
+
+ bb.event.fire(bb.event.MetadataEvent("ImageFileSize", image_info_data), d)
+ bb.event.fire(bb.event.MetadataEvent("ArtifactFileSize", artifact_info_data), d)
+}
+
+python toaster_artifact_dumpdata() {
+ """
+ Dump data about artifacts in the SDK_DEPLOY directory
+ """
+
+ dir_to_walk = d.getVar("SDK_DEPLOY", True)
+ artifact_info_data = {}
+
+ # collect all artifacts in the sdk directory
+ for dirpath, dirnames, filenames in os.walk(dir_to_walk):
+ for filename in filenames:
+ full_path = os.path.join(dirpath, filename)
+ try:
+ if not os.path.islink(full_path):
+ artifact_info_data[full_path] = os.stat(full_path).st_size
+ except OSError as e:
+ bb.event.fire(bb.event.MetadataEvent("OSErrorException", e), d)
+
+ bb.event.fire(bb.event.MetadataEvent("ArtifactFileSize", artifact_info_data), d)
+}
+
+# collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data
+
+python toaster_collect_task_stats() {
+ import bb.build
+ import bb.event
+ import bb.data
+ import bb.utils
+ import os
+
+ toaster_statlist_file = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")
+
+ if not e.data.getVar('BUILDSTATS_BASE', True):
+ return # if we don't have buildstats, we cannot collect stats
+
+ def stat_to_float(value):
+ return float(value.strip('% \n\r'))
+
+ def _append_read_list(v):
+ lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True)
+
+ with open(toaster_statlist_file, "a") as fout:
+ taskdir = e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}")
+ fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task), e.data.expand("${PN}")))
+
+ bb.utils.unlockfile(lock)
+
+ def _read_stats(filename):
+ # seconds
+ cpu_time_user = 0
+ cpu_time_system = 0
+
+ # bytes
+ disk_io_read = 0
+ disk_io_write = 0
+
+ started = 0
+ ended = 0
+
+ taskname = ''
+
+ statinfo = {}
+
+ with open(filename, 'r') as task_bs:
+ for line in task_bs.readlines():
+ k,v = line.strip().split(": ", 1)
+ statinfo[k] = v
+
+ if "Started" in statinfo:
+ started = stat_to_float(statinfo["Started"])
+
+ if "Ended" in statinfo:
+ ended = stat_to_float(statinfo["Ended"])
+
+ if "Child rusage ru_utime" in statinfo:
+ cpu_time_user = cpu_time_user + stat_to_float(statinfo["Child rusage ru_utime"])
+
+ if "Child rusage ru_stime" in statinfo:
+ cpu_time_system = cpu_time_system + stat_to_float(statinfo["Child rusage ru_stime"])
+
+ if "IO write_bytes" in statinfo:
+ write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r'))
+ disk_io_write = disk_io_write + write_bytes
+
+ if "IO read_bytes" in statinfo:
+ read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r'))
+ disk_io_read = disk_io_read + read_bytes
+
+ return {
+ 'stat_file': filename,
+ 'cpu_time_user': cpu_time_user,
+ 'cpu_time_system': cpu_time_system,
+ 'disk_io_read': disk_io_read,
+ 'disk_io_write': disk_io_write,
+ 'started': started,
+ 'ended': ended
+ }
+
+ if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)):
+ _append_read_list(e)
+ pass
+
+ if isinstance(e, bb.event.BuildCompleted) and os.path.exists(toaster_statlist_file):
+ events = []
+ with open(toaster_statlist_file, "r") as fin:
+ for line in fin:
+ (taskfile, taskname, filename, recipename) = line.strip().split("::")
+ stats = _read_stats(filename)
+ events.append((taskfile, taskname, stats, recipename))
+ bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data)
+ os.unlink(toaster_statlist_file)
+}
+
+# dump relevant build history data as an event when the build is completed
+
+python toaster_buildhistory_dump() {
+ import re
+ BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory")
+ BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR)
+ pkgdata_dir = e.data.getVar("PKGDATA_DIR", True)
+
+
+ # scan the build targets for this build
+ images = {}
+ allpkgs = {}
+ files = {}
+ for target in e._pkgs:
+ target = target.split(':')[0] # strip ':<task>' suffix from the target
+ installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target))
+ if os.path.exists(installed_img_path):
+ images[target] = {}
+ files[target] = {}
+ files[target]['dirs'] = []
+ files[target]['syms'] = []
+ files[target]['files'] = []
+ with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin:
+ for line in fin:
+ line = line.rstrip(";")
+ psize, px = line.split("\t")
+ punit, pname = px.split(" ")
+ # this size is "installed-size" as it measures how much space it takes on disk
+ images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []}
+
+ with open("%s/depends.dot" % installed_img_path, "r") as fin:
+ p = re.compile(r' -> ')
+ dot = re.compile(r'.*style=dotted')
+ for line in fin:
+ line = line.rstrip(';')
+ linesplit = p.split(line)
+ if len(linesplit) == 2:
+ pname = linesplit[0].rstrip('"').strip('"')
+ dependsname = linesplit[1].split(" ")[0].strip().strip(";").strip('"').rstrip('"')
+ deptype = "depends"
+ if dot.match(line):
+ deptype = "recommends"
+ if not pname in images[target]:
+ images[target][pname] = {'size': 0, 'depends' : []}
+ if not dependsname in images[target]:
+ images[target][dependsname] = {'size': 0, 'depends' : []}
+ images[target][pname]['depends'].append((dependsname, deptype))
+
+ with open("%s/files-in-image.txt" % installed_img_path, "r") as fin:
+ for line in fin:
+ lc = [ x for x in line.strip().split(" ") if len(x) > 0 ]
+ if lc[0].startswith("l"):
+ files[target]['syms'].append(lc)
+ elif lc[0].startswith("d"):
+ files[target]['dirs'].append(lc)
+ else:
+ files[target]['files'].append(lc)
+
+ for pname in images[target]:
+ if not pname in allpkgs:
+ try:
+ pkgdata = _toaster_load_pkgdatafile("%s/runtime-reverse/" % pkgdata_dir, pname)
+ except IOError as err:
+ if err.errno == 2:
+ # We expect this e.g. for RRECOMMENDS that are unsatisfied at runtime
+ continue
+ else:
+ raise
+ allpkgs[pname] = pkgdata
+
+
+ data = { 'pkgdata' : allpkgs, 'imgdata' : images, 'filedata' : files }
+
+ bb.event.fire(bb.event.MetadataEvent("ImagePkgList", data), e.data)
+
+}
+
+# dump information related to license manifest path
+
+python toaster_licensemanifest_dump() {
+ deploy_dir = d.getVar('DEPLOY_DIR', True);
+ image_name = d.getVar('IMAGE_NAME', True);
+
+ data = { 'deploy_dir' : deploy_dir, 'image_name' : image_name }
+
+ bb.event.fire(bb.event.MetadataEvent("LicenseManifestPath", data), d)
+}
+
+# set event handlers
+addhandler toaster_layerinfo_dumpdata
+toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted"
+
+addhandler toaster_collect_task_stats
+toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed"
+
+addhandler toaster_buildhistory_dump
+toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted"
+
+do_packagedata_setscene[postfuncs] += "toaster_package_dumpdata "
+do_packagedata_setscene[vardepsexclude] += "toaster_package_dumpdata "
+
+do_package[postfuncs] += "toaster_package_dumpdata "
+do_package[vardepsexclude] += "toaster_package_dumpdata "
+
+do_image_complete[postfuncs] += "toaster_image_dumpdata "
+do_image_complete[vardepsexclude] += "toaster_image_dumpdata "
+
+do_rootfs[postfuncs] += "toaster_licensemanifest_dump "
+do_rootfs[vardepsexclude] += "toaster_licensemanifest_dump "
+
+do_populate_sdk[postfuncs] += "toaster_artifact_dumpdata "
+do_populate_sdk[vardepsexclude] += "toaster_artifact_dumpdata "
OpenPOWER on IntegriCloud