diff options
Diffstat (limited to 'poky/meta/classes/distrodata.bbclass')
| -rw-r--r-- | poky/meta/classes/distrodata.bbclass | 427 |
1 files changed, 427 insertions, 0 deletions
diff --git a/poky/meta/classes/distrodata.bbclass b/poky/meta/classes/distrodata.bbclass new file mode 100644 index 000000000..59ee8cea6 --- /dev/null +++ b/poky/meta/classes/distrodata.bbclass @@ -0,0 +1,427 @@ +include conf/distro/include/upstream_tracking.inc +include conf/distro/include/distro_alias.inc +include conf/distro/include/maintainers.inc + +addhandler distro_eventhandler +distro_eventhandler[eventmask] = "bb.event.BuildStarted" +python distro_eventhandler() { + import oe.distro_check as dc + import csv + logfile = dc.create_log_file(e.data, "distrodata.csv") + + lf = bb.utils.lockfile("%s.lock" % logfile) + with open(logfile, "a") as f: + writer = csv.writer(f) + writer.writerow(['Package', 'Description', 'Owner', 'License', + 'VerMatch', 'Version', 'Upstream', 'Reason', 'Recipe Status', + 'Distro 1', 'Distro 2', 'Distro 3']) + f.close() + bb.utils.unlockfile(lf) + + return +} + +addtask distrodata_np +do_distrodata_np[nostamp] = "1" +python do_distrodata_np() { + localdata = bb.data.createCopy(d) + pn = d.getVar("PN") + bb.note("Package Name: %s" % pn) + + import oe.distro_check as dist_check + tmpdir = d.getVar('TMPDIR') + distro_check_dir = os.path.join(tmpdir, "distro_check") + datetime = localdata.getVar('DATETIME') + dist_check.update_distro_data(distro_check_dir, datetime, localdata) + + if pn.find("-native") != -1: + pnstripped = pn.split("-native") + bb.note("Native Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + + if pn.find("-cross") != -1: + pnstripped = pn.split("-cross") + bb.note("cross Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + + if pn.find("-crosssdk") != -1: + pnstripped = pn.split("-crosssdk") + bb.note("cross Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + + if pn.startswith("nativesdk-"): + pnstripped = pn.replace("nativesdk-", "") + bb.note("NativeSDK Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped + ":" + d.getVar('OVERRIDES')) + + + if pn.find("-initial") != -1: + pnstripped = pn.split("-initial") + bb.note("initial Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + + """generate package information from .bb file""" + pname = localdata.getVar('PN') + pcurver = localdata.getVar('PV') + pdesc = localdata.getVar('DESCRIPTION') + if pdesc is not None: + pdesc = pdesc.replace(',','') + pdesc = pdesc.replace('\n','') + + pgrp = localdata.getVar('SECTION') + plicense = localdata.getVar('LICENSE').replace(',','_') + + rstatus = localdata.getVar('RECIPE_COLOR') + if rstatus is not None: + rstatus = rstatus.replace(',','') + + pupver = localdata.getVar('RECIPE_UPSTREAM_VERSION') + if pcurver == pupver: + vermatch="1" + else: + vermatch="0" + noupdate_reason = localdata.getVar('RECIPE_NO_UPDATE_REASON') + if noupdate_reason is None: + noupdate="0" + else: + noupdate="1" + noupdate_reason = noupdate_reason.replace(',','') + + maintainer = localdata.getVar('RECIPE_MAINTAINER') + rlrd = localdata.getVar('RECIPE_UPSTREAM_DATE') + result = dist_check.compare_in_distro_packages_list(distro_check_dir, localdata) + + bb.note("DISTRO: %s,%s,%s,%s,%s,%s,%s,%s,%s\n" % \ + (pname, pdesc, maintainer, plicense, vermatch, pcurver, pupver, noupdate_reason, rstatus)) + line = pn + for i in result: + line = line + "," + i + bb.note("%s\n" % line) +} +do_distrodata_np[vardepsexclude] = "DATETIME" + +addtask distrodata +do_distrodata[nostamp] = "1" +python do_distrodata() { + import csv + logpath = d.getVar('LOG_DIR') + bb.utils.mkdirhier(logpath) + logfile = os.path.join(logpath, "distrodata.csv") + + import oe.distro_check as dist_check + localdata = bb.data.createCopy(d) + tmpdir = d.getVar('TMPDIR') + distro_check_dir = os.path.join(tmpdir, "distro_check") + datetime = localdata.getVar('DATETIME') + dist_check.update_distro_data(distro_check_dir, datetime, localdata) + + pn = d.getVar("PN") + bb.note("Package Name: %s" % pn) + + if pn.find("-native") != -1: + pnstripped = pn.split("-native") + bb.note("Native Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + + if pn.startswith("nativesdk-"): + pnstripped = pn.replace("nativesdk-", "") + bb.note("NativeSDK Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped + ":" + d.getVar('OVERRIDES')) + + if pn.find("-cross") != -1: + pnstripped = pn.split("-cross") + bb.note("cross Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + + if pn.find("-crosssdk") != -1: + pnstripped = pn.split("-crosssdk") + bb.note("cross Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + + if pn.find("-initial") != -1: + pnstripped = pn.split("-initial") + bb.note("initial Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + + """generate package information from .bb file""" + pname = localdata.getVar('PN') + pcurver = localdata.getVar('PV') + pdesc = localdata.getVar('DESCRIPTION') + if pdesc is not None: + pdesc = pdesc.replace(',','') + pdesc = pdesc.replace('\n','') + + pgrp = localdata.getVar('SECTION') + plicense = localdata.getVar('LICENSE').replace(',','_') + + rstatus = localdata.getVar('RECIPE_COLOR') + if rstatus is not None: + rstatus = rstatus.replace(',','') + + pupver = localdata.getVar('RECIPE_UPSTREAM_VERSION') + if pcurver == pupver: + vermatch="1" + else: + vermatch="0" + + noupdate_reason = localdata.getVar('RECIPE_NO_UPDATE_REASON') + if noupdate_reason is None: + noupdate="0" + else: + noupdate="1" + noupdate_reason = noupdate_reason.replace(',','') + + maintainer = localdata.getVar('RECIPE_MAINTAINER') + rlrd = localdata.getVar('RECIPE_UPSTREAM_DATE') + # do the comparison + result = dist_check.compare_in_distro_packages_list(distro_check_dir, localdata) + + lf = bb.utils.lockfile("%s.lock" % logfile) + with open(logfile, "a") as f: + row = [pname, pdesc, maintainer, plicense, vermatch, pcurver, pupver, noupdate_reason, rstatus] + row.extend(result) + + writer = csv.writer(f) + writer.writerow(row) + f.close() + bb.utils.unlockfile(lf) +} +do_distrodata[vardepsexclude] = "DATETIME" + +addhandler checkpkg_eventhandler +checkpkg_eventhandler[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted" +python checkpkg_eventhandler() { + import csv + + def parse_csv_file(filename): + package_dict = {} + + with open(filename, "r") as f: + reader = csv.reader(f, delimiter='\t') + for row in reader: + pn = row[0] + + if reader.line_num == 1: + header = row + continue + + if not pn in package_dict.keys(): + package_dict[pn] = row + f.close() + + with open(filename, "w") as f: + writer = csv.writer(f, delimiter='\t') + writer.writerow(header) + for pn in package_dict.keys(): + writer.writerow(package_dict[pn]) + f.close() + + del package_dict + + if bb.event.getName(e) == "BuildStarted": + import oe.distro_check as dc + logfile = dc.create_log_file(e.data, "checkpkg.csv") + + lf = bb.utils.lockfile("%s.lock" % logfile) + with open(logfile, "a") as f: + writer = csv.writer(f, delimiter='\t') + headers = ['Package', 'Version', 'Upver', 'License', 'Section', + 'Home', 'Release', 'Depends', 'BugTracker', 'PE', 'Description', + 'Status', 'Tracking', 'URI', 'MAINTAINER', 'NoUpReason'] + writer.writerow(headers) + f.close() + bb.utils.unlockfile(lf) + elif bb.event.getName(e) == "BuildCompleted": + import os + filename = "tmp/log/checkpkg.csv" + if os.path.isfile(filename): + lf = bb.utils.lockfile("%s.lock"%filename) + parse_csv_file(filename) + bb.utils.unlockfile(lf) + return +} + +addtask checkpkg +do_checkpkg[nostamp] = "1" +python do_checkpkg() { + localdata = bb.data.createCopy(d) + import csv + import re + import tempfile + import subprocess + import oe.recipeutils + from bb.utils import vercmp_string + from bb.fetch2 import FetchError, NoMethodError, decodeurl + + def get_upstream_version_and_status(): + + # set if the upstream check fails reliably, e.g. absent git tags, or weird version format used on our or on upstream side. + upstream_version_unknown = localdata.getVar('UPSTREAM_VERSION_UNKNOWN') + # set if the upstream check cannot be reliably performed due to transient network failures, or server behaving weirdly. + # This one should be used sparingly, as it completely excludes a recipe from upstream checking. + upstream_check_unreliable = localdata.getVar('UPSTREAM_CHECK_UNRELIABLE') + + if upstream_check_unreliable == "1": + return "N/A", "CHECK_IS_UNRELIABLE" + + uv = oe.recipeutils.get_recipe_upstream_version(localdata) + pupver = uv['version'] if uv['version'] else "N/A" + pversion = uv['current_version'] + revision = uv['revision'] if uv['revision'] else "N/A" + + if pupver == "N/A": + pstatus = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" + else: + cmp = vercmp_string(pversion, pupver) + if cmp == -1: + pstatus = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN" + elif cmp == 0: + pstatus = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN" + else: + pstatus = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" + + return pversion, pupver, pstatus, revision + + + """initialize log files.""" + logpath = d.getVar('LOG_DIR') + bb.utils.mkdirhier(logpath) + logfile = os.path.join(logpath, "checkpkg.csv") + + """generate package information from .bb file""" + pname = d.getVar('PN') + + if pname.find("-native") != -1: + if d.getVar('BBCLASSEXTEND'): + return + pnstripped = pname.split("-native") + bb.note("Native Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + + if pname.startswith("nativesdk-"): + if d.getVar('BBCLASSEXTEND'): + return + pnstripped = pname.replace("nativesdk-", "") + bb.note("NativeSDK Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped + ":" + d.getVar('OVERRIDES')) + + if pname.find("-cross") != -1: + pnstripped = pname.split("-cross") + bb.note("cross Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + + if pname.find("-initial") != -1: + pnstripped = pname.split("-initial") + bb.note("initial Split: %s" % pnstripped) + localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES')) + + pdesc = localdata.getVar('DESCRIPTION') + pgrp = localdata.getVar('SECTION') + plicense = localdata.getVar('LICENSE') + psection = localdata.getVar('SECTION') + phome = localdata.getVar('HOMEPAGE') + prelease = localdata.getVar('PR') + pdepends = localdata.getVar('DEPENDS') + pbugtracker = localdata.getVar('BUGTRACKER') + ppe = localdata.getVar('PE') + psrcuri = localdata.getVar('SRC_URI') + maintainer = localdata.getVar('RECIPE_MAINTAINER') + + pversion, pupver, pstatus, prevision = get_upstream_version_and_status() + + if psrcuri: + psrcuri = psrcuri.split()[0] + else: + psrcuri = "none" + pdepends = "".join(pdepends.split("\t")) + pdesc = "".join(pdesc.split("\t")) + no_upgr_reason = d.getVar('RECIPE_NO_UPDATE_REASON') + lf = bb.utils.lockfile("%s.lock" % logfile) + with open(logfile, "a") as f: + writer = csv.writer(f, delimiter='\t') + writer.writerow([pname, pversion, pupver, plicense, psection, phome, + prelease, pdepends, pbugtracker, ppe, pdesc, pstatus, prevision, + psrcuri, maintainer, no_upgr_reason]) + f.close() + bb.utils.unlockfile(lf) +} + +addhandler distro_check_eventhandler +distro_check_eventhandler[eventmask] = "bb.event.BuildStarted" +python distro_check_eventhandler() { + """initialize log files.""" + import oe.distro_check as dc + result_file = dc.create_log_file(e.data, "distrocheck.csv") + return +} + +addtask distro_check +do_distro_check[nostamp] = "1" +do_distro_check[vardepsexclude] += "DATETIME" +python do_distro_check() { + """checks if the package is present in other public Linux distros""" + import oe.distro_check as dc + import shutil + if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('nativesdk',d): + return + + localdata = bb.data.createCopy(d) + tmpdir = d.getVar('TMPDIR') + distro_check_dir = os.path.join(tmpdir, "distro_check") + logpath = d.getVar('LOG_DIR') + bb.utils.mkdirhier(logpath) + result_file = os.path.join(logpath, "distrocheck.csv") + datetime = localdata.getVar('DATETIME') + dc.update_distro_data(distro_check_dir, datetime, localdata) + + # do the comparison + result = dc.compare_in_distro_packages_list(distro_check_dir, d) + + # save the results + dc.save_distro_check_result(result, datetime, result_file, d) +} + +# +#Check Missing License Text. +#Use this task to generate the missing license text data for pkg-report system, +#then we can search those recipes which license text isn't exsit in common-licenses directory +# +addhandler checklicense_eventhandler +checklicense_eventhandler[eventmask] = "bb.event.BuildStarted" +python checklicense_eventhandler() { + """initialize log files.""" + import csv + import oe.distro_check as dc + logfile = dc.create_log_file(e.data, "missinglicense.csv") + lf = bb.utils.lockfile("%s.lock" % logfile) + with open(logfile, "a") as f: + writer = csv.writer(f, delimiter='\t') + writer.writerow(['Package', 'License', 'MissingLicense']) + f.close() + bb.utils.unlockfile(lf) + return +} + +addtask checklicense +do_checklicense[nostamp] = "1" +python do_checklicense() { + import csv + import shutil + logpath = d.getVar('LOG_DIR') + bb.utils.mkdirhier(logpath) + pn = d.getVar('PN') + logfile = os.path.join(logpath, "missinglicense.csv") + generic_directory = d.getVar('COMMON_LICENSE_DIR') + license_types = d.getVar('LICENSE') + for license_type in ((license_types.replace('+', '').replace('|', '&') + .replace('(', '').replace(')', '').replace(';', '') + .replace(',', '').replace(" ", "").split("&"))): + if not os.path.isfile(os.path.join(generic_directory, license_type)): + lf = bb.utils.lockfile("%s.lock" % logfile) + with open(logfile, "a") as f: + writer = csv.writer(f, delimiter='\t') + writer.writerow([pn, license_types, license_type]) + f.close() + bb.utils.unlockfile(lf) + return +} |
