diff options
Diffstat (limited to 'poky/bitbake/lib/bb/fetch2')
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/__init__.py | 28 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/git.py | 22 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/gitsm.py | 20 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/hg.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/npm.py | 2 | ||||
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/wget.py | 77 |
6 files changed, 88 insertions, 63 deletions
diff --git a/poky/bitbake/lib/bb/fetch2/__init__.py b/poky/bitbake/lib/bb/fetch2/__init__.py index 709372e16..8fecc809d 100644 --- a/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/poky/bitbake/lib/bb/fetch2/__init__.py @@ -524,7 +524,7 @@ def fetcher_parse_save(): def fetcher_parse_done(): _checksum_cache.save_merge() -def fetcher_compare_revisions(): +def fetcher_compare_revisions(d): """ Compare the revisions in the persistant cache with current values and return true/false on whether they've changed. @@ -777,7 +777,8 @@ def get_srcrev(d, method_name='sortable_revision'): # format = d.getVar('SRCREV_FORMAT') if not format: - raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.") + raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.\n"\ + "The SCMs are:\n%s" % '\n'.join(scms)) name_to_rev = {} seenautoinc = False @@ -858,7 +859,10 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): # Disable pseudo as it may affect ssh, potentially causing it to hang. cmd = 'export PSEUDO_DISABLED=1; ' + cmd - logger.debug(1, "Running %s", cmd) + if workdir: + logger.debug(1, "Running '%s' in %s" % (cmd, workdir)) + else: + logger.debug(1, "Running %s", cmd) success = False error_message = "" @@ -894,7 +898,7 @@ def check_network_access(d, info, url): log remote network access, and error if BB_NO_NETWORK is set or the given URI is untrusted """ - if d.getVar("BB_NO_NETWORK") == "1": + if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): raise NetworkAccess(url, info) elif not trusted_network(d, url): raise UntrustedUrl(url, info) @@ -1027,7 +1031,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False): raise except IOError as e: - if e.errno in [os.errno.ESTALE]: + if e.errno in [errno.ESTALE]: logger.warning("Stale Error Observed %s." % ud.url) return False raise @@ -1094,7 +1098,7 @@ def trusted_network(d, url): BB_ALLOWED_NETWORKS is set globally or for a specific recipe. Note: modifies SRC_URI & mirrors. """ - if d.getVar('BB_NO_NETWORK') == "1": + if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): return True pkgname = d.expand(d.getVar('PN', False)) @@ -1403,7 +1407,7 @@ class FetchMethod(object): Fetch urls Assumes localpath was called first """ - raise NoMethodError(url) + raise NoMethodError(urldata.url) def unpack(self, urldata, rootdir, data): iterate = False @@ -1547,7 +1551,7 @@ class FetchMethod(object): Check the status of a URL Assumes localpath was called first """ - logger.info("URL %s could not be checked for status since no method exists.", url) + logger.info("URL %s could not be checked for status since no method exists.", urldata.url) return True def latest_revision(self, ud, d, name): @@ -1555,7 +1559,7 @@ class FetchMethod(object): Look in the cache for the latest revision, if not present ask the SCM. """ if not hasattr(self, "_latest_revision"): - raise ParameterError("The fetcher for this URL does not support _latest_revision", url) + raise ParameterError("The fetcher for this URL does not support _latest_revision", ud.url) revs = bb.persist_data.persist('BB_URI_HEADREVS', d) key = self.generate_revision_key(ud, d, name) @@ -1638,7 +1642,7 @@ class Fetch(object): urls = self.urls network = self.d.getVar("BB_NO_NETWORK") - premirroronly = (self.d.getVar("BB_FETCH_PREMIRRORONLY") == "1") + premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY")) for u in urls: ud = self.ud[u] @@ -1716,7 +1720,7 @@ class Fetch(object): update_stamp(ud, self.d) except IOError as e: - if e.errno in [os.errno.ESTALE]: + if e.errno in [errno.ESTALE]: logger.error("Stale Error Observed %s." % u) raise ChecksumError("Stale Error Detected") @@ -1786,7 +1790,7 @@ class Fetch(object): for url in urls: if url not in self.ud: - self.ud[url] = FetchData(url, d) + self.ud[url] = FetchData(url, self.d) ud = self.ud[url] ud.setup_localpath(self.d) diff --git a/poky/bitbake/lib/bb/fetch2/git.py b/poky/bitbake/lib/bb/fetch2/git.py index 59a2ee8f8..8185bf4db 100644 --- a/poky/bitbake/lib/bb/fetch2/git.py +++ b/poky/bitbake/lib/bb/fetch2/git.py @@ -199,7 +199,7 @@ class Git(FetchMethod): depth_default = 1 ud.shallow_depths = collections.defaultdict(lambda: depth_default) - revs_default = d.getVar("BB_GIT_SHALLOW_REVS", True) + revs_default = d.getVar("BB_GIT_SHALLOW_REVS") ud.shallow_revs = [] ud.branches = {} for pos, name in enumerate(ud.names): @@ -318,7 +318,7 @@ class Git(FetchMethod): def try_premirror(self, ud, d): # If we don't do this, updating an existing checkout with only premirrors # is not possible - if d.getVar("BB_FETCH_PREMIRRORONLY") is not None: + if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): return True if os.path.exists(ud.clonedir): return False @@ -522,9 +522,17 @@ class Git(FetchMethod): def clean(self, ud, d): """ clean the git directory """ - bb.utils.remove(ud.localpath, True) - bb.utils.remove(ud.fullmirror) - bb.utils.remove(ud.fullmirror + ".done") + to_remove = [ud.localpath, ud.fullmirror, ud.fullmirror + ".done"] + # The localpath is a symlink to clonedir when it is cloned from a + # mirror, so remove both of them. + if os.path.islink(ud.localpath): + clonedir = os.path.realpath(ud.localpath) + to_remove.append(clonedir) + + for r in to_remove: + if os.path.exists(r): + bb.note('Removing %s' % r) + bb.utils.remove(r, True) def supports_srcrev(self): return True @@ -615,7 +623,7 @@ class Git(FetchMethod): """ pupver = ('', '') - tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or "(?P<pver>([0-9][\.|_]?)+)") + tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)") try: output = self._lsremote(ud, d, "refs/tags/*") except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: @@ -630,7 +638,7 @@ class Git(FetchMethod): tag_head = line.split("/")[-1] # Ignore non-released branches - m = re.search("(alpha|beta|rc|final)+", tag_head) + m = re.search(r"(alpha|beta|rc|final)+", tag_head) if m: continue diff --git a/poky/bitbake/lib/bb/fetch2/gitsm.py b/poky/bitbake/lib/bb/fetch2/gitsm.py index b21fed266..32389130b 100644 --- a/poky/bitbake/lib/bb/fetch2/gitsm.py +++ b/poky/bitbake/lib/bb/fetch2/gitsm.py @@ -147,6 +147,23 @@ class GitSM(Git): return submodules != [] + def need_update(self, ud, d): + if Git.need_update(self, ud, d): + return True + + try: + # Check for the nugget dropped by the download operation + known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \ + (ud.basecmd), d, workdir=ud.clonedir) + + if ud.revisions[ud.names[0]] not in known_srcrevs.split(): + return True + except bb.fetch2.FetchError: + # No srcrev nuggets, so this is new and needs to be updated + return True + + return False + def download(self, ud, d): def download_submodule(ud, url, module, modpath, d): url += ";bareclone=1;nobranch=1" @@ -157,6 +174,9 @@ class GitSM(Git): try: newfetch = Fetch([url], d, cache=False) newfetch.download() + # Drop a nugget to add each of the srcrevs we've fetched (used by need_update) + runfetchcmd("%s config --add bitbake.srcrev %s" % \ + (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) except Exception as e: logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) raise diff --git a/poky/bitbake/lib/bb/fetch2/hg.py b/poky/bitbake/lib/bb/fetch2/hg.py index 936d04311..5a2985e16 100644 --- a/poky/bitbake/lib/bb/fetch2/hg.py +++ b/poky/bitbake/lib/bb/fetch2/hg.py @@ -99,7 +99,7 @@ class Hg(FetchMethod): def try_premirror(self, ud, d): # If we don't do this, updating an existing checkout with only premirrors # is not possible - if d.getVar("BB_FETCH_PREMIRRORONLY") is not None: + if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): return True if os.path.exists(ud.moddir): return False diff --git a/poky/bitbake/lib/bb/fetch2/npm.py b/poky/bitbake/lib/bb/fetch2/npm.py index 408dfc3d0..65bf5a364 100644 --- a/poky/bitbake/lib/bb/fetch2/npm.py +++ b/poky/bitbake/lib/bb/fetch2/npm.py @@ -226,7 +226,7 @@ class Npm(FetchMethod): self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest, False) return outputurl = "invalid" - if ('resolved' not in data) or (not data['resolved'].startswith('http')): + if ('resolved' not in data) or (not data['resolved'].startswith('http://') and not data['resolved'].startswith('https://')): # will be the case for ${PN} fetchcmd = "npm view %s@%s dist.tarball --registry %s" % (pkg, version, ud.registry) logger.debug(2, "Found this matching URL: %s" % str(fetchcmd)) diff --git a/poky/bitbake/lib/bb/fetch2/wget.py b/poky/bitbake/lib/bb/fetch2/wget.py index 8f505b6de..3bb3e3bb0 100644 --- a/poky/bitbake/lib/bb/fetch2/wget.py +++ b/poky/bitbake/lib/bb/fetch2/wget.py @@ -33,11 +33,14 @@ import logging import errno import bb import bb.progress +import socket +import http.client import urllib.request, urllib.parse, urllib.error from bb.fetch2 import FetchMethod from bb.fetch2 import FetchError from bb.fetch2 import logger from bb.fetch2 import runfetchcmd +from bb.fetch2 import FetchConnectionCache from bb.utils import export_proxies from bs4 import BeautifulSoup from bs4 import SoupStrainer @@ -132,10 +135,6 @@ class Wget(FetchMethod): return True def checkstatus(self, fetch, ud, d, try_again=True): - import urllib.request, urllib.error, urllib.parse, socket, http.client - from urllib.response import addinfourl - from bb.fetch2 import FetchConnectionCache - class HTTPConnectionCache(http.client.HTTPConnection): if fetch.connection_cache: def connect(self): @@ -168,7 +167,7 @@ class Wget(FetchMethod): """ host = req.host if not host: - raise urlllib2.URLError('no host given') + raise urllib.error.URLError('no host given') h = http_class(host, timeout=req.timeout) # will parse host:port h.set_debuglevel(self._debuglevel) @@ -185,7 +184,7 @@ class Wget(FetchMethod): # request. # Don't close connection when connection_cache is enabled, - if fetch.connection_cache is None: + if fetch.connection_cache is None: headers["Connection"] = "close" else: headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0 @@ -252,7 +251,7 @@ class Wget(FetchMethod): pass closed = False - resp = addinfourl(fp_dummy(), r.msg, req.get_full_url()) + resp = urllib.response.addinfourl(fp_dummy(), r.msg, req.get_full_url()) resp.code = r.status resp.msg = r.reason @@ -271,17 +270,16 @@ class Wget(FetchMethod): fp.read() fp.close() - newheaders = dict((k,v) for k,v in list(req.headers.items()) + newheaders = dict((k, v) for k, v in list(req.headers.items()) if k.lower() not in ("content-length", "content-type")) return self.parent.open(urllib.request.Request(req.get_full_url(), headers=newheaders, origin_req_host=req.origin_req_host, unverifiable=True)) - """ - Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403 - Forbidden when they actually mean 405 Method Not Allowed. - """ + + # Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403 + # Forbidden when they actually mean 405 Method Not Allowed. http_error_403 = http_error_405 @@ -292,15 +290,15 @@ class Wget(FetchMethod): """ def redirect_request(self, req, fp, code, msg, headers, newurl): newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) - newreq.get_method = lambda: req.get_method() + newreq.get_method = req.get_method return newreq exported_proxies = export_proxies(d) handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] - if export_proxies: + if exported_proxies: handlers.append(urllib.request.ProxyHandler()) handlers.append(CacheHTTPHandler()) - # XXX: Since Python 2.7.9 ssl cert validation is enabled by default + # Since Python 2.7.9 ssl cert validation is enabled by default # see PEP-0476, this causes verification errors on some https servers # so disable by default. import ssl @@ -319,19 +317,19 @@ class Wget(FetchMethod): '''Adds Basic auth to http request, pass in login:password as string''' import base64 encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8") - authheader = "Basic %s" % encodeuser + authheader = "Basic %s" % encodeuser r.add_header("Authorization", authheader) - if ud.user: - add_basic_auth(ud.user, r) + if ud.user and ud.pswd: + add_basic_auth(ud.user + ':' + ud.pswd, r) try: - import netrc, urllib.parse + import netrc n = netrc.netrc() login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname) add_basic_auth("%s:%s" % (login, password), r) except (TypeError, ImportError, IOError, netrc.NetrcParseError): - pass + pass with opener.open(r) as response: pass @@ -396,18 +394,14 @@ class Wget(FetchMethod): (oldpn, oldpv, oldsuffix) = old (newpn, newpv, newsuffix) = new - """ - Check for a new suffix type that we have never heard of before - """ - if (newsuffix): + # Check for a new suffix type that we have never heard of before + if newsuffix: m = self.suffix_regex_comp.search(newsuffix) if not m: bb.warn("%s has a possible unknown suffix: %s" % (newpn, newsuffix)) return False - """ - Not our package so ignore it - """ + # Not our package so ignore it if oldpn != newpn: return False @@ -473,15 +467,14 @@ class Wget(FetchMethod): return "" - def _check_latest_version_by_dir(self, dirver, package, package_regex, - current_version, ud, d): + def _check_latest_version_by_dir(self, dirver, package, package_regex, current_version, ud, d): """ - Scan every directory in order to get upstream version. + Scan every directory in order to get upstream version. """ version_dir = ['', '', ''] version = ['', '', ''] - dirver_regex = re.compile("(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))") + dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))") s = dirver_regex.search(dirver) if s: version_dir[1] = s.group('ver') @@ -541,26 +534,26 @@ class Wget(FetchMethod): gst-fluendo-mp3 """ # match most patterns which uses "-" as separator to version digits - pn_prefix1 = "[a-zA-Z][a-zA-Z0-9]*([-_][a-zA-Z]\w+)*\+?[-_]" + pn_prefix1 = r"[a-zA-Z][a-zA-Z0-9]*([-_][a-zA-Z]\w+)*\+?[-_]" # a loose pattern such as for unzip552.tar.gz - pn_prefix2 = "[a-zA-Z]+" + pn_prefix2 = r"[a-zA-Z]+" # a loose pattern such as for 80325-quicky-0.4.tar.gz - pn_prefix3 = "[0-9]+[-]?[a-zA-Z]+" + pn_prefix3 = r"[0-9]+[-]?[a-zA-Z]+" # Save the Package Name (pn) Regex for use later - pn_regex = "(%s|%s|%s)" % (pn_prefix1, pn_prefix2, pn_prefix3) + pn_regex = r"(%s|%s|%s)" % (pn_prefix1, pn_prefix2, pn_prefix3) # match version - pver_regex = "(([A-Z]*\d+[a-zA-Z]*[\.\-_]*)+)" + pver_regex = r"(([A-Z]*\d+[a-zA-Z]*[\.\-_]*)+)" # match arch parch_regex = "-source|_all_" # src.rpm extension was added only for rpm package. Can be removed if the rpm # packaged will always be considered as having to be manually upgraded - psuffix_regex = "(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)" + psuffix_regex = r"(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)" # match name, version and archive type of a package - package_regex_comp = re.compile("(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)" + package_regex_comp = re.compile(r"(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)" % (pn_regex, pver_regex, parch_regex, psuffix_regex)) self.suffix_regex_comp = re.compile(psuffix_regex) @@ -572,7 +565,7 @@ class Wget(FetchMethod): version = self._parse_path(package_regex_comp, package) if version: package_custom_regex_comp = re.compile( - "(?P<name>%s)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s)" % + r"(?P<name>%s)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s)" % (re.escape(version[0]), pver_regex, parch_regex, psuffix_regex)) else: package_custom_regex_comp = None @@ -589,7 +582,7 @@ class Wget(FetchMethod): current_version = ['', d.getVar('PV'), ''] """possible to have no version in pkg name, such as spectrum-fw""" - if not re.search("\d+", package): + if not re.search(r"\d+", package): current_version[1] = re.sub('_', '.', current_version[1]) current_version[1] = re.sub('-', '.', current_version[1]) return (current_version[1], '') @@ -607,13 +600,13 @@ class Wget(FetchMethod): # search for version matches on folders inside the path, like: # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz - dirver_regex = re.compile("(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/") + dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/") m = dirver_regex.search(path) if m: pn = d.getVar('PN') dirver = m.group('dirver') - dirver_pn_regex = re.compile("%s\d?" % (re.escape(pn))) + dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn))) if not dirver_pn_regex.search(dirver): return (self._check_latest_version_by_dir(dirver, package, package_regex, current_version, ud, d), '') |
