diff options
Diffstat (limited to 'yocto-poky/bitbake/lib/bb/fetch2/wget.py')
| -rw-r--r-- | yocto-poky/bitbake/lib/bb/fetch2/wget.py | 118 |
1 files changed, 71 insertions, 47 deletions
diff --git a/yocto-poky/bitbake/lib/bb/fetch2/wget.py b/yocto-poky/bitbake/lib/bb/fetch2/wget.py index bd2a8972a..8bc9e93ca 100644 --- a/yocto-poky/bitbake/lib/bb/fetch2/wget.py +++ b/yocto-poky/bitbake/lib/bb/fetch2/wget.py @@ -37,7 +37,9 @@ from bb.fetch2 import FetchMethod from bb.fetch2 import FetchError from bb.fetch2 import logger from bb.fetch2 import runfetchcmd +from bb.utils import export_proxies from bs4 import BeautifulSoup +from bs4 import SoupStrainer class Wget(FetchMethod): """Class to fetch urls via 'wget'""" @@ -61,6 +63,8 @@ class Wget(FetchMethod): ud.basename = os.path.basename(ud.path) ud.localfile = data.expand(urllib.unquote(ud.basename), d) + if not ud.localfile: + ud.localfile = data.expand(urllib.unquote(ud.host + ud.path).replace("/", "."), d) self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 -nv --passive-ftp --no-check-certificate" @@ -218,54 +222,64 @@ class Wget(FetchMethod): return resp - def export_proxies(d): - variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY', - 'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY'] - exported = False - - for v in variables: - if v in os.environ.keys(): - exported = True - else: - v_proxy = d.getVar(v, True) - if v_proxy is not None: - os.environ[v] = v_proxy - exported = True - - return exported - - def head_method(self): - return "HEAD" - + class HTTPMethodFallback(urllib2.BaseHandler): + """ + Fallback to GET if HEAD is not allowed (405 HTTP error) + """ + def http_error_405(self, req, fp, code, msg, headers): + fp.read() + fp.close() + + newheaders = dict((k,v) for k,v in req.headers.items() + if k.lower() not in ("content-length", "content-type")) + return self.parent.open(urllib2.Request(req.get_full_url(), + headers=newheaders, + origin_req_host=req.get_origin_req_host(), + unverifiable=True)) + + """ + Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403 + Forbidden when they actually mean 405 Method Not Allowed. + """ + http_error_403 = http_error_405 + + """ + Some servers (e.g. FusionForge) returns 406 Not Acceptable when they + actually mean 405 Method Not Allowed. + """ + http_error_406 = http_error_405 + + class FixedHTTPRedirectHandler(urllib2.HTTPRedirectHandler): + """ + urllib2.HTTPRedirectHandler resets the method to GET on redirect, + when we want to follow redirects using the original method. + """ + def redirect_request(self, req, fp, code, msg, headers, newurl): + newreq = urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) + newreq.get_method = lambda: req.get_method() + return newreq exported_proxies = export_proxies(d) + handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] + if export_proxies: + handlers.append(urllib2.ProxyHandler()) + handlers.append(CacheHTTPHandler()) # XXX: Since Python 2.7.9 ssl cert validation is enabled by default # see PEP-0476, this causes verification errors on some https servers # so disable by default. import ssl - ssl_context = None if hasattr(ssl, '_create_unverified_context'): - ssl_context = ssl._create_unverified_context() - - if exported_proxies == True and ssl_context is not None: - opener = urllib2.build_opener(urllib2.ProxyHandler, CacheHTTPHandler, - urllib2.HTTPSHandler(context=ssl_context)) - elif exported_proxies == False and ssl_context is not None: - opener = urllib2.build_opener(CacheHTTPHandler, - urllib2.HTTPSHandler(context=ssl_context)) - elif exported_proxies == True and ssl_context is None: - opener = urllib2.build_opener(urllib2.ProxyHandler, CacheHTTPHandler) - else: - opener = urllib2.build_opener(CacheHTTPHandler) - - urllib2.Request.get_method = head_method - urllib2.install_opener(opener) - - uri = ud.url.split(";")[0] + handlers.append(urllib2.HTTPSHandler(context=ssl._create_unverified_context())) + opener = urllib2.build_opener(*handlers) try: - urllib2.urlopen(uri) - except: + uri = ud.url.split(";")[0] + r = urllib2.Request(uri) + r.get_method = lambda: "HEAD" + opener.open(r) + except urllib2.URLError as e: + # debug for now to avoid spamming the logs in e.g. remote sstate searches + logger.debug(2, "checkstatus() urlopen failed: %s" % e) return False return True @@ -367,7 +381,7 @@ class Wget(FetchMethod): version = ['', '', ''] bb.debug(3, "VersionURL: %s" % (url)) - soup = BeautifulSoup(self._fetch_index(url, ud, d)) + soup = BeautifulSoup(self._fetch_index(url, ud, d), "html.parser", parse_only=SoupStrainer("a")) if not soup: bb.debug(3, "*** %s NO SOUP" % (url)) return "" @@ -406,10 +420,10 @@ class Wget(FetchMethod): version_dir = ['', '', ''] version = ['', '', ''] - dirver_regex = re.compile("(\D*)((\d+[\.\-_])+(\d+))") + dirver_regex = re.compile("(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))") s = dirver_regex.search(dirver) if s: - version_dir[1] = s.group(2) + version_dir[1] = s.group('ver') else: version_dir[1] = dirver @@ -417,16 +431,26 @@ class Wget(FetchMethod): ud.path.split(dirver)[0], ud.user, ud.pswd, {}]) bb.debug(3, "DirURL: %s, %s" % (dirs_uri, package)) - soup = BeautifulSoup(self._fetch_index(dirs_uri, ud, d)) + soup = BeautifulSoup(self._fetch_index(dirs_uri, ud, d), "html.parser", parse_only=SoupStrainer("a")) if not soup: return version[1] for line in soup.find_all('a', href=True): s = dirver_regex.search(line['href'].strip("/")) if s: - version_dir_new = ['', s.group(2), ''] + sver = s.group('ver') + + # When prefix is part of the version directory it need to + # ensure that only version directory is used so remove previous + # directories if exists. + # + # Example: pfx = '/dir1/dir2/v' and version = '2.5' the expected + # result is v2.5. + spfx = s.group('pfx').split('/')[-1] + + version_dir_new = ['', sver, ''] if self._vercmp(version_dir, version_dir_new) <= 0: - dirver_new = s.group(1) + s.group(2) + dirver_new = spfx + sver path = ud.path.replace(dirver, dirver_new, True) \ .split(package)[0] uri = bb.fetch.encodeurl([ud.type, ud.host, path, @@ -480,7 +504,7 @@ class Wget(FetchMethod): self.suffix_regex_comp = re.compile(psuffix_regex) # compile regex, can be specific by package or generic regex - pn_regex = d.getVar('REGEX', True) + pn_regex = d.getVar('UPSTREAM_CHECK_REGEX', True) if pn_regex: package_custom_regex_comp = re.compile(pn_regex) else: @@ -516,7 +540,7 @@ class Wget(FetchMethod): bb.debug(3, "latest_versionstring, regex: %s" % (package_regex.pattern)) uri = "" - regex_uri = d.getVar("REGEX_URI", True) + regex_uri = d.getVar("UPSTREAM_CHECK_URI", True) if not regex_uri: path = ud.path.split(package)[0] |
