diff options
Diffstat (limited to 'poky/bitbake/lib/bb/fetch2/wget.py')
| -rw-r--r-- | poky/bitbake/lib/bb/fetch2/wget.py | 626 |
1 files changed, 626 insertions, 0 deletions
diff --git a/poky/bitbake/lib/bb/fetch2/wget.py b/poky/bitbake/lib/bb/fetch2/wget.py new file mode 100644 index 000000000..8f505b6de --- /dev/null +++ b/poky/bitbake/lib/bb/fetch2/wget.py @@ -0,0 +1,626 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' implementations + +Classes for obtaining upstream sources for the +BitBake build tools. + +""" + +# Copyright (C) 2003, 2004 Chris Larson +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +import re +import tempfile +import subprocess +import os +import logging +import errno +import bb +import bb.progress +import urllib.request, urllib.parse, urllib.error +from bb.fetch2 import FetchMethod +from bb.fetch2 import FetchError +from bb.fetch2 import logger +from bb.fetch2 import runfetchcmd +from bb.utils import export_proxies +from bs4 import BeautifulSoup +from bs4 import SoupStrainer + +class WgetProgressHandler(bb.progress.LineFilterProgressHandler): + """ + Extract progress information from wget output. + Note: relies on --progress=dot (with -v or without -q/-nv) being + specified on the wget command line. + """ + def __init__(self, d): + super(WgetProgressHandler, self).__init__(d) + # Send an initial progress event so the bar gets shown + self._fire_progress(0) + + def writeline(self, line): + percs = re.findall(r'(\d+)%\s+([\d.]+[A-Z])', line) + if percs: + progress = int(percs[-1][0]) + rate = percs[-1][1] + '/s' + self.update(progress, rate) + return False + return True + + +class Wget(FetchMethod): + """Class to fetch urls via 'wget'""" + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with wget. + """ + return ud.type in ['http', 'https', 'ftp'] + + def recommends_checksum(self, urldata): + return True + + def urldata_init(self, ud, d): + if 'protocol' in ud.parm: + if ud.parm['protocol'] == 'git': + raise bb.fetch2.ParameterError("Invalid protocol - if you wish to fetch from a git repository using http, you need to instead use the git:// prefix with protocol=http", ud.url) + + if 'downloadfilename' in ud.parm: + ud.basename = ud.parm['downloadfilename'] + else: + ud.basename = os.path.basename(ud.path) + + ud.localfile = d.expand(urllib.parse.unquote(ud.basename)) + if not ud.localfile: + ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", ".")) + + self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate" + + def _runwget(self, ud, d, command, quiet, workdir=None): + + progresshandler = WgetProgressHandler(d) + + logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command)) + bb.fetch2.check_network_access(d, command, ud.url) + runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir) + + def download(self, ud, d): + """Fetch urls""" + + fetchcmd = self.basecmd + + if 'downloadfilename' in ud.parm: + dldir = d.getVar("DL_DIR") + bb.utils.mkdirhier(os.path.dirname(dldir + os.sep + ud.localfile)) + fetchcmd += " -O " + dldir + os.sep + ud.localfile + + if ud.user and ud.pswd: + fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd) + + uri = ud.url.split(";")[0] + if os.path.exists(ud.localpath): + # file exists, but we didnt complete it.. trying again.. + fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % uri) + else: + fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % uri) + + self._runwget(ud, d, fetchcmd, False) + + # Sanity check since wget can pretend it succeed when it didn't + # Also, this used to happen if sourceforge sent us to the mirror page + if not os.path.exists(ud.localpath): + raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, ud.localpath), uri) + + if os.path.getsize(ud.localpath) == 0: + os.remove(ud.localpath) + raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri) + + return True + + def checkstatus(self, fetch, ud, d, try_again=True): + import urllib.request, urllib.error, urllib.parse, socket, http.client + from urllib.response import addinfourl + from bb.fetch2 import FetchConnectionCache + + class HTTPConnectionCache(http.client.HTTPConnection): + if fetch.connection_cache: + def connect(self): + """Connect to the host and port specified in __init__.""" + + sock = fetch.connection_cache.get_connection(self.host, self.port) + if sock: + self.sock = sock + else: + self.sock = socket.create_connection((self.host, self.port), + self.timeout, self.source_address) + fetch.connection_cache.add_connection(self.host, self.port, self.sock) + + if self._tunnel_host: + self._tunnel() + + class CacheHTTPHandler(urllib.request.HTTPHandler): + def http_open(self, req): + return self.do_open(HTTPConnectionCache, req) + + def do_open(self, http_class, req): + """Return an addinfourl object for the request, using http_class. + + http_class must implement the HTTPConnection API from httplib. + The addinfourl return value is a file-like object. It also + has methods and attributes including: + - info(): return a mimetools.Message object for the headers + - geturl(): return the original request URL + - code: HTTP status code + """ + host = req.host + if not host: + raise urlllib2.URLError('no host given') + + h = http_class(host, timeout=req.timeout) # will parse host:port + h.set_debuglevel(self._debuglevel) + + headers = dict(req.unredirected_hdrs) + headers.update(dict((k, v) for k, v in list(req.headers.items()) + if k not in headers)) + + # We want to make an HTTP/1.1 request, but the addinfourl + # class isn't prepared to deal with a persistent connection. + # It will try to read all remaining data from the socket, + # which will block while the server waits for the next request. + # So make sure the connection gets closed after the (only) + # request. + + # Don't close connection when connection_cache is enabled, + if fetch.connection_cache is None: + headers["Connection"] = "close" + else: + headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0 + + headers = dict( + (name.title(), val) for name, val in list(headers.items())) + + if req._tunnel_host: + tunnel_headers = {} + proxy_auth_hdr = "Proxy-Authorization" + if proxy_auth_hdr in headers: + tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr] + # Proxy-Authorization should not be sent to origin + # server. + del headers[proxy_auth_hdr] + h.set_tunnel(req._tunnel_host, headers=tunnel_headers) + + try: + h.request(req.get_method(), req.selector, req.data, headers) + except socket.error as err: # XXX what error? + # Don't close connection when cache is enabled. + # Instead, try to detect connections that are no longer + # usable (for example, closed unexpectedly) and remove + # them from the cache. + if fetch.connection_cache is None: + h.close() + elif isinstance(err, OSError) and err.errno == errno.EBADF: + # This happens when the server closes the connection despite the Keep-Alive. + # Apparently urllib then uses the file descriptor, expecting it to be + # connected, when in reality the connection is already gone. + # We let the request fail and expect it to be + # tried once more ("try_again" in check_status()), + # with the dead connection removed from the cache. + # If it still fails, we give up, which can happend for bad + # HTTP proxy settings. + fetch.connection_cache.remove_connection(h.host, h.port) + raise urllib.error.URLError(err) + else: + try: + r = h.getresponse(buffering=True) + except TypeError: # buffering kw not supported + r = h.getresponse() + + # Pick apart the HTTPResponse object to get the addinfourl + # object initialized properly. + + # Wrap the HTTPResponse object in socket's file object adapter + # for Windows. That adapter calls recv(), so delegate recv() + # to read(). This weird wrapping allows the returned object to + # have readline() and readlines() methods. + + # XXX It might be better to extract the read buffering code + # out of socket._fileobject() and into a base class. + r.recv = r.read + + # no data, just have to read + r.read() + class fp_dummy(object): + def read(self): + return "" + def readline(self): + return "" + def close(self): + pass + closed = False + + resp = addinfourl(fp_dummy(), r.msg, req.get_full_url()) + resp.code = r.status + resp.msg = r.reason + + # Close connection when server request it. + if fetch.connection_cache is not None: + if 'Connection' in r.msg and r.msg['Connection'] == 'close': + fetch.connection_cache.remove_connection(h.host, h.port) + + return resp + + class HTTPMethodFallback(urllib.request.BaseHandler): + """ + Fallback to GET if HEAD is not allowed (405 HTTP error) + """ + def http_error_405(self, req, fp, code, msg, headers): + fp.read() + fp.close() + + newheaders = dict((k,v) for k,v in list(req.headers.items()) + if k.lower() not in ("content-length", "content-type")) + return self.parent.open(urllib.request.Request(req.get_full_url(), + headers=newheaders, + origin_req_host=req.origin_req_host, + unverifiable=True)) + + """ + Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403 + Forbidden when they actually mean 405 Method Not Allowed. + """ + http_error_403 = http_error_405 + + + class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): + """ + urllib2.HTTPRedirectHandler resets the method to GET on redirect, + when we want to follow redirects using the original method. + """ + def redirect_request(self, req, fp, code, msg, headers, newurl): + newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) + newreq.get_method = lambda: req.get_method() + return newreq + exported_proxies = export_proxies(d) + + handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] + if export_proxies: + handlers.append(urllib.request.ProxyHandler()) + handlers.append(CacheHTTPHandler()) + # XXX: Since Python 2.7.9 ssl cert validation is enabled by default + # see PEP-0476, this causes verification errors on some https servers + # so disable by default. + import ssl + if hasattr(ssl, '_create_unverified_context'): + handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context())) + opener = urllib.request.build_opener(*handlers) + + try: + uri = ud.url.split(";")[0] + r = urllib.request.Request(uri) + r.get_method = lambda: "HEAD" + # Some servers (FusionForge, as used on Alioth) require that the + # optional Accept header is set. + r.add_header("Accept", "*/*") + def add_basic_auth(login_str, request): + '''Adds Basic auth to http request, pass in login:password as string''' + import base64 + encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8") + authheader = "Basic %s" % encodeuser + r.add_header("Authorization", authheader) + + if ud.user: + add_basic_auth(ud.user, r) + + try: + import netrc, urllib.parse + n = netrc.netrc() + login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname) + add_basic_auth("%s:%s" % (login, password), r) + except (TypeError, ImportError, IOError, netrc.NetrcParseError): + pass + + with opener.open(r) as response: + pass + except urllib.error.URLError as e: + if try_again: + logger.debug(2, "checkstatus: trying again") + return self.checkstatus(fetch, ud, d, False) + else: + # debug for now to avoid spamming the logs in e.g. remote sstate searches + logger.debug(2, "checkstatus() urlopen failed: %s" % e) + return False + return True + + def _parse_path(self, regex, s): + """ + Find and group name, version and archive type in the given string s + """ + + m = regex.search(s) + if m: + pname = '' + pver = '' + ptype = '' + + mdict = m.groupdict() + if 'name' in mdict.keys(): + pname = mdict['name'] + if 'pver' in mdict.keys(): + pver = mdict['pver'] + if 'type' in mdict.keys(): + ptype = mdict['type'] + + bb.debug(3, "_parse_path: %s, %s, %s" % (pname, pver, ptype)) + + return (pname, pver, ptype) + + return None + + def _modelate_version(self, version): + if version[0] in ['.', '-']: + if version[1].isdigit(): + version = version[1] + version[0] + version[2:len(version)] + else: + version = version[1:len(version)] + + version = re.sub('-', '.', version) + version = re.sub('_', '.', version) + version = re.sub('(rc)+', '.1000.', version) + version = re.sub('(beta)+', '.100.', version) + version = re.sub('(alpha)+', '.10.', version) + if version[0] == 'v': + version = version[1:len(version)] + return version + + def _vercmp(self, old, new): + """ + Check whether 'new' is newer than 'old' version. We use existing vercmp() for the + purpose. PE is cleared in comparison as it's not for build, and PR is cleared too + for simplicity as it's somehow difficult to get from various upstream format + """ + + (oldpn, oldpv, oldsuffix) = old + (newpn, newpv, newsuffix) = new + + """ + Check for a new suffix type that we have never heard of before + """ + if (newsuffix): + m = self.suffix_regex_comp.search(newsuffix) + if not m: + bb.warn("%s has a possible unknown suffix: %s" % (newpn, newsuffix)) + return False + + """ + Not our package so ignore it + """ + if oldpn != newpn: + return False + + oldpv = self._modelate_version(oldpv) + newpv = self._modelate_version(newpv) + + return bb.utils.vercmp(("0", oldpv, ""), ("0", newpv, "")) + + def _fetch_index(self, uri, ud, d): + """ + Run fetch checkstatus to get directory information + """ + f = tempfile.NamedTemporaryFile() + with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f: + agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12" + fetchcmd = self.basecmd + fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'" + try: + self._runwget(ud, d, fetchcmd, True, workdir=workdir) + fetchresult = f.read() + except bb.fetch2.BBFetchException: + fetchresult = "" + + return fetchresult + + def _check_latest_version(self, url, package, package_regex, current_version, ud, d): + """ + Return the latest version of a package inside a given directory path + If error or no version, return "" + """ + valid = 0 + version = ['', '', ''] + + bb.debug(3, "VersionURL: %s" % (url)) + soup = BeautifulSoup(self._fetch_index(url, ud, d), "html.parser", parse_only=SoupStrainer("a")) + if not soup: + bb.debug(3, "*** %s NO SOUP" % (url)) + return "" + + for line in soup.find_all('a', href=True): + bb.debug(3, "line['href'] = '%s'" % (line['href'])) + bb.debug(3, "line = '%s'" % (str(line))) + + newver = self._parse_path(package_regex, line['href']) + if not newver: + newver = self._parse_path(package_regex, str(line)) + + if newver: + bb.debug(3, "Upstream version found: %s" % newver[1]) + if valid == 0: + version = newver + valid = 1 + elif self._vercmp(version, newver) < 0: + version = newver + + pupver = re.sub('_', '.', version[1]) + + bb.debug(3, "*** %s -> UpstreamVersion = %s (CurrentVersion = %s)" % + (package, pupver or "N/A", current_version[1])) + + if valid: + return pupver + + return "" + + def _check_latest_version_by_dir(self, dirver, package, package_regex, + current_version, ud, d): + """ + Scan every directory in order to get upstream version. + """ + version_dir = ['', '', ''] + version = ['', '', ''] + + dirver_regex = re.compile("(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))") + s = dirver_regex.search(dirver) + if s: + version_dir[1] = s.group('ver') + else: + version_dir[1] = dirver + + dirs_uri = bb.fetch.encodeurl([ud.type, ud.host, + ud.path.split(dirver)[0], ud.user, ud.pswd, {}]) + bb.debug(3, "DirURL: %s, %s" % (dirs_uri, package)) + + soup = BeautifulSoup(self._fetch_index(dirs_uri, ud, d), "html.parser", parse_only=SoupStrainer("a")) + if not soup: + return version[1] + + for line in soup.find_all('a', href=True): + s = dirver_regex.search(line['href'].strip("/")) + if s: + sver = s.group('ver') + + # When prefix is part of the version directory it need to + # ensure that only version directory is used so remove previous + # directories if exists. + # + # Example: pfx = '/dir1/dir2/v' and version = '2.5' the expected + # result is v2.5. + spfx = s.group('pfx').split('/')[-1] + + version_dir_new = ['', sver, ''] + if self._vercmp(version_dir, version_dir_new) <= 0: + dirver_new = spfx + sver + path = ud.path.replace(dirver, dirver_new, True) \ + .split(package)[0] + uri = bb.fetch.encodeurl([ud.type, ud.host, path, + ud.user, ud.pswd, {}]) + + pupver = self._check_latest_version(uri, + package, package_regex, current_version, ud, d) + if pupver: + version[1] = pupver + + version_dir = version_dir_new + + return version[1] + + def _init_regexes(self, package, ud, d): + """ + Match as many patterns as possible such as: + gnome-common-2.20.0.tar.gz (most common format) + gtk+-2.90.1.tar.gz + xf86-input-synaptics-12.6.9.tar.gz + dri2proto-2.3.tar.gz + blktool_4.orig.tar.gz + libid3tag-0.15.1b.tar.gz + unzip552.tar.gz + icu4c-3_6-src.tgz + genext2fs_1.3.orig.tar.gz + gst-fluendo-mp3 + """ + # match most patterns which uses "-" as separator to version digits + pn_prefix1 = "[a-zA-Z][a-zA-Z0-9]*([-_][a-zA-Z]\w+)*\+?[-_]" + # a loose pattern such as for unzip552.tar.gz + pn_prefix2 = "[a-zA-Z]+" + # a loose pattern such as for 80325-quicky-0.4.tar.gz + pn_prefix3 = "[0-9]+[-]?[a-zA-Z]+" + # Save the Package Name (pn) Regex for use later + pn_regex = "(%s|%s|%s)" % (pn_prefix1, pn_prefix2, pn_prefix3) + + # match version + pver_regex = "(([A-Z]*\d+[a-zA-Z]*[\.\-_]*)+)" + + # match arch + parch_regex = "-source|_all_" + + # src.rpm extension was added only for rpm package. Can be removed if the rpm + # packaged will always be considered as having to be manually upgraded + psuffix_regex = "(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)" + + # match name, version and archive type of a package + package_regex_comp = re.compile("(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)" + % (pn_regex, pver_regex, parch_regex, psuffix_regex)) + self.suffix_regex_comp = re.compile(psuffix_regex) + + # compile regex, can be specific by package or generic regex + pn_regex = d.getVar('UPSTREAM_CHECK_REGEX') + if pn_regex: + package_custom_regex_comp = re.compile(pn_regex) + else: + version = self._parse_path(package_regex_comp, package) + if version: + package_custom_regex_comp = re.compile( + "(?P<name>%s)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s)" % + (re.escape(version[0]), pver_regex, parch_regex, psuffix_regex)) + else: + package_custom_regex_comp = None + + return package_custom_regex_comp + + def latest_versionstring(self, ud, d): + """ + Manipulate the URL and try to obtain the latest package version + + sanity check to ensure same name and type. + """ + package = ud.path.split("/")[-1] + current_version = ['', d.getVar('PV'), ''] + + """possible to have no version in pkg name, such as spectrum-fw""" + if not re.search("\d+", package): + current_version[1] = re.sub('_', '.', current_version[1]) + current_version[1] = re.sub('-', '.', current_version[1]) + return (current_version[1], '') + + package_regex = self._init_regexes(package, ud, d) + if package_regex is None: + bb.warn("latest_versionstring: package %s don't match pattern" % (package)) + return ('', '') + bb.debug(3, "latest_versionstring, regex: %s" % (package_regex.pattern)) + + uri = "" + regex_uri = d.getVar("UPSTREAM_CHECK_URI") + if not regex_uri: + path = ud.path.split(package)[0] + + # search for version matches on folders inside the path, like: + # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz + dirver_regex = re.compile("(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/") + m = dirver_regex.search(path) + if m: + pn = d.getVar('PN') + dirver = m.group('dirver') + + dirver_pn_regex = re.compile("%s\d?" % (re.escape(pn))) + if not dirver_pn_regex.search(dirver): + return (self._check_latest_version_by_dir(dirver, + package, package_regex, current_version, ud, d), '') + + uri = bb.fetch.encodeurl([ud.type, ud.host, path, ud.user, ud.pswd, {}]) + else: + uri = regex_uri + + return (self._check_latest_version(uri, package, package_regex, + current_version, ud, d), '') |
