tools: move scanpypi out of support/scripts/

Move it to the top-level tools/ directory, so that it is easier to
find for users.

Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr>
Cc: Yegor Yefremov <yegorslists@googlemail.com>
Cc: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Cc: Arnout Vandecappelle <arnout@mind.be>
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>

1 files changed, 0 insertions, 653 deletions

diff --git a/support/scripts/scanpypi b/support/scripts/scanpypi
deleted file mode 100755
index bb3899241b..0000000000
--- a/support/scripts/scanpypi
+++ /dev/null
@@ -1,653 +0,0 @@
-#!/usr/bin/env python2
-"""
-
-Utility for building Buildroot packages for existing PyPI packages
-
-Any package built by scanpypi should be manually checked for
-errors.
-"""
-from __future__ import print_function
-import argparse
-import json
-import urllib2
-import sys
-import os
-import shutil
-import StringIO
-import tarfile
-import zipfile
-import errno
-import hashlib
-import re
-import textwrap
-import tempfile
-import imp
-from functools import wraps
-
-def setup_decorator(func, method):
-    """
-    Decorator for distutils.core.setup and setuptools.setup.
-    Puts the arguments with which setup is called as a dict
-    Add key 'method' which should be either 'setuptools' or 'distutils'.
-
-    Keyword arguments:
-    func -- either setuptools.setup or distutils.core.setup
-    method -- either 'setuptools' or 'distutils'
-    """
-
-    @wraps(func)
-    def closure(*args, **kwargs):
-        # Any python packages calls its setup function to be installed.
-        # Argument 'name' of this setup function is the package's name
-        BuildrootPackage.setup_args[kwargs['name']] = kwargs
-        BuildrootPackage.setup_args[kwargs['name']]['method'] = method
-    return closure
-
-# monkey patch
-import setuptools
-setuptools.setup = setup_decorator(setuptools.setup, 'setuptools')
-import distutils
-distutils.core.setup = setup_decorator(setuptools.setup, 'distutils')
-
-def find_file_upper_case(filenames, path='./'):
-    """
-    List generator:
-    Recursively find files that matches one of the specified filenames.
-    Returns a relative path starting with path argument.
-
-    Keyword arguments:
-    filenames -- List of filenames to be found
-    path -- Path to the directory to search
-    """
-    for root, dirs, files in os.walk(path):
-        for file in files:
-            if file.upper() in filenames:
-                yield (os.path.join(root, file))
-
-
-def pkg_buildroot_name(pkg_name):
-    """
-    Returns the Buildroot package name for the PyPI package pkg_name.
-    Remove all non alphanumeric characters except -
-    Also lowers the name and adds 'python-' suffix
-
-    Keyword arguments:
-    pkg_name -- String to rename
-    """
-    name = re.sub('[^\w-]', '', pkg_name.lower())
-    prefix = 'python-'
-    pattern = re.compile('^(?!' + prefix + ')(.+?)$')
-    name = pattern.sub(r'python-\1', name)
-    return name
-
-class DownloadFailed(Exception):
-    pass
-
-class BuildrootPackage():
-    """This class's methods are not meant to be used individually please
-    use them in the correct order:
-
-    __init__
-
-    download_package
-
-    extract_package
-
-    load_module
-
-    get_requirements
-
-    create_package_mk
-
-    create_hash_file
-
-    create_config_in
-
-    """
-    setup_args = {}
-
-    def __init__(self, real_name, pkg_folder):
-        self.real_name = real_name
-        self.buildroot_name = pkg_buildroot_name(self.real_name)
-        self.pkg_dir = os.path.join(pkg_folder, self.buildroot_name)
-        self.mk_name = self.buildroot_name.upper().replace('-', '_')
-        self.as_string = None
-        self.md5_sum = None
-        self.metadata = None
-        self.metadata_name = None
-        self.metadata_url = None
-        self.pkg_req = None
-        self.setup_metadata = None
-        self.tmp_extract = None
-        self.used_url = None
-        self.filename = None
-        self.url = None
-        self.version = None
-
-    def fetch_package_info(self):
-        """
-        Fetch a package's metadata from the python package index
-        """
-        self.metadata_url = 'https://pypi.python.org/pypi/{pkg}/json'.format(
-            pkg=self.real_name)
-        try:
-            pkg_json = urllib2.urlopen(self.metadata_url).read().decode()
-        except urllib2.HTTPError as error:
-            print('ERROR:', error.getcode(), error.msg, file=sys.stderr)
-            print('ERROR: Could not find package {pkg}.\n'
-                  'Check syntax inside the python package index:\n'
-                  'https://pypi.python.org/pypi/ '
-                  .format(pkg=self.real_name))
-            raise
-        except urllib2.URLError:
-            print('ERROR: Could not find package {pkg}.\n'
-                  'Check syntax inside the python package index:\n'
-                  'https://pypi.python.org/pypi/ '
-                  .format(pkg=self.real_name))
-            raise
-        self.metadata = json.loads(pkg_json)
-        self.version = self.metadata['info']['version']
-        self.metadata_name = self.metadata['info']['name']
-
-    def download_package(self):
-        """
-        Download a package using metadata from pypi
-        """
-        try:
-            self.metadata['urls'][0]['filename']
-        except IndexError:
-            print(
-                'Non-conventional package, ',
-                'please check carefully after creation')
-            self.metadata['urls'] = [{
-                'packagetype': 'sdist',
-                'url': self.metadata['info']['download_url'],
-                'md5_digest': None}]
-            # In this case, we can't get the name of the downloaded file
-            # from the pypi api, so we need to find it, this should work
-            urlpath = urllib2.urlparse.urlparse(
-                self.metadata['info']['download_url']).path
-            # urlparse().path give something like
-            # /path/to/file-version.tar.gz
-            # We use basename to remove /path/to
-            self.metadata['urls'][0]['filename'] = os.path.basename(urlpath)
-        for download_url in self.metadata['urls']:
-            if 'bdist' in download_url['packagetype']:
-                continue
-            try:
-                print('Downloading package {pkg} from {url}...'.format(
-                      pkg=self.real_name, url=download_url['url']))
-                download = urllib2.urlopen(download_url['url'])
-            except urllib2.HTTPError as http_error:
-                download = http_error
-            else:
-                self.used_url = download_url
-                self.as_string = download.read()
-                if not download_url['md5_digest']:
-                    break
-                self.md5_sum = hashlib.md5(self.as_string).hexdigest()
-                if self.md5_sum == download_url['md5_digest']:
-                    break
-        else:
-            if download.__class__ == urllib2.HTTPError:
-                raise download
-            raise DownloadFailed('Failed to downloas package {pkg}'
-                                 .format(pkg=self.real_name))
-        self.filename = self.used_url['filename']
-        self.url = self.used_url['url']
-
-    def extract_package(self, tmp_path):
-        """
-        Extract the package contents into a directrory
-
-        Keyword arguments:
-        tmp_path -- directory where you want the package to be extracted
-        """
-        as_file = StringIO.StringIO(self.as_string)
-        if self.filename[-3:] == 'zip':
-            with zipfile.ZipFile(as_file) as as_zipfile:
-                tmp_pkg = os.path.join(tmp_path, self.buildroot_name)
-                try:
-                    os.makedirs(tmp_pkg)
-                except OSError as exception:
-                    if exception.errno != errno.EEXIST:
-                        print("ERROR: ", exception.message, file=sys.stderr)
-                        return None, None
-                    print('WARNING:', exception.message, file=sys.stderr)
-                    print('Removing {pkg}...'.format(pkg=tmp_pkg))
-                    shutil.rmtree(tmp_pkg)
-                    os.makedirs(tmp_pkg)
-                as_zipfile.extractall(tmp_pkg)
-        else:
-            with tarfile.open(fileobj=as_file) as as_tarfile:
-                tmp_pkg = os.path.join(tmp_path, self.buildroot_name)
-                try:
-                    os.makedirs(tmp_pkg)
-                except OSError as exception:
-                    if exception.errno != errno.EEXIST:
-                        print("ERROR: ", exception.message, file=sys.stderr)
-                        return None, None
-                    print('WARNING:', exception.message, file=sys.stderr)
-                    print('Removing {pkg}...'.format(pkg=tmp_pkg))
-                    shutil.rmtree(tmp_pkg)
-                    os.makedirs(tmp_pkg)
-                as_tarfile.extractall(tmp_pkg)
-
-        tmp_extract = '{folder}/{name}-{version}'
-        self.tmp_extract = tmp_extract.format(
-            folder=tmp_pkg,
-            name=self.metadata_name,
-            version=self.version)
-
-    def load_setup(self):
-        """
-        Loads the corresponding setup and store its metadata
-        """
-        current_dir = os.getcwd()
-        os.chdir(self.tmp_extract)
-        sys.path.append(self.tmp_extract)
-        s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
-        setup = imp.load_module('setup', s_file, s_path, s_desc)
-        try:
-            self.setup_metadata = self.setup_args[self.metadata_name]
-        except KeyError:
-            # This means setup was not called which most likely mean that it is
-            # called through the if __name__ == '__main__' directive.
-            # In this case, we can only pray that it is called through a
-            # function called main() in setup.py.
-            setup.main([]) # Will raise AttributeError if not found
-            self.setup_metadata = self.setup_args[self.metadata_name]
-        # Here we must remove the module the hard way.
-        # We must do this because of a very specific case: if a package calls
-        # setup from the __main__ but does not come with a 'main()' function,
-        # for some reason setup.main([]) will successfully call the main
-        # function of a previous package...
-        sys.modules.pop('setup',None)
-        del setup
-        os.chdir(current_dir)
-        sys.path.remove(self.tmp_extract)
-
-    def get_requirements(self, pkg_folder):
-        """
-        Retrieve dependencies from the metadata found in the setup.py script of
-        a pypi package.
-
-        Keyword Arguments:
-        pkg_folder -- location of the already created packages
-        """
-        if 'install_requires' not in self.setup_metadata:
-            self.pkg_req = None
-            return set()
-        self.pkg_req = self.setup_metadata['install_requires']
-        self.pkg_req = [re.sub('([-.\w]+).*', r'\1', req)
-                        for req in self.pkg_req]
-        req_not_found = self.pkg_req
-        self.pkg_req = map(pkg_buildroot_name, self.pkg_req)
-        pkg_tuples = zip(req_not_found, self.pkg_req)
-        # pkg_tuples is a list of tuples that looks like
-        # ('werkzeug','python-werkzeug') because I need both when checking if
-        # dependencies already exist or are already in the download list
-        req_not_found = set(
-            pkg[0] for pkg in pkg_tuples
-            if not os.path.isdir(pkg[1])
-            )
-        return req_not_found
-
-    def __create_mk_header(self):
-        """
-        Create the header of the <package_name>.mk file
-        """
-        header = ['#' * 80 + '\n']
-        header.append('#\n')
-        header.append('# {name}\n'.format(name=self.buildroot_name))
-        header.append('#\n')
-        header.append('#' * 80 + '\n')
-        header.append('\n')
-        return header
-
-    def __create_mk_download_info(self):
-        """
-        Create the lines refering to the download information of the
-        <package_name>.mk file
-        """
-        lines = []
-        version_line = '{name}_VERSION = {version}\n'.format(
-            name=self.mk_name,
-            version=self.version)
-        lines.append(version_line)
-
-        targz = self.filename.replace(
-            self.version,
-            '$({name}_VERSION)'.format(name=self.mk_name))
-        targz_line = '{name}_SOURCE = {filename}\n'.format(
-            name=self.mk_name,
-            filename=targz)
-        lines.append(targz_line)
-
-        if self.filename not in self.url:
-            # Sometimes the filename is in the url, sometimes it's not
-            site_url = self.url
-        else:
-            site_url = self.url[:self.url.find(self.filename)]
-        site_line = '{name}_SITE = {url}'.format(name=self.mk_name,
-                                                 url=site_url)
-        site_line = site_line.rstrip('/') + '\n'
-        lines.append(site_line)
-        return lines
-
-    def __create_mk_setup(self):
-        """
-        Create the line refering to the setup method of the package of the
-        <package_name>.mk file
-
-        There are two things you can use to make an installer
-        for a python package: distutils or setuptools
-        distutils comes with python but does not support dependencies.
-        distutils is mostly still there for backward support.
-        setuptools is what smart people use,
-        but it is not shipped with python :(
-        """
-        lines = []
-        setup_type_line = '{name}_SETUP_TYPE = {method}\n'.format(
-            name=self.mk_name,
-            method=self.setup_metadata['method'])
-        lines.append(setup_type_line)
-        return lines
-
-    def __create_mk_license(self):
-        """
-        Create the lines referring to the package's license informations of the
-        <package_name>.mk file
-
-        The license is found using the metadata from pypi.
-        In the metadata, the license can be found either with standard names in
-        the classifiers part or with naming from the packager in the "License"
-        part.
-
-        From the classifiers, the license is "translated" according to
-        buildroot standards if need be (i.e. from Apache Software License to
-        Apache-2.0).
-
-        From the License part, we cannot guess what formatting the packager
-        used. Hence, it is likely to be incorrect. (i.e. Apache License 2.0
-        instead of Apache-2.0).
-
-        The license's files are found by searching the package for files named
-        license or license.txt (case insensitive).
-        If more than one license file is found, the user is asked to select
-        which ones he wants to use.
-        """
-        license_dict = {
-            'Apache Software License': 'Apache-2.0',
-            'BSD License': 'BSD',
-            'European Union Public Licence 1.0': 'EUPL-1.0',
-            'European Union Public Licence 1.1': 'EUPL-1.1',
-            "GNU General Public License": "GPL",
-            "GNU General Public License v2": "GPL-2.0",
-            "GNU General Public License v2 or later": "GPL-2.0+",
-            "GNU General Public License v3": "GPL-3.0",
-            "GNU General Public License v3 or later": "GPL-3.0+",
-            "GNU Lesser General Public License v2": "LGPL-2.1",
-            "GNU Lesser General Public License v2 or later": "LGPL-2.1+",
-            "GNU Lesser General Public License v3": "LGPL-3.0",
-            "GNU Lesser General Public License v3 or later": "LGPL-3.0+",
-            "GNU Library or Lesser General Public License": "LGPL-2.0",
-            "ISC License": "ISC",
-            "MIT License": "MIT",
-            "Mozilla Public License 1.0": "MPL-1.0",
-            "Mozilla Public License 1.1": "MPL-1.1",
-            "Mozilla Public License 2.0": "MPL-2.0",
-            "Zope Public License": "ZPL"
-            }
-        regexp = re.compile('^License :* *.* *:+ (.*)( \(.*\))?$')
-        classifiers_licenses = [regexp.sub(r"\1", lic)
-                                for lic in self.metadata['info']['classifiers']
-                                if regexp.match(lic)]
-        licenses = map(lambda x: license_dict[x] if x in license_dict else x,
-                       classifiers_licenses)
-        lines = []
-        if not len(licenses):
-            print('WARNING: License has been set to "{license}". It is most'
-                  ' likely wrong, please change it if need be'.format(
-                      license=', '.join(licenses)))
-            licenses = [self.metadata['info']['license']]
-        license_line = '{name}_LICENSE = {license}\n'.format(
-            name=self.mk_name,
-            license=', '.join(licenses))
-        lines.append(license_line)
-
-        filenames = ['LICENCE', 'LICENSE', 'LICENSE.TXT', 'COPYING',
-		     'COPYING.TXT']
-        license_files = list(find_file_upper_case(filenames, self.tmp_extract))
-        license_files = [license.replace(self.tmp_extract, '')[1:]
-                         for license in license_files]
-        if len(license_files) > 0:
-            if len(license_files) > 1:
-                print('More than one file found for license:',
-                      ', '.join(license_files))
-            license_files = [filename
-                             for index, filename in enumerate(license_files)]
-            license_file_line = ('{name}_LICENSE_FILES ='
-                                 ' {files}\n'.format(
-                                     name=self.mk_name,
-                                     files=' '.join(license_files)))
-            lines.append(license_file_line)
-        else:
-            print('WARNING: No license file found,'
-                  ' please specify it manually afterwards')
-            license_file_line = '# No license file found\n'
-
-        return lines
-
-    def __create_mk_requirements(self):
-        """
-        Create the lines referring to the dependencies of the of the
-        <package_name>.mk file
-
-        Keyword Arguments:
-        pkg_name -- name of the package
-        pkg_req -- dependencies of the package
-        """
-        lines = []
-        dependencies_line = ('{name}_DEPENDENCIES ='
-                             ' {reqs}\n'.format(
-                                 name=self.mk_name,
-                                 reqs=' '.join(self.pkg_req)))
-        lines.append(dependencies_line)
-        return lines
-
-    def create_package_mk(self):
-        """
-        Create the lines corresponding to the <package_name>.mk file
-        """
-        pkg_mk = '{name}.mk'.format(name=self.buildroot_name)
-        path_to_mk = os.path.join(self.pkg_dir, pkg_mk)
-        print('Creating {file}...'.format(file=path_to_mk))
-        lines = self.__create_mk_header()
-        lines += self.__create_mk_download_info()
-        lines += self.__create_mk_setup()
-        lines += self.__create_mk_license()
-
-        lines.append('\n')
-        lines.append('$(eval $(python-package))')
-        lines.append('\n')
-        with open(path_to_mk, 'w') as mk_file:
-            mk_file.writelines(lines)
-
-    def create_hash_file(self):
-        """
-        Create the lines corresponding to the <package_name>.hash files
-        """
-        pkg_hash = '{name}.hash'.format(name=self.buildroot_name)
-        path_to_hash = os.path.join(self.pkg_dir, pkg_hash)
-        print('Creating {filename}...'.format(filename=path_to_hash))
-        lines = []
-        if self.used_url['md5_digest']:
-            md5_comment = '# md5 from {url}, sha256 locally computed\n'.format(
-		url=self.metadata_url)
-            lines.append(md5_comment)
-            hash_line = '{method}\t{digest}  {filename}\n'.format(
-                method='md5',
-                digest=self.used_url['md5_digest'],
-                filename=self.filename)
-            lines.append(hash_line)
-        digest = hashlib.sha256(self.as_string).hexdigest()
-        hash_line = '{method}\t{digest}  {filename}\n'.format(
-            method='sha256',
-            digest=digest,
-            filename=self.filename)
-        lines.append(hash_line)
-
-        with open(path_to_hash, 'w') as hash_file:
-            hash_file.writelines(lines)
-
-    def create_config_in(self):
-        """
-        Creates the Config.in file of a package
-        """
-        path_to_config = os.path.join(self.pkg_dir, 'Config.in')
-        print('Creating {file}...'.format(file=path_to_config))
-        lines = []
-        config_line = 'config BR2_PACKAGE_{name}\n'.format(
-            name=self.mk_name)
-        lines.append(config_line)
-
-        bool_line = '\tbool "{name}"\n'.format(name=self.buildroot_name)
-        lines.append(bool_line)
-        if self.pkg_req:
-            for dep in self.pkg_req:
-                dep_line = '\tselect BR2_PACKAGE_{req} # runtime\n'.format(
-                    req=dep.upper().replace('-', '_'))
-                lines.append(dep_line)
-
-        lines.append('\thelp\n')
-
-        help_lines = textwrap.wrap(self.metadata['info']['summary'],
-                                   initial_indent='\t  ',
-                                   subsequent_indent='\t  ')
-
-        # make sure a help text is terminated with a full stop
-        if help_lines[-1][-1] != '.':
-            help_lines[-1] += '.'
-
-        # \t + two spaces is 3 char long
-        help_lines.append('')
-        help_lines.append('\t  ' + self.metadata['info']['home_page'])
-        help_lines = map(lambda x: x + '\n', help_lines)
-        lines += help_lines
-
-        with open(path_to_config, 'w') as config_file:
-            config_file.writelines(lines)
-
-
-def main():
-    # Building the parser
-    parser = argparse.ArgumentParser(
-        description="Creates buildroot packages from the metadata of "
-                    "an existing PyPI packages and include it "
-                    "in menuconfig")
-    parser.add_argument("packages",
-                        help="list of packages to be created",
-                        nargs='+')
-    parser.add_argument("-o", "--output",
-                        help="""
-                        Output directory for packages.
-                        Default is ./package
-                        """,
-                        default='./package')
-
-    args = parser.parse_args()
-    packages = list(set(args.packages))
-
-    # tmp_path is where we'll extract the files later
-    tmp_prefix = 'scanpypi-'
-    pkg_folder = args.output
-    tmp_path = tempfile.mkdtemp(prefix=tmp_prefix)
-    try:
-        for real_pkg_name in packages:
-            package = BuildrootPackage(real_pkg_name, pkg_folder)
-            print('buildroot package name for {}:'.format(package.real_name),
-                  package.buildroot_name)
-            # First we download the package
-            # Most of the info we need can only be found inside the package
-            print('Package:', package.buildroot_name)
-            print('Fetching package', package.real_name)
-            try:
-                package.fetch_package_info()
-            except (urllib2.URLError, urllib2.HTTPError):
-                continue
-            if package.metadata_name.lower() == 'setuptools':
-                # setuptools imports itself, that does not work very well
-                # with the monkey path at the begining
-                print('Error: setuptools cannot be built using scanPyPI')
-                continue
-
-            try:
-                package.download_package()
-            except urllib2.HTTPError as error:
-                print('Error: {code} {reason}'.format(code=error.code,
-                                                      reason=error.reason))
-                print('Error downloading package :', package.buildroot_name)
-                print()
-                continue
-
-            # extract the tarball
-            try:
-                package.extract_package(tmp_path)
-            except (tarfile.ReadError, zipfile.BadZipfile):
-                print('Error extracting package {}'.format(package.real_name))
-                print()
-                continue
-
-            # Loading the package install info from the package
-            try:
-                package.load_setup()
-            except ImportError as err:
-                if 'buildutils' in err.message:
-                    print('This package needs buildutils')
-                else:
-                    raise
-                continue
-            except AttributeError:
-                print('Error: Could not install package {pkg}'.format(
-                    pkg=package.real_name))
-                continue
-
-            # Package requirement are an argument of the setup function
-            req_not_found = package.get_requirements(pkg_folder)
-            req_not_found = req_not_found.difference(packages)
-
-            packages += req_not_found
-            if req_not_found:
-                print('Added packages \'{pkgs}\' as dependencies of {pkg}'
-                      .format(pkgs=", ".join(req_not_found),
-                              pkg=package.buildroot_name))
-            print('Checking if package {name} already exists...'.format(
-                name=package.pkg_dir))
-            try:
-                os.makedirs(package.pkg_dir)
-            except OSError as exception:
-                if exception.errno != errno.EEXIST:
-                    print("ERROR: ", exception.message, file=sys.stderr)
-                    continue
-                print('Error: Package {name} already exists'
-                      .format(name=package.pkg_dir))
-                del_pkg = raw_input(
-                    'Do you want to delete existing package ? [y/N]')
-                if del_pkg.lower() == 'y':
-                    shutil.rmtree(package.pkg_dir)
-                    os.makedirs(package.pkg_dir)
-                else:
-                    continue
-            package.create_package_mk()
-
-            package.create_hash_file()
-
-            package.create_config_in()
-            print()
-            # printing an empty line for visual confort
-    finally:
-        shutil.rmtree(tmp_path)
-
-if __name__ == "__main__":
-    main()