8 files changed, 640 insertions, 0 deletions
diff --git a/utils/checkpackagelib/__init__.py b/utils/checkpackagelib/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/utils/checkpackagelib/__init__.py
diff --git a/utils/checkpackagelib/base.py b/utils/checkpackagelib/base.py
new file mode 100644
index 0000000000..fc09bec9a2
--- /dev/null
+++ b/utils/checkpackagelib/base.py
@@ -0,0 +1,16 @@
+# See utils/checkpackagelib/readme.txt before editing this file.
+
+
+class _CheckFunction(object):
+    def __init__(self, filename, url_to_manual):
+        self.filename = filename
+        self.url_to_manual = url_to_manual
+
+    def before(self):
+        pass
+
+    def check_line(self, lineno, text):
+        pass
+
+    def after(self):
+        pass
diff --git a/utils/checkpackagelib/lib.py b/utils/checkpackagelib/lib.py
new file mode 100644
index 0000000000..91f4ad49b7
--- /dev/null
+++ b/utils/checkpackagelib/lib.py
@@ -0,0 +1,54 @@
+# See utils/checkpackagelib/readme.txt before editing this file.
+
+from base import _CheckFunction
+
+
+class ConsecutiveEmptyLines(_CheckFunction):
+    def before(self):
+        self.lastline = "non empty"
+
+    def check_line(self, lineno, text):
+        if text.strip() == "" == self.lastline.strip():
+            return ["{}:{}: consecutive empty lines"
+                    .format(self.filename, lineno)]
+        self.lastline = text
+
+
+class EmptyLastLine(_CheckFunction):
+    def before(self):
+        self.lastlineno = 0
+        self.lastline = "non empty"
+
+    def check_line(self, lineno, text):
+        self.lastlineno = lineno
+        self.lastline = text
+
+    def after(self):
+        if self.lastline.strip() == "":
+            return ["{}:{}: empty line at end of file"
+                    .format(self.filename, self.lastlineno)]
+
+
+class NewlineAtEof(_CheckFunction):
+    def before(self):
+        self.lastlineno = 0
+        self.lastline = "\n"
+
+    def check_line(self, lineno, text):
+        self.lastlineno = lineno
+        self.lastline = text
+
+    def after(self):
+        if self.lastline == self.lastline.rstrip("\r\n"):
+            return ["{}:{}: missing newline at end of file"
+                    .format(self.filename, self.lastlineno),
+                    self.lastline]
+
+
+class TrailingSpace(_CheckFunction):
+    def check_line(self, lineno, text):
+        line = text.rstrip("\r\n")
+        if line != line.rstrip():
+            return ["{}:{}: line contains trailing whitespace"
+                    .format(self.filename, lineno),
+                    text]
diff --git a/utils/checkpackagelib/lib_config.py b/utils/checkpackagelib/lib_config.py
new file mode 100644
index 0000000000..26ebb393d1
--- /dev/null
+++ b/utils/checkpackagelib/lib_config.py
@@ -0,0 +1,138 @@
+# See utils/checkpackagelib/readme.txt before editing this file.
+# Kconfig generates errors if someone introduces a typo like "boool" instead of
+# "bool", so below check functions don't need to check for things already
+# checked by running "make menuconfig".
+
+import re
+
+from base import _CheckFunction
+# Notice: ignore 'imported but unused' from pyflakes for check functions.
+from lib import ConsecutiveEmptyLines
+from lib import EmptyLastLine
+from lib import NewlineAtEof
+from lib import TrailingSpace
+
+
+def _empty_or_comment(text):
+    line = text.strip()
+    # ignore empty lines and comment lines indented or not
+    return line == "" or line.startswith("#")
+
+
+def _part_of_help_text(text):
+    return text.startswith("\t  ")
+
+
+# used in more than one check
+entries_that_should_not_be_indented = [
+    "choice", "comment", "config", "endchoice", "endif", "endmenu", "if",
+    "menu", "menuconfig", "source"]
+
+
+class AttributesOrder(_CheckFunction):
+    attributes_order_convention = {
+        "bool": 1, "prompt": 1, "string": 1, "default": 2, "depends": 3,
+        "select": 4, "help": 5}
+
+    def before(self):
+        self.state = 0
+
+    def check_line(self, lineno, text):
+        if _empty_or_comment(text) or _part_of_help_text(text):
+            return
+
+        attribute = text.split()[0]
+
+        if attribute in entries_that_should_not_be_indented:
+            self.state = 0
+            return
+        if attribute not in self.attributes_order_convention.keys():
+            return
+        new_state = self.attributes_order_convention[attribute]
+        wrong_order = self.state > new_state
+
+        # save to process next line
+        self.state = new_state
+
+        if wrong_order:
+            return ["{}:{}: attributes order: type, default, depends on,"
+                    " select, help ({}#_config_files)"
+                    .format(self.filename, lineno, self.url_to_manual),
+                    text]
+
+
+class HelpText(_CheckFunction):
+    HELP_TEXT_FORMAT = re.compile("^\t  .{,62}$")
+    URL_ONLY = re.compile("^(http|https|git)://\S*$")
+
+    def before(self):
+        self.help_text = False
+
+    def check_line(self, lineno, text):
+        if _empty_or_comment(text):
+            return
+
+        entry = text.split()[0]
+
+        if entry in entries_that_should_not_be_indented:
+            self.help_text = False
+            return
+        if text.strip() == "help":
+            self.help_text = True
+            return
+
+        if not self.help_text:
+            return
+
+        if self.HELP_TEXT_FORMAT.match(text.rstrip()):
+            return
+        if self.URL_ONLY.match(text.strip()):
+            return
+        return ["{}:{}: help text: <tab><2 spaces><62 chars>"
+                " ({}#writing-rules-config-in)"
+                .format(self.filename, lineno, self.url_to_manual),
+                text,
+                "\t  " + "123456789 " * 6 + "12"]
+
+
+class Indent(_CheckFunction):
+    ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$")
+    entries_that_should_be_indented = [
+        "bool", "default", "depends", "help", "prompt", "select", "string"]
+
+    def before(self):
+        self.backslash = False
+
+    def check_line(self, lineno, text):
+        if _empty_or_comment(text) or _part_of_help_text(text):
+            self.backslash = False
+            return
+
+        entry = text.split()[0]
+
+        last_line_ends_in_backslash = self.backslash
+
+        # calculate for next line
+        if self.ENDS_WITH_BACKSLASH.search(text):
+            self.backslash = True
+        else:
+            self.backslash = False
+
+        if last_line_ends_in_backslash:
+            if text.startswith("\t"):
+                return
+            return ["{}:{}: continuation line should be indented using tabs"
+                    .format(self.filename, lineno),
+                    text]
+
+        if entry in self.entries_that_should_be_indented:
+            if not text.startswith("\t{}".format(entry)):
+                return ["{}:{}: should be indented with one tab"
+                        " ({}#_config_files)"
+                        .format(self.filename, lineno, self.url_to_manual),
+                        text]
+        elif entry in entries_that_should_not_be_indented:
+            if not text.startswith(entry):
+                return ["{}:{}: should not be indented"
+                        .format(self.filename, lineno),
+                        text]
diff --git a/utils/checkpackagelib/lib_hash.py b/utils/checkpackagelib/lib_hash.py
new file mode 100644
index 0000000000..10b27482bc
--- /dev/null
+++ b/utils/checkpackagelib/lib_hash.py
@@ -0,0 +1,72 @@
+# See utils/checkpackagelib/readme.txt before editing this file.
+# The validity of the hashes itself is checked when building, so below check
+# functions don't need to check for things already checked by running
+# "make package-dirclean package-source".
+
+import re
+
+from base import _CheckFunction
+# Notice: ignore 'imported but unused' from pyflakes for check functions.
+from lib import ConsecutiveEmptyLines
+from lib import EmptyLastLine
+from lib import NewlineAtEof
+from lib import TrailingSpace
+
+
+def _empty_line_or_comment(text):
+    return text.strip() == "" or text.startswith("#")
+
+
+class HashFilename(_CheckFunction):
+    def check_line(self, lineno, text):
+        if _empty_line_or_comment(text):
+            return
+
+        fields = text.split()
+        if len(fields) < 3:
+            return
+
+        if '/' in fields[2]:
+            return ["{}:{}: use filename without directory component"
+                    " ({}#adding-packages-hash)"
+                    .format(self.filename, lineno, self.url_to_manual),
+                    text]
+
+
+class HashNumberOfFields(_CheckFunction):
+    def check_line(self, lineno, text):
+        if _empty_line_or_comment(text):
+            return
+
+        fields = text.split()
+        if len(fields) != 3:
+            return ["{}:{}: expected three fields ({}#adding-packages-hash)"
+                    .format(self.filename, lineno, self.url_to_manual),
+                    text]
+
+
+class HashType(_CheckFunction):
+    len_of_hash = {"md5": 32, "sha1": 40, "sha224": 56, "sha256": 64,
+                   "sha384": 96, "sha512": 128}
+
+    def check_line(self, lineno, text):
+        if _empty_line_or_comment(text):
+            return
+
+        fields = text.split()
+        if len(fields) < 2:
+            return
+
+        htype, hexa = fields[:2]
+        if htype == "none":
+            return
+        if htype not in self.len_of_hash.keys():
+            return ["{}:{}: unexpected type of hash ({}#adding-packages-hash)"
+                    .format(self.filename, lineno, self.url_to_manual),
+                    text]
+        if not re.match("^[0-9A-Fa-f]{%s}$" % self.len_of_hash[htype], hexa):
+            return ["{}:{}: hash size does not match type "
+                    "({}#adding-packages-hash)"
+                    .format(self.filename, lineno, self.url_to_manual),
+                    text,
+                    "expected {} hex digits".format(self.len_of_hash[htype])]
diff --git a/utils/checkpackagelib/lib_mk.py b/utils/checkpackagelib/lib_mk.py
new file mode 100644
index 0000000000..6ed6011921
--- /dev/null
+++ b/utils/checkpackagelib/lib_mk.py
@@ -0,0 +1,223 @@
+# See utils/checkpackagelib/readme.txt before editing this file.
+# There are already dependency checks during the build, so below check
+# functions don't need to check for things already checked by exploring the
+# menu options using "make menuconfig" and by running "make" with appropriate
+# packages enabled.
+
+import re
+
+from base import _CheckFunction
+# Notice: ignore 'imported but unused' from pyflakes for check functions.
+from lib import ConsecutiveEmptyLines
+from lib import EmptyLastLine
+from lib import NewlineAtEof
+from lib import TrailingSpace
+
+
+class Indent(_CheckFunction):
+    COMMENT = re.compile("^\s*#")
+    CONDITIONAL = re.compile("^\s*(ifeq|ifneq|endif)\s")
+    ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$")
+    END_DEFINE = re.compile("^\s*endef\s")
+    MAKEFILE_TARGET = re.compile("^[^# \t]+:\s")
+    START_DEFINE = re.compile("^\s*define\s")
+
+    def before(self):
+        self.define = False
+        self.backslash = False
+        self.makefile_target = False
+
+    def check_line(self, lineno, text):
+        if self.START_DEFINE.search(text):
+            self.define = True
+            return
+        if self.END_DEFINE.search(text):
+            self.define = False
+            return
+
+        expect_tabs = False
+        if self.define or self.backslash or self.makefile_target:
+            expect_tabs = True
+        if self.CONDITIONAL.search(text):
+            expect_tabs = False
+
+        # calculate for next line
+        if self.ENDS_WITH_BACKSLASH.search(text):
+            self.backslash = True
+        else:
+            self.backslash = False
+
+        if self.MAKEFILE_TARGET.search(text):
+            self.makefile_target = True
+            return
+        if text.strip() == "":
+            self.makefile_target = False
+            return
+
+        # comment can be indented or not inside define ... endef, so ignore it
+        if self.define and self.COMMENT.search(text):
+            return
+
+        if expect_tabs:
+            if not text.startswith("\t"):
+                return ["{}:{}: expected indent with tabs"
+                        .format(self.filename, lineno),
+                        text]
+        else:
+            if text.startswith("\t"):
+                return ["{}:{}: unexpected indent with tabs"
+                        .format(self.filename, lineno),
+                        text]
+
+
+class PackageHeader(_CheckFunction):
+    def before(self):
+        self.skip = False
+
+    def check_line(self, lineno, text):
+        if self.skip or lineno > 6:
+            return
+
+        if lineno in [1, 5]:
+            if lineno == 1 and text.startswith("include "):
+                self.skip = True
+                return
+            if text.rstrip() != "#" * 80:
+                return ["{}:{}: should be 80 hashes ({}#writing-rules-mk)"
+                        .format(self.filename, lineno, self.url_to_manual),
+                        text,
+                        "#" * 80]
+        elif lineno in [2, 4]:
+            if text.rstrip() != "#":
+                return ["{}:{}: should be 1 hash ({}#writing-rules-mk)"
+                        .format(self.filename, lineno, self.url_to_manual),
+                        text]
+        elif lineno == 6:
+            if text.rstrip() != "":
+                return ["{}:{}: should be a blank line ({}#writing-rules-mk)"
+                        .format(self.filename, lineno, self.url_to_manual),
+                        text]
+
+
+class SpaceBeforeBackslash(_CheckFunction):
+    TAB_OR_MULTIPLE_SPACES_BEFORE_BACKSLASH = re.compile(r"^.*(  |\t)\\$")
+
+    def check_line(self, lineno, text):
+        if self.TAB_OR_MULTIPLE_SPACES_BEFORE_BACKSLASH.match(text.rstrip()):
+            return ["{}:{}: use only one space before backslash"
+                    .format(self.filename, lineno),
+                    text]
+
+
+class TrailingBackslash(_CheckFunction):
+    ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$")
+
+    def before(self):
+        self.backslash = False
+
+    def check_line(self, lineno, text):
+        last_line_ends_in_backslash = self.backslash
+
+        # calculate for next line
+        if self.ENDS_WITH_BACKSLASH.search(text):
+            self.backslash = True
+            self.lastline = text
+            return
+        self.backslash = False
+
+        if last_line_ends_in_backslash and text.strip() == "":
+            return ["{}:{}: remove trailing backslash"
+                    .format(self.filename, lineno - 1),
+                    self.lastline]
+
+
+class TypoInPackageVariable(_CheckFunction):
+    ALLOWED = re.compile("|".join([
+        "ACLOCAL_DIR",
+        "ACLOCAL_HOST_DIR",
+        "BR_CCACHE_INITIAL_SETUP",
+        "BR_NO_CHECK_HASH_FOR",
+        "LINUX_POST_PATCH_HOOKS",
+        "LINUX_TOOLS",
+        "LUA_RUN",
+        "MKFS_JFFS2",
+        "MKIMAGE_ARCH",
+        "PKG_CONFIG_HOST_BINARY",
+        "TARGET_FINALIZE_HOOKS",
+        "XTENSA_CORE_NAME"]))
+    PACKAGE_NAME = re.compile("/([^/]+)\.mk")
+    VARIABLE = re.compile("^([A-Z0-9_]+_[A-Z0-9_]+)\s*(\+|)=")
+
+    def before(self):
+        package = self.PACKAGE_NAME.search(self.filename).group(1)
+        package = package.replace("-", "_").upper()
+        # linux tools do not use LINUX_TOOL_ prefix for variables
+        package = package.replace("LINUX_TOOL_", "")
+        self.package = package
+        self.REGEX = re.compile("^(HOST_)?({}_[A-Z0-9_]+)".format(package))
+        self.FIND_VIRTUAL = re.compile(
+            "^{}_PROVIDES\s*(\+|)=\s*(.*)".format(package))
+        self.virtual = []
+
+    def check_line(self, lineno, text):
+        m = self.VARIABLE.search(text)
+        if m is None:
+            return
+
+        variable = m.group(1)
+
+        # allow to set variables for virtual package this package provides
+        v = self.FIND_VIRTUAL.search(text)
+        if v:
+            self.virtual += v.group(2).upper().split()
+            return
+        for virtual in self.virtual:
+            if variable.startswith("{}_".format(virtual)):
+                return
+
+        if self.ALLOWED.match(variable):
+            return
+        if self.REGEX.search(text) is None:
+            return ["{}:{}: possible typo: {} -> *{}*"
+                    .format(self.filename, lineno, variable, self.package),
+                    text]
+
+
+class UselessFlag(_CheckFunction):
+    DEFAULT_AUTOTOOLS_FLAG = re.compile("^.*{}".format("|".join([
+        "_AUTORECONF\s*=\s*NO",
+        "_LIBTOOL_PATCH\s*=\s*YES"])))
+    DEFAULT_GENERIC_FLAG = re.compile("^.*{}".format("|".join([
+        "_INSTALL_IMAGES\s*=\s*NO",
+        "_INSTALL_REDISTRIBUTE\s*=\s*YES",
+        "_INSTALL_STAGING\s*=\s*NO",
+        "_INSTALL_TARGET\s*=\s*YES"])))
+    END_CONDITIONAL = re.compile("^\s*(endif)")
+    START_CONDITIONAL = re.compile("^\s*(ifeq|ifneq)")
+
+    def before(self):
+        self.conditional = 0
+
+    def check_line(self, lineno, text):
+        if self.START_CONDITIONAL.search(text):
+            self.conditional += 1
+            return
+        if self.END_CONDITIONAL.search(text):
+            self.conditional -= 1
+            return
+
+        # allow non-default conditionally overridden by default
+        if self.conditional > 0:
+            return
+
+        if self.DEFAULT_GENERIC_FLAG.search(text):
+            return ["{}:{}: useless default value ({}#"
+                    "_infrastructure_for_packages_with_specific_build_systems)"
+                    .format(self.filename, lineno, self.url_to_manual),
+                    text]
+
+        if self.DEFAULT_AUTOTOOLS_FLAG.search(text):
+            return ["{}:{}: useless default value "
+                    "({}#_infrastructure_for_autotools_based_packages)"
+                    .format(self.filename, lineno, self.url_to_manual),
+                    text]
diff --git a/utils/checkpackagelib/lib_patch.py b/utils/checkpackagelib/lib_patch.py
new file mode 100644
index 0000000000..591f13c494
--- /dev/null
+++ b/utils/checkpackagelib/lib_patch.py
@@ -0,0 +1,62 @@
+# See utils/checkpackagelib/readme.txt before editing this file.
+# The format of the patch files is tested during the build, so below check
+# functions don't need to check for things already checked by running
+# "make package-dirclean package-patch".
+
+import re
+
+from base import _CheckFunction
+# Notice: ignore 'imported but unused' from pyflakes for check functions.
+from lib import NewlineAtEof
+
+
+class ApplyOrder(_CheckFunction):
+    APPLY_ORDER = re.compile("/\d{1,4}-[^/]*$")
+
+    def before(self):
+        if not self.APPLY_ORDER.search(self.filename):
+            return ["{}:0: use name <number>-<description>.patch "
+                    "({}#_providing_patches)"
+                    .format(self.filename, self.url_to_manual)]
+
+
+class NumberedSubject(_CheckFunction):
+    NUMBERED_PATCH = re.compile("Subject:\s*\[PATCH\s*\d+/\d+\]")
+
+    def before(self):
+        self.git_patch = False
+        self.lineno = 0
+        self.text = None
+
+    def check_line(self, lineno, text):
+        if text.startswith("diff --git"):
+            self.git_patch = True
+            return
+        if self.NUMBERED_PATCH.search(text):
+            self.lineno = lineno
+            self.text = text
+
+    def after(self):
+        if self.git_patch and self.text:
+            return ["{}:{}: generate your patches with 'git format-patch -N'"
+                    .format(self.filename, self.lineno),
+                    self.text]
+
+
+class Sob(_CheckFunction):
+    SOB_ENTRY = re.compile("^Signed-off-by: .*$")
+
+    def before(self):
+        self.found = False
+
+    def check_line(self, lineno, text):
+        if self.found:
+            return
+        if self.SOB_ENTRY.search(text):
+            self.found = True
+
+    def after(self):
+        if not self.found:
+            return ["{}:0: missing Signed-off-by in the header "
+                    "({}#_format_and_licensing_of_the_package_patches)"
+                    .format(self.filename, self.url_to_manual)]
diff --git a/utils/checkpackagelib/readme.txt b/utils/checkpackagelib/readme.txt
new file mode 100644
index 0000000000..85e10fe34b
--- /dev/null
+++ b/utils/checkpackagelib/readme.txt
@@ -0,0 +1,75 @@
+How the scripts are structured:
+- check-package is the main engine, called by the user.
+  For each input file, this script decides which parser should be used and it
+  collects all classes declared in the library file and instantiates them.
+  The main engine opens the input files and it serves each raw line (including
+  newline!) to the method check_line() of every check object.
+  Two special methods before() and after() are used to call the initialization
+  of variables (for the case it needs to keep data across calls) and the
+  equivalent finalization (e.g. for the case a warning must be issued if some
+  pattern is not in the input file).
+- base.py contains the base class for all check functions.
+- lib.py contains the classes for common check functions.
+  Each check function is explicitly included in a given type-parsing library.
+  Do not include every single check function in this file, a class that will
+  only parse hash files should be implemented in the hash-parsing library.
+  When a warning must be issued, the check function returns an array of strings.
+  Each string is a warning message and is displayed if the corresponding verbose
+  level is active. When the script is called without --verbose only the first
+  warning in the returned array is printed; when called with --verbose both
+  first and second warnings are printed; when called with -vv until the third
+  warning is printed; an so on.
+  Helper functions can be defined and will not be called by the main script.
+- lib_type.py contains check functions specific to files of this type.
+
+Some hints when changing this code:
+- prefer O(n) algorithms, where n is the total number of lines in the files
+  processed.
+- when there is no other reason for ordering, use alphabetical order (e.g. keep
+  the check functions in alphabetical order, keep the imports in alphabetical
+  order, and so on).
+- use pyflakes to detect and fix potential problems.
+- use pep8 formatting.
+- keep in mind that for every class the method before() will be called before
+  any line is served to be checked by the method check_line(). A class that
+  checks the filename should only implement the method before(). A function that
+  needs to keep data across calls (e.g. keep the last line before the one being
+  processed) should initialize all variables using this method.
+- keep in mind that for every class the method after() will be called after all
+  lines were served to be checked by the method check_line(). A class that
+  checks the absence of a pattern in the file will need to use this method.
+- try to avoid false warnings. It's better to not issue a warning message to a
+  corner case than have too many false warnings. The second can make users stop
+  using the script.
+- do not check spacing in the input line in every single function. Trailing
+  whitespace and wrong indentation should be checked by separate functions.
+- avoid duplicate tests. Try to test only one thing in each function.
+- in the warning message, include the url to a section from the manual, when
+  applicable. It potentially will make more people know the manual.
+- use short sentences in the warning messages. A complete explanation can be
+  added to show when --verbose is used.
+- when testing, verify the error message is displayed when the error pattern is
+  found, but also verify the error message is not displayed for few
+  well-formatted packages... there are many of these, just pick your favorite
+  as golden package that should not trigger any warning message.
+- check the url displayed by the warning message works.
+
+Usage examples:
+- to get a list of check functions that would be called without actually
+  calling them you can use the --dry-run option:
+$ utils/check-package --dry-run package/yourfavorite/*
+
+- when you just added a new check function, e.g. Something, check how it behaves
+  for all current packages:
+$ utils/check-package --include-only Something $(find package -type f)
+
+- the effective processing time (when the .pyc were already generated and all
+  files to be processed are cached in the RAM) should stay in the order of few
+  seconds:
+$ utils/check-package $(find package -type f) >/dev/null ; \
+  time utils/check-package $(find package -type f) >/dev/null
+
+- vim users can navigate the warnings (most editors probably have similar
+  function) since warnings are generated in the form 'path/file:line: warning':
+$ find package/ -name 'Config.*' > filelist && vim -c \
+  'set makeprg=utils/check-package\ $(cat\ filelist)' -c make -c copen