[lit] Refactor out some more common lit configuration code.

debuginfo-tests has need to reuse a lot of common configuration
from clang and lld, and in general it seems like all of the
projects which are tightly coupled (e.g. lld, clang, llvm, lldb,
etc) can benefit from knowing about one other.  For example,
lldb needs to know various things about how to run clang in its
test suite.  Since there's a lot of common substitutions and
operations that need to be shared among projects, sinking this
up into LLVM makes sense.

In addition, this patch introduces a function add_tool_substitution
which handles all the dirty intricacies of matching tool names
which was previously copied around the various config files.  This
is now a simple straightforward interface which is hard to mess
up.

Differential Revision: https://reviews.llvm.org/D37944

llvm-svn: 313919

6 files changed, 216 insertions, 244 deletions

diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py
index f1bb4578dc0..0f1733c73bb 100644
--- a/clang/test/lit.cfg.py
+++ b/clang/test/lit.cfg.py
@@ -10,6 +10,7 @@ import lit.formats
 import lit.util
 
 from lit.llvm import llvm_config
+from lit.llvm import ToolFilter
 
 # Configuration file for the 'lit' test runner.
 
@@ -112,55 +113,12 @@ config.substitutions.append( ('%PATH%', config.environment['PATH']) )
 if config.clang_examples:
     config.available_features.add('examples')
 
-# Note that when substituting %clang_cc1 also fill in the include directory of
-# the builtin headers. Those are part of even a freestanding environment, but
-# Clang relies on the driver to locate them.
-def getClangBuiltinIncludeDir(clang):
-    # FIXME: Rather than just getting the version, we should have clang print
-    # out its resource dir here in an easy to scrape form.
-    cmd = subprocess.Popen([clang, '-print-file-name=include'],
-                           stdout=subprocess.PIPE,
-                           env=config.environment)
-    if not cmd.stdout:
-      lit_config.fatal("Couldn't find the include dir for Clang ('%s')" % clang)
-    dir = cmd.stdout.read().strip()
-    if sys.platform in ['win32'] and not llvm_config.use_lit_shell:
-        # Don't pass dosish path separator to msys bash.exe.
-        dir = dir.replace('\\', '/')
-    # Ensure the result is an ascii string, across Python2.5+ - Python3.
-    return str(dir.decode('ascii'))
-
-def makeItaniumABITriple(triple):
-    m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
-    if not m:
-      lit_config.fatal("Could not turn '%s' into Itanium ABI triple" % triple)
-    if m.group(3).lower() != 'win32':
-      # All non-win32 triples use the Itanium ABI.
-      return triple
-    return m.group(1) + '-' + m.group(2) + '-mingw32'
-
-def makeMSABITriple(triple):
-    m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
-    if not m:
-      lit_config.fatal("Could not turn '%s' into MS ABI triple" % triple)
-    isa = m.group(1).lower()
-    vendor = m.group(2).lower()
-    os = m.group(3).lower()
-    if os == 'win32':
-      # If the OS is win32, we're done.
-      return triple
-    if isa.startswith('x86') or isa == 'amd64' or re.match(r'i\d86', isa):
-      # For x86 ISAs, adjust the OS.
-      return isa + '-' + vendor + '-win32'
-    # -win32 is not supported for non-x86 targets; use a default.
-    return 'i686-pc-win32'
-
+builtin_include_dir = llvm_config.get_clang_builtin_include_dir(config.clang)
 config.substitutions.append( ('%clang_analyze_cc1',
                               '%clang_cc1 -analyze %analyze') )
 config.substitutions.append( ('%clang_cc1',
                               '%s -cc1 -internal-isystem %s -nostdsysteminc'
-                              % (config.clang,
-                                 getClangBuiltinIncludeDir(config.clang))) )
+                              % (config.clang, builtin_include_dir)) )
 config.substitutions.append( ('%clang_cpp', ' ' + config.clang +
                               ' --driver-mode=cpp '))
 config.substitutions.append( ('%clang_cl', ' ' + config.clang +
@@ -168,16 +126,20 @@ config.substitutions.append( ('%clang_cl', ' ' + config.clang +
 config.substitutions.append( ('%clangxx', ' ' + config.clang +
                               ' --driver-mode=g++ '))
 config.substitutions.append( ('%clang', ' ' + config.clang + ' ') )
-config.substitutions.append( ('%test_debuginfo', ' ' + config.llvm_src_root + '/utils/test_debuginfo.pl ') )
-config.substitutions.append( ('%itanium_abi_triple', makeItaniumABITriple(config.target_triple)) )
-config.substitutions.append( ('%ms_abi_triple', makeMSABITriple(config.target_triple)) )
-config.substitutions.append( ('%resource_dir', getClangBuiltinIncludeDir(config.clang)) )
+config.substitutions.append( ('%test_debuginfo',
+                             ' ' + config.llvm_src_root +  '/utils/test_debuginfo.pl ') )
+config.substitutions.append( ('%itanium_abi_triple',
+                             llvm_config.make_itanium_abi_triple(config.target_triple)) )
+config.substitutions.append( ('%ms_abi_triple',
+                             llvm_config.make_msabi_triple(config.target_triple)) )
+config.substitutions.append( ('%resource_dir', builtin_include_dir) )
 config.substitutions.append( ('%python', config.python_executable) )
 
 # The host triple might not be set, at least if we're compiling clang from
 # an already installed llvm.
 if config.host_triple and config.host_triple != '@LLVM_HOST_TRIPLE@':
-    config.substitutions.append( ('%target_itanium_abi_host_triple', '--target=%s' % makeItaniumABITriple(config.host_triple)) )
+    config.substitutions.append( ('%target_itanium_abi_host_triple',
+                                 '--target=%s' % llvm_config.make_itanium_abi_triple(config.host_triple)) )
 else:
     config.substitutions.append( ('%target_itanium_abi_host_triple', '') )
 
@@ -207,50 +169,27 @@ config.substitutions.append(
     (' %clang-cl ',
      """*** invalid substitution, use '%clang_cl'. ***""") )
 
-# For each occurrence of a clang tool name as its own word, replace it
-# with the full path to the build directory holding that tool.  This
-# ensures that we are testing the tools just built and not some random
+# For each occurrence of a clang tool name, replace it with the full path to
+# the build directory holding that tool.  We explicitly specify the directories
+# to search to ensure that we get the tools just built and not some random
 # tools that might happen to be in the user's PATH.
-tool_dirs = os.path.pathsep.join((config.clang_tools_dir, config.llvm_tools_dir))
-
-# Regex assertions to reject neighbor hyphens/dots (seen in some tests).
-# For example, don't match 'clang-check-' or '.clang-format'.
-NoPreHyphenDot = r"(?<!(-|\.))"
-NoPostHyphenDot = r"(?!(-|\.))"
-NoPostBar = r"(?!(/|\\))"
-
-tool_patterns = [r"\bFileCheck\b",
-                 r"\bc-index-test\b",
-                 NoPreHyphenDot + r"\bclang-check\b" + NoPostHyphenDot,
-                 NoPreHyphenDot + r"\bclang-diff\b" + NoPostHyphenDot,
-                 NoPreHyphenDot + r"\bclang-format\b" + NoPostHyphenDot,
-                 # FIXME: Some clang test uses opt?
-                 NoPreHyphenDot + r"\bopt\b" + NoPostBar + NoPostHyphenDot,
-                 # Handle these specially as they are strings searched
-                 # for during testing.
-                 r"\| \bcount\b",
-                 r"\| \bnot\b"]
+tool_dirs = [config.clang_tools_dir, config.llvm_tools_dir]
+
+tool_patterns = [
+    'FileCheck', 'c-index-test',
+    ToolFilter('clang-check', pre='-.', post='-.'),
+    ToolFilter('clang-diff', pre='-.', post='-.'),
+    ToolFilter('clang-format', pre='-.', post='-.'),
+    # FIXME: Some clang test uses opt?
+    ToolFilter('opt', pre='-.', post=r'/\-.'),
+    # Handle these specially as they are strings searched for during testing.
+    ToolFilter(r'\| \bcount\b', verbatim=True),
+    ToolFilter(r'\| \bnot\b', verbatim=True)]
 
 if config.clang_examples:
-    tool_patterns.append(NoPreHyphenDot + r"\bclang-interpreter\b" + NoPostHyphenDot)
-
-for pattern in tool_patterns:
-    # Extract the tool name from the pattern.  This relies on the tool
-    # name being surrounded by \b word match operators.  If the
-    # pattern starts with "| ", include it in the string to be
-    # substituted.
-    tool_match = re.match(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_]+)\\b\W*$",
-                          pattern)
-    tool_pipe = tool_match.group(2)
-    tool_name = tool_match.group(4)
-    tool_path = lit.util.which(tool_name, tool_dirs)
-    if not tool_path:
-        # Warn, but still provide a substitution.
-        lit_config.note('Did not find ' + tool_name + ' in ' + tool_dirs)
-        tool_path = config.clang_tools_dir + '/' + tool_name
-    config.substitutions.append((pattern, tool_pipe + tool_path))
-
-###
+    tool_patterns.append(ToolFilter('clang-interpreter', '-.', '-.'))
+
+llvm_config.add_tool_substitutions(tool_patterns, tool_dirs)
 
 # Set available features we allow tests to conditionalize on.
 #
diff --git a/lld/test/lit.cfg.py b/lld/test/lit.cfg.py
index f1ee3852a0a..f3d14150247 100644
--- a/lld/test/lit.cfg.py
+++ b/lld/test/lit.cfg.py
@@ -10,6 +10,7 @@ import lit.formats
 import lit.util
 
 from lit.llvm import llvm_config
+from lit.llvm import ToolFilter
 
 # Configuration file for the 'lit' test runner.
 
@@ -36,58 +37,30 @@ config.test_source_root = os.path.dirname(__file__)
 config.test_exec_root = os.path.join(config.lld_obj_root, 'test')
 
 # Tweak the PATH to include the tools dir and the scripts dir.
-llvm_config.with_environment('PATH', [config.llvm_tools_dir, config.lld_tools_dir], append_path=True)
+llvm_config.with_environment('PATH',
+                             [config.llvm_tools_dir, config.lld_tools_dir], append_path=True)
 
-llvm_config.with_environment('LD_LIBRARY_PATH', [config.lld_libs_dir, config.llvm_libs_dir], append_path=True)
+llvm_config.with_environment('LD_LIBRARY_PATH',
+                             [config.lld_libs_dir, config.llvm_libs_dir], append_path=True)
 
-# For each occurrence of a lld tool name as its own word, replace it
-# with the full path to the build directory holding that tool.  This
-# ensures that we are testing the tools just built and not some random
+# For each occurrence of a clang tool name, replace it with the full path to
+# the build directory holding that tool.  We explicitly specify the directories
+# to search to ensure that we get the tools just built and not some random
 # tools that might happen to be in the user's PATH.
-
-# Regex assertions to reject neighbor hyphens/dots (seen in some tests).
-# For example, we want to prefix 'lld' and 'ld.lld' but not the 'lld' inside
-# of 'ld.lld'.
-NoPreJunk = r"(?<!(-|\.|/))"
-NoPostJunk = r"(?!(-|\.))"
+tool_dirs = [config.lld_tools_dir, config.llvm_tools_dir]
 
 config.substitutions.append( (r"\bld.lld\b", 'ld.lld --full-shutdown') )
 
-tool_patterns = [r"\bFileCheck\b",
-                 r"\bnot\b",
-                 NoPreJunk + r"\blld\b" + NoPostJunk,
-                 r"\bld.lld\b",
-                 r"\blld-link\b",
-                 r"\bllvm-as\b",
-                 r"\bllvm-mc\b",
-                 r"\bllvm-nm\b",
-                 r"\bllvm-objdump\b",
-                 r"\bllvm-pdbutil\b",
-                 r"\bllvm-readobj\b",
-                 r"\bobj2yaml\b",
-                 r"\byaml2obj\b"]
-
-for pattern in tool_patterns:
-    # Extract the tool name from the pattern.  This relies on the tool
-    # name being surrounded by \b word match operators.  If the
-    # pattern starts with "| ", include it in the string to be
-    # substituted.
-    tool_match = re.match(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_\.]+)\\b\W*$",
-                          pattern)
-    tool_pipe = tool_match.group(2)
-    tool_name = tool_match.group(4)
-    tool_path = lit.util.which(tool_name, config.environment['PATH'])
-    if not tool_path:
-        # Warn, but still provide a substitution.
-        lit_config.note('Did not find ' + tool_name + ' in ' + path)
-        tool_path = config.llvm_tools_dir + '/' + tool_name
-    config.substitutions.append((pattern, tool_pipe + tool_path))
+tool_patterns = [
+    'FileCheck', 'not', 'ld.lld', 'lld-link', 'llvm-as', 'llvm-mc', 'llvm-nm',
+    'llvm-objdump', 'llvm-pdbutil', 'llvm-readobj', 'obj2yaml', 'yaml2obj',
+    ToolFilter('lld', pre='-.', post='-.')]
+
+llvm_config.add_tool_substitutions(tool_patterns, tool_dirs)
 
 # Add site-specific substitutions.
 config.substitutions.append( ('%python', config.python_executable) )
 
-###
-
 # When running under valgrind, we mangle '-vg' onto the end of the triple so we
 # can check it with XFAIL and XTARGET.
 if lit_config.useValgrind:
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 87e2a06e387..28e770d6308 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -11,6 +11,7 @@ import subprocess
 import lit.util
 import lit.formats
 from lit.llvm import llvm_config
+from lit.llvm import ToolFilter
 
 # name: The name of this test suite.
 config.name = 'LLVM'
@@ -134,108 +135,39 @@ else:
 # The regex is a pre-assertion to avoid matching a preceding
 # dot, hyphen, carat, or slash (.foo, -foo, etc.).  Some patterns
 # also have a post-assertion to not match a trailing hyphen (foo-).
-NOJUNK = r"(?<!\.|-|\^|/|<)"
-
-
-def find_tool_substitution(pattern):
-    # Extract the tool name from the pattern.  This relies on the tool
-    # name being surrounded by \b word match operators.  If the
-    # pattern starts with "| ", include it in the string to be
-    # substituted.
-    tool_match = re.match(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_]+)\\b\W*$",
-                          pattern)
-    tool_pipe = tool_match.group(2)
-    tool_name = tool_match.group(4)
-    # Did the user specify the tool path + arguments? This allows things like
-    # llvm-lit "-Dllc=llc -enable-misched -verify-machineinstrs"
-    tool_path = lit_config.params.get(tool_name)
-    if tool_path is None:
-        tool_path = lit.util.which(tool_name, config.llvm_tools_dir)
-        if tool_path is None:
-            return tool_name, tool_path, tool_pipe
-    if (tool_name == "llc" and
-       'LLVM_ENABLE_MACHINE_VERIFIER' in os.environ and
-       os.environ['LLVM_ENABLE_MACHINE_VERIFIER'] == "1"):
-        tool_path += " -verify-machineinstrs"
-    if (tool_name == "llvm-go"):
-        tool_path += " go=" + config.go_executable
-    return tool_name, tool_path, tool_pipe
-
-
-for pattern in [r"\bbugpoint\b(?!-)",
-                NOJUNK + r"\bllc\b",
-                r"\blli\b",
-                r"\bllvm-ar\b",
-                r"\bllvm-as\b",
-                r"\bllvm-bcanalyzer\b",
-                r"\bllvm-config\b",
-                r"\bllvm-cov\b",
-                r"\bllvm-cxxdump\b",
-                r"\bllvm-cvtres\b",
-                r"\bllvm-diff\b",
-                r"\bllvm-dis\b",
-                r"\bllvm-dsymutil\b",
-                r"\bllvm-dwarfdump\b",
-                r"\bllvm-extract\b",
-                r"\bllvm-isel-fuzzer\b",
-                r"\bllvm-lib\b",
-                r"\bllvm-link\b",
-                r"\bllvm-lto\b",
-                r"\bllvm-lto2\b",
-                r"\bllvm-mc\b",
-                r"\bllvm-mcmarkup\b",
-                r"\bllvm-modextract\b",
-                r"\bllvm-nm\b",
-                r"\bllvm-objcopy\b",
-                r"\bllvm-objdump\b",
-                r"\bllvm-pdbutil\b",
-                r"\bllvm-profdata\b",
-                r"\bllvm-ranlib\b",
-                r"\bllvm-readobj\b",
-                r"\bllvm-rtdyld\b",
-                r"\bllvm-size\b",
-                r"\bllvm-split\b",
-                r"\bllvm-strings\b",
-                r"\bllvm-tblgen\b",
-                r"\bllvm-c-test\b",
-                r"\bllvm-cxxfilt\b",
-                r"\bllvm-xray\b",
-                NOJUNK + r"\bllvm-symbolizer\b",
-                NOJUNK + r"\bopt\b",
-                r"\bFileCheck\b",
-                r"\bobj2yaml\b",
-                NOJUNK + r"\bsancov\b",
-                NOJUNK + r"\bsanstats\b",
-                r"\byaml2obj\b",
-                r"\byaml-bench\b",
-                r"\bverify-uselistorder\b",
-                # Handle these specially as they are strings searched
-                # for during testing.
-                r"\| \bcount\b",
-                r"\| \bnot\b"]:
-    tool_name, tool_path, tool_pipe = find_tool_substitution(pattern)
-    if not tool_path:
-        # Warn, but still provide a substitution.
-        lit_config.note('Did not find ' + tool_name + ' in ' + config.llvm_tools_dir)
-        tool_path = config.llvm_tools_dir + '/' + tool_name
-    config.substitutions.append((pattern, tool_pipe + tool_path))
+JUNKCHARS = r".-^/<"
+
+required_tools = [
+    'lli', 'llvm-ar', 'llvm-as', 'llvm-bcanalyzer', 'llvm-config', 'llvm-cov',
+    'llvm-cxxdump', 'llvm-cvtres', 'llvm-diff', 'llvm-dis', 'llvm-dsymutil',
+    'llvm-dwarfdump', 'llvm-extract', 'llvm-isel-fuzzer', 'llvm-lib',
+    'llvm-link', 'llvm-lto', 'llvm-lto2', 'llvm-mc', 'llvm-mcmarkup',
+    'llvm-modextract', 'llvm-nm', 'llvm-objcopy', 'llvm-objdump',
+    'llvm-pdbutil', 'llvm-profdata', 'llvm-ranlib', 'llvm-readobj',
+    'llvm-rtdyld', 'llvm-size', 'llvm-split', 'llvm-strings', 'llvm-tblgen',
+    'llvm-c-test', 'llvm-cxxfilt', 'llvm-xray', 'yaml2obj', 'obj2yaml',
+    'FileCheck', 'yaml-bench', 'verify-uselistorder',
+    ToolFilter('bugpoint', post='-'),
+    ToolFilter('llc', pre=JUNKCHARS),
+    ToolFilter('llvm-symbolizer', pre=JUNKCHARS),
+    ToolFilter('opt', JUNKCHARS),
+    ToolFilter('sancov', pre=JUNKCHARS),
+    ToolFilter('sanstats', pre=JUNKCHARS),
+    # Handle these specially as they are strings searched for during testing.
+    ToolFilter(r'\| \bcount\b', verbatim=True),
+    ToolFilter(r'\| \bnot\b', verbatim=True)]
+
+llvm_config.add_tool_substitutions(required_tools, config.llvm_tools_dir)
 
 # For tools that are optional depending on the config, we won't warn
 # if they're missing.
-for pattern in [r"\bllvm-go\b",
-                r"\bllvm-mt\b",
-                r"\bKaleidoscope-Ch3\b",
-                r"\bKaleidoscope-Ch4\b",
-                r"\bKaleidoscope-Ch5\b",
-                r"\bKaleidoscope-Ch6\b",
-                r"\bKaleidoscope-Ch7\b",
-                r"\bKaleidoscope-Ch8\b"]:
-    tool_name, tool_path, tool_pipe = find_tool_substitution(pattern)
-    if not tool_path:
-        # Provide a substitution anyway, for the sake of consistent errors.
-        tool_path = config.llvm_tools_dir + '/' + tool_name
-    config.substitutions.append((pattern, tool_pipe + tool_path))
 
+optional_tools = [
+    'llvm-go', 'llvm-mt', 'Kaleidoscope-Ch3', 'Kaleidoscope-Ch4',
+    'Kaleidoscope-Ch5', 'Kaleidoscope-Ch6', 'Kaleidoscope-Ch7',
+    'Kaleidoscope-Ch8']
+llvm_config.add_tool_substitutions(optional_tools, config.llvm_tools_dir,
+                                   warn_missing=False)
 
 ### Targets
 
diff --git a/llvm/utils/lit/lit/llvm/__init__.py b/llvm/utils/lit/lit/llvm/__init__.py
index c4cad04649f..4a9249978dd 100644
--- a/llvm/utils/lit/lit/llvm/__init__.py
+++ b/llvm/utils/lit/lit/llvm/__init__.py
@@ -1,9 +1,51 @@
-
 from lit.llvm import config
+import lit.util
+import re
 
 llvm_config = None
 
+class ToolFilter(object):
+    """
+        String-like class used to build regex substitution patterns for
+        llvm tools.  Handles things like adding word-boundary patterns,
+        and filtering characters from the beginning an end of a tool name
+    """
+
+    def __init__(self, name, pre=None, post=None, verbatim=False):
+        """
+            Construct a ToolFilter.
+
+            name: the literal name of the substitution to look for.
+
+            pre: If specified, the substitution will not find matches where
+            the character immediately preceding the word-boundary that begins
+            `name` is any of the characters in the string `pre`.
+
+            post: If specified, the substitution will not find matches where
+            the character immediately after the word-boundary that ends `name`
+            is any of the characters specified in the string `post`.
+
+            verbatim: If True, `name` is an exact regex that is passed to the
+            underlying substitution
+        """
+        if verbatim:
+            self.regex = name
+            return
+
+        def not_in(chars, where=''):
+            if not chars:
+                return ''
+            pattern_str = '|'.join(re.escape(x) for x in chars)
+            return r'(?{}!({}))'.format(where, pattern_str)
+
+        self.regex = not_in(pre, '<') + r'\b' + name + r'\b' + not_in(post)
+
+    def __str__(self):
+        return self.regex
+
+
 def initialize(lit_config, test_config):
     global llvm_config
+
     llvm_config = config.LLVMConfig(lit_config, test_config)
 
diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py
index d2db46135ad..cc0f6750444 100644
--- a/llvm/utils/lit/lit/llvm/config.py
+++ b/llvm/utils/lit/lit/llvm/config.py
@@ -136,22 +136,24 @@ class LLVMConfig(object):
             if name in self.config.environment:
                 del self.config.environment[name]
 
-    def feature_config(self, features, encoding = 'ascii'):
-        # Ask llvm-config about the specified feature.
-        arguments = [x for (x, _) in features]
+    def get_process_output(self, command):
         try:
-            config_path = os.path.join(self.config.llvm_tools_dir, 'llvm-config')
-
-            llvm_config_cmd = subprocess.Popen(
-                [config_path] + arguments,
-                stdout = subprocess.PIPE,
-                env=self.config.environment)
+            cmd = subprocess.Popen(
+                command, stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE, env=self.config.environment)
+            stdout, stderr = cmd.communicate()
+            return (stdout, stderr)
         except OSError:
-            self.lit_config.fatal("Could not find llvm-config in " + self.config.llvm_tools_dir)
+            self.lit_config.fatal("Could not run process %s" % command)
+
+    def feature_config(self, features):
+        # Ask llvm-config about the specified feature.
+        arguments = [x for (x, _) in features]
+        config_path = os.path.join(self.config.llvm_tools_dir, 'llvm-config')
 
-        output, _ = llvm_config_cmd.communicate()
-        output = output.decode(encoding)
+        output, _ = self.get_process_output([config_path] + arguments)
         lines = output.split('\n')
+
         for (feature_line, (_, patterns)) in zip(lines, features):
             # We should have either a callable or a dictionary.  If it's a
             # dictionary, grep each key against the output and use the value if
@@ -163,3 +165,85 @@ class LLVMConfig(object):
                 for (re_pattern, feature) in patterns.items():
                     if re.search(re_pattern, feature_line):
                         self.config.available_features.add(feature)
+
+
+    # Note that when substituting %clang_cc1 also fill in the include directory of
+    # the builtin headers. Those are part of even a freestanding environment, but
+    # Clang relies on the driver to locate them.
+    def get_clang_builtin_include_dir(self, clang):
+        # FIXME: Rather than just getting the version, we should have clang print
+        # out its resource dir here in an easy to scrape form.
+        clang_dir, _ = self.get_process_output([clang, '-print-file-name=include'])
+
+        if not clang_dir:
+          self.lit_config.fatal("Couldn't find the include dir for Clang ('%s')" % clang)
+
+        clang_dir = clang_dir.strip()
+        if sys.platform in ['win32'] and not self.use_lit_shell:
+            # Don't pass dosish path separator to msys bash.exe.
+            clang_dir = clang_dir.replace('\\', '/')
+        # Ensure the result is an ascii string, across Python2.5+ - Python3.
+        return clang_dir
+
+    def make_itanium_abi_triple(self, triple):
+        m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
+        if not m:
+          self.lit_config.fatal("Could not turn '%s' into Itanium ABI triple" % triple)
+        if m.group(3).lower() != 'win32':
+          # All non-win32 triples use the Itanium ABI.
+          return triple
+        return m.group(1) + '-' + m.group(2) + '-mingw32'
+
+    def make_msabi_triple(self, triple):
+        m = re.match(r'(\w+)-(\w+)-(\w+)', triple)
+        if not m:
+          self.lit_config.fatal("Could not turn '%s' into MS ABI triple" % triple)
+        isa = m.group(1).lower()
+        vendor = m.group(2).lower()
+        os = m.group(3).lower()
+        if os == 'win32':
+          # If the OS is win32, we're done.
+          return triple
+        if isa.startswith('x86') or isa == 'amd64' or re.match(r'i\d86', isa):
+          # For x86 ISAs, adjust the OS.
+          return isa + '-' + vendor + '-win32'
+        # -win32 is not supported for non-x86 targets; use a default.
+        return 'i686-pc-win32'
+
+    def add_tool_substitutions(self, tools, search_dirs, warn_missing = True):
+        if lit.util.is_string(search_dirs):
+            search_dirs = [search_dirs]
+
+        search_dirs = os.pathsep.join(search_dirs)
+        for tool in tools:
+            # Extract the tool name from the pattern.  This relies on the tool
+            # name being surrounded by \b word match operators.  If the
+            # pattern starts with "| ", include it in the string to be
+            # substituted.
+            if lit.util.is_string(tool):
+                tool = lit.util.make_word_regex(tool)
+            else:
+                tool = str(tool)
+
+            tool_match = re.match(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_\.]+)\\b\W*$",
+                                  tool)
+            if not tool_match:
+                continue
+
+            tool_pipe = tool_match.group(2)
+            tool_name = tool_match.group(4)
+            tool_path = lit.util.which(tool_name, search_dirs)
+            if not tool_path:
+                if warn_missing:
+                    # Warn, but still provide a substitution.
+                    self.lit_config.note('Did not find ' + tool_name + ' in %s' % search_dirs)
+                tool_path = self.config.llvm_tools_dir + '/' + tool_name
+
+            if tool_name == 'llc' and os.environ.get('LLVM_ENABLE_MACHINE_VERIFIER') == '1':
+                tool_path += ' -verify-machineinstrs'
+            if tool_name == 'llvm-go':
+                exe = getattr(self.config, 'go_executable', None)
+                if exe:
+                    tool_path += " go=" + exe
+
+            self.config.substitutions.append((tool, tool_pipe + tool_path))
diff --git a/llvm/utils/lit/lit/util.py b/llvm/utils/lit/lit/util.py
index 174194f81be..e072a9ef81b 100644
--- a/llvm/utils/lit/lit/util.py
+++ b/llvm/utils/lit/lit/util.py
@@ -36,6 +36,8 @@ def pythonize_bool(value):
             return False
     raise ValueError('"{}" is not a valid boolean'.format(value))
 
+def make_word_regex(word):
+    return r'\b' + word + r'\b'
 
 def to_bytes(s):
     """Return the parameter as type 'bytes', possibly encoding it.