lit: Incremental test scheduling

Add option -i to prioritize test runs by source file modification time and
previous failure state.

This optimal scheduling reduces typical test-and-fix iteration times to a
matter of seconds by rapidly answering the questions:

  1) Did my recent change fix tests that were previously failing?
  2) Do the tests I just wrote / modified still work?

The current implementation requires write permissions to the source tree
because it uses mtimes to track failures.

llvm-svn: 198150

3 files changed, 36 insertions, 2 deletions

diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index b4988f530db..e51bf129776 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -128,10 +128,11 @@ class TestSuite:
 class Test:
     """Test - Information on a single test instance."""
 
-    def __init__(self, suite, path_in_suite, config):
+    def __init__(self, suite, path_in_suite, config, file_path = None):
         self.suite = suite
         self.path_in_suite = path_in_suite
         self.config = config
+        self.file_path = file_path
         # A list of conditions under which this test is expected to fail. These
         # can optionally be provided by test format handlers, and will be
         # honored when the test result is supplied.
@@ -157,6 +158,11 @@ class Test:
     def getFullName(self):
         return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite)
 
+    def getFilePath(self):
+        if self.file_path:
+            return self.file_path
+        return self.getSourcePath()
+
     def getSourcePath(self):
         return self.suite.getSourcePath(self.path_in_suite)
 
diff --git a/llvm/utils/lit/lit/formats/googletest.py b/llvm/utils/lit/lit/formats/googletest.py
index b77e184d2f6..3d14b729ed0 100644
--- a/llvm/utils/lit/lit/formats/googletest.py
+++ b/llvm/utils/lit/lit/formats/googletest.py
@@ -66,7 +66,7 @@ class GoogleTest(TestFormat):
         # Discover the tests in this executable.
         for testname in self.getGTestTests(execpath, litConfig, localConfig):
             testPath = path_in_suite + (basename, testname)
-            yield lit.Test.Test(testSuite, testPath, localConfig)
+            yield lit.Test.Test(testSuite, testPath, localConfig, file_path=execpath)
 
     def getTestsInDirectory(self, testSuite, path_in_suite,
                             litConfig, localConfig):
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 6f672a01eb3..3c49e854067 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -34,6 +34,10 @@ class TestingProgressDisplay(object):
 
     def update(self, test):
         self.completed += 1
+
+        if self.opts.incremental:
+            update_incremental_cache(test)
+
         if self.progressBar:
             self.progressBar.update(float(self.completed)/self.numTests,
                                     test.getFullName())
@@ -108,6 +112,24 @@ def write_test_results(run, lit_config, testing_time, output_path):
     finally:
         f.close()
 
+def update_incremental_cache(test):
+    if not test.result.code.isFailure:
+        return
+    fname = test.getFilePath()
+    os.utime(fname, None)
+
+def sort_by_incremental_cache(run, litConfig):
+    def sortIndex(test):
+        index = 0
+        fname = test.getFilePath()
+        try:
+            index = -os.path.getmtime(fname)
+        except os.error as e:
+            if litConfig.debug:
+                litConfig.note(e)
+        return index
+    run.tests.sort(key = lambda t: sortIndex(t))
+
 def main(builtinParameters = {}):
     # Use processes by default on Unix platforms.
     isWindows = platform.system() == 'Windows'
@@ -179,6 +201,10 @@ def main(builtinParameters = {}):
     group.add_option("", "--shuffle", dest="shuffle",
                      help="Run tests in random order",
                      action="store_true", default=False)
+    group.add_option("-i", "--incremental", dest="incremental",
+                     help="Run modified and failing tests first (updates "
+                     "mtimes)",
+                     action="store_true", default=False)
     group.add_option("", "--filter", dest="filter", metavar="REGEX",
                      help=("Only run tests with paths matching the given "
                            "regular expression"),
@@ -292,6 +318,8 @@ def main(builtinParameters = {}):
     # Then select the order.
     if opts.shuffle:
         random.shuffle(run.tests)
+    elif opts.incremental:
+        sort_by_incremental_cache(run, litConfig)
     else:
         run.tests.sort(key = lambda t: t.getFullName())