[lit] Report line number for failed RUN command

When debugging test failures with -vv (or -v in the case of the
internal shell), this makes it easier to locate the RUN line that
failed.  For example, clang's test/Driver/linux-ld.c has 892 total RUN
lines, and clang's test/Driver/arm-cortex-cpus.c has 424 RUN lines
after concatenation for line continuations.

When reading the generated shell script, this also makes it easier to
locate the RUN line that produced each command.

To support reporting RUN line numbers in the case of the internal
shell, this patch extends the internal shell to support the null
command, ":", except pipelines are not supported.

Reviewed By: asmith, delcypher

Differential Revision: https://reviews.llvm.org/D44598

llvm-svn: 330755

15 files changed, 153 insertions, 16 deletions

diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index a6f9276f94e..07cda7fa228 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -789,6 +789,13 @@ def _executeShCmd(cmd, shenv, results, timeoutHelper):
         results.append(cmdResult)
         return cmdResult.exitCode
 
+    if cmd.commands[0].args[0] == ':':
+        if len(cmd.commands) != 1:
+            raise InternalShellError(cmd.commands[0], "Unsupported: ':' "
+                                     "cannot be part of a pipeline")
+        results.append(ShellCommandResult(cmd.commands[0], '', '', 0, False))
+        return 0;
+
     procs = []
     default_stdin = subprocess.PIPE
     stderrTempFiles = []
@@ -1318,7 +1325,8 @@ class IntegratedTestKeywordParser(object):
     def parseLine(self, line_number, line):
         try:
             self.parsed_lines += [(line_number, line)]
-            self.value = self.parser(line_number, line, self.value)
+            self.value = self.parser(line_number, line, self.value,
+                                     self.keyword)
         except ValueError as e:
             raise ValueError(str(e) + ("\nin %s directive on test line %d" %
                                        (self.keyword, line_number)))
@@ -1327,12 +1335,12 @@ class IntegratedTestKeywordParser(object):
         return self.value
 
     @staticmethod
-    def _handleTag(line_number, line, output):
+    def _handleTag(line_number, line, output, keyword):
         """A helper for parsing TAG type keywords"""
         return (not line.strip() or output)
 
     @staticmethod
-    def _handleCommand(line_number, line, output):
+    def _handleCommand(line_number, line, output, keyword):
         """A helper for parsing COMMAND type keywords"""
         # Trim trailing whitespace.
         line = line.rstrip()
@@ -1351,11 +1359,15 @@ class IntegratedTestKeywordParser(object):
         else:
             if output is None:
                 output = []
+            line = ": '{keyword} at line {line}'; {real_command}".format(
+                keyword=keyword,
+                line=line_number,
+                real_command=line)
             output.append(line)
         return output
 
     @staticmethod
-    def _handleList(line_number, line, output):
+    def _handleList(line_number, line, output, keyword):
         """A parser for LIST type keywords"""
         if output is None:
             output = []
@@ -1363,7 +1375,7 @@ class IntegratedTestKeywordParser(object):
         return output
 
     @staticmethod
-    def _handleBooleanExpr(line_number, line, output):
+    def _handleBooleanExpr(line_number, line, output, keyword):
         """A parser for BOOLEAN_EXPR type keywords"""
         if output is None:
             output = []
@@ -1376,17 +1388,18 @@ class IntegratedTestKeywordParser(object):
         return output
 
     @staticmethod
-    def _handleRequiresAny(line_number, line, output):
+    def _handleRequiresAny(line_number, line, output, keyword):
         """A custom parser to transform REQUIRES-ANY: into REQUIRES:"""
 
         # Extract the conditions specified in REQUIRES-ANY: as written.
         conditions = []
-        IntegratedTestKeywordParser._handleList(line_number, line, conditions)
+        IntegratedTestKeywordParser._handleList(line_number, line, conditions,
+                                                keyword)
 
         # Output a `REQUIRES: a || b || c` expression in its place.
         expression = ' || '.join(conditions)
-        IntegratedTestKeywordParser._handleBooleanExpr(line_number,
-                                                       expression, output)
+        IntegratedTestKeywordParser._handleBooleanExpr(line_number, expression,
+                                                       output, keyword)
         return output
 
 def parseIntegratedTestScript(test, additional_parsers=[],
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/basic.txt b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/basic.txt
new file mode 100644
index 00000000000..ba2695431bd
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/basic.txt
@@ -0,0 +1,3 @@
+# RUN: true
+# RUN: false
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/line-continuation.txt b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/line-continuation.txt
new file mode 100644
index 00000000000..1e00bcb9616
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/line-continuation.txt
@@ -0,0 +1,11 @@
+# RUN: : first line continued \
+# RUN:   to second line
+# RUN: echo 'foo bar' \
+# RUN: | FileCheck %s
+# RUN: echo \
+# RUN: 'foo baz' \
+# RUN: | FileCheck %s
+# RUN: echo 'foo bar' \
+# RUN: | FileCheck %s
+
+# CHECK: foo bar
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/lit.local.cfg b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/lit.local.cfg
new file mode 100644
index 00000000000..5e87c729919
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/lit.local.cfg
@@ -0,0 +1,2 @@
+import lit.formats
+config.test_format = lit.formats.ShTest(execute_external=True)
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/basic.txt b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/basic.txt
new file mode 100644
index 00000000000..ba2695431bd
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/basic.txt
@@ -0,0 +1,3 @@
+# RUN: true
+# RUN: false
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/line-continuation.txt b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/line-continuation.txt
new file mode 100644
index 00000000000..1e00bcb9616
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/line-continuation.txt
@@ -0,0 +1,11 @@
+# RUN: : first line continued \
+# RUN:   to second line
+# RUN: echo 'foo bar' \
+# RUN: | FileCheck %s
+# RUN: echo \
+# RUN: 'foo baz' \
+# RUN: | FileCheck %s
+# RUN: echo 'foo bar' \
+# RUN: | FileCheck %s
+
+# CHECK: foo bar
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/lit.local.cfg b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/lit.local.cfg
new file mode 100644
index 00000000000..b76b7a24c99
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/lit.local.cfg
@@ -0,0 +1,2 @@
+import lit.formats
+config.test_format = lit.formats.ShTest(execute_external=False)
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/lit.cfg
new file mode 100644
index 00000000000..b14c52a1842
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/lit.cfg
@@ -0,0 +1,3 @@
+import lit.formats
+config.name = 'shtest-run-at-line'
+config.suffixes = ['.txt']
diff --git a/llvm/utils/lit/tests/Inputs/shtest-shell/colon-error.txt b/llvm/utils/lit/tests/Inputs/shtest-shell/colon-error.txt
new file mode 100644
index 00000000000..8b84c08ce09
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-shell/colon-error.txt
@@ -0,0 +1,3 @@
+# Check error on an unsupported ":". (cannot be part of a pipeline)
+#
+# RUN: : | echo "hello"
diff --git a/llvm/utils/lit/tests/max-failures.py b/llvm/utils/lit/tests/max-failures.py
index 1613eee8b5c..ca107bc29b9 100644
--- a/llvm/utils/lit/tests/max-failures.py
+++ b/llvm/utils/lit/tests/max-failures.py
@@ -8,7 +8,7 @@
 #
 # END.
 
-# CHECK: Failing Tests (26)
+# CHECK: Failing Tests (27)
 # CHECK: Failing Tests (1)
 # CHECK: Failing Tests (2)
 # CHECK: error: Setting --max-failures to 0 does not have any effect.
diff --git a/llvm/utils/lit/tests/shtest-format.py b/llvm/utils/lit/tests/shtest-format.py
index 94d74e3a920..33ed2fe44d6 100644
--- a/llvm/utils/lit/tests/shtest-format.py
+++ b/llvm/utils/lit/tests/shtest-format.py
@@ -39,6 +39,7 @@
 #
 # CHECK: Command Output (stdout):
 # CHECK-NEXT: --
+# CHECK-NEXT: $ ":" "RUN: at line 1"
 # CHECK-NEXT: $ "printf"
 # CHECK-NEXT: # command output:
 # CHECK-NEXT: line 1: failed test output on stdout
diff --git a/llvm/utils/lit/tests/shtest-output-printing.py b/llvm/utils/lit/tests/shtest-output-printing.py
index 2a85cf975c9..2344ef20036 100644
--- a/llvm/utils/lit/tests/shtest-output-printing.py
+++ b/llvm/utils/lit/tests/shtest-output-printing.py
@@ -16,12 +16,15 @@
 #
 # CHECK:      Command Output
 # CHECK-NEXT: --
+# CHECK-NEXT: $ ":" "RUN: at line 1"
 # CHECK-NEXT: $ "true"
+# CHECK-NEXT: $ ":" "RUN: at line 2"
 # CHECK-NEXT: $ "echo" "hi"
 # CHECK-NEXT: # command output:
 # CHECK-NEXT: hi
 #
-# CHECK:      $ "wc" "missing-file"
+# CHECK:      $ ":" "RUN: at line 3"
+# CHECK-NEXT: $ "wc" "missing-file"
 # CHECK-NEXT: # redirected output from '{{.*(/|\\\\)}}basic.txt.tmp.out':
 # CHECK-NEXT: missing-file{{.*}} No such file or directory
 # CHECK:      note: command had no output on stdout or stderr
diff --git a/llvm/utils/lit/tests/shtest-run-at-line.py b/llvm/utils/lit/tests/shtest-run-at-line.py
new file mode 100644
index 00000000000..03a516eed85
--- /dev/null
+++ b/llvm/utils/lit/tests/shtest-run-at-line.py
@@ -0,0 +1,74 @@
+# Check that -vv makes the line number of the failing RUN command clear.
+# (-v is actually sufficient in the case of the internal shell.)
+#
+# RUN: not %{lit} -j 1 -vv %{inputs}/shtest-run-at-line > %t.out
+# RUN: FileCheck --input-file %t.out %s
+#
+# END.
+
+
+# CHECK: Testing: 4 tests
+
+
+# In the case of the external shell, we check for only RUN lines in stderr in
+# case some shell implementations format "set -x" output differently.
+
+# CHECK-LABEL: FAIL: shtest-run-at-line :: external-shell/basic.txt
+
+# CHECK:      Script:
+# CHECK:      : 'RUN: at line 1';  true
+# CHECK-NEXT: : 'RUN: at line 2';  false
+# CHECK-NEXT: : 'RUN: at line 3';  true
+
+# CHECK:     Command Output (stderr)
+# CHECK:     RUN: at line 1
+# CHECK:     RUN: at line 2
+# CHECK-NOT: RUN
+
+# CHECK-LABEL: FAIL: shtest-run-at-line :: external-shell/line-continuation.txt
+
+# CHECK:      Script:
+# CHECK:      : 'RUN: at line 1';  : first line continued    to second line
+# CHECK-NEXT: : 'RUN: at line 3';  echo 'foo bar'  | FileCheck
+# CHECK-NEXT: : 'RUN: at line 5';  echo  'foo baz'  | FileCheck
+# CHECK-NEXT: : 'RUN: at line 8';  echo 'foo bar'  | FileCheck
+
+# CHECK:     Command Output (stderr)
+# CHECK:     RUN: at line 1
+# CHECK:     RUN: at line 3
+# CHECK:     RUN: at line 5
+# CHECK-NOT: RUN
+
+
+# CHECK-LABEL: FAIL: shtest-run-at-line :: internal-shell/basic.txt
+
+# CHECK:      Script:
+# CHECK:      : 'RUN: at line 1';  true
+# CHECK-NEXT: : 'RUN: at line 2';  false
+# CHECK-NEXT: : 'RUN: at line 3';  true
+
+# CHECK:      Command Output (stdout)
+# CHECK:      $ ":" "RUN: at line 1"
+# CHECK-NEXT: $ "true"
+# CHECK-NEXT: $ ":" "RUN: at line 2"
+# CHECK-NEXT: $ "false"
+# CHECK-NOT:  RUN
+
+# CHECK-LABEL: FAIL: shtest-run-at-line :: internal-shell/line-continuation.txt
+
+# CHECK:      Script:
+# CHECK:      : 'RUN: at line 1';  : first line continued to second line
+# CHECK-NEXT: : 'RUN: at line 3';  echo 'foo bar'  | FileCheck
+# CHECK-NEXT: : 'RUN: at line 5';  echo  'foo baz'  | FileCheck
+# CHECK-NEXT: : 'RUN: at line 8';  echo 'foo bar'  | FileCheck
+
+# CHECK:      Command Output (stdout)
+# CHECK:      $ ":" "RUN: at line 1"
+# CHECK-NEXT: $ ":" "first" "line" "continued" "to" "second" "line"
+# CHECK-NEXT: $ ":" "RUN: at line 3"
+# CHECK-NEXT: $ "echo" "foo bar"
+# CHECK-NEXT: $ "FileCheck" "{{.*}}"
+# CHECK-NEXT: $ ":" "RUN: at line 5"
+# CHECK-NEXT: $ "echo" "foo baz"
+# CHECK-NEXT: $ "FileCheck" "{{.*}}"
+# CHECK-NOT:  RUN
diff --git a/llvm/utils/lit/tests/shtest-shell.py b/llvm/utils/lit/tests/shtest-shell.py
index b3d55a0c6a9..ed0bdf35fe1 100644
--- a/llvm/utils/lit/tests/shtest-shell.py
+++ b/llvm/utils/lit/tests/shtest-shell.py
@@ -26,6 +26,14 @@
 # CHECK: error: command failed with exit status: 1
 # CHECK: ***
 
+# CHECK: FAIL: shtest-shell :: colon-error.txt
+# CHECK: *** TEST 'shtest-shell :: colon-error.txt' FAILED ***
+# CHECK: $ ":"
+# CHECK: # command stderr:
+# CHECK: Unsupported: ':' cannot be part of a pipeline
+# CHECK: error: command failed with exit status: 127
+# CHECK: ***
+
 # CHECK: FAIL: shtest-shell :: diff-error-0.txt
 # CHECK: *** TEST 'shtest-shell :: diff-error-0.txt' FAILED ***
 # CHECK: $ "diff" "diff-error-0.txt" "diff-error-0.txt"
@@ -153,7 +161,7 @@
 #
 # CHECK: FAIL: shtest-shell :: error-1.txt
 # CHECK: *** TEST 'shtest-shell :: error-1.txt' FAILED ***
-# CHECK: shell parser error on: 'echo "missing quote'
+# CHECK: shell parser error on: ': \'RUN: at line 3\'; echo "missing quote'
 # CHECK: ***
 
 # CHECK: FAIL: shtest-shell :: error-2.txt
@@ -219,4 +227,4 @@
 # CHECK: PASS: shtest-shell :: sequencing-0.txt
 # CHECK: XFAIL: shtest-shell :: sequencing-1.txt
 # CHECK: PASS: shtest-shell :: valid-shell.txt
-# CHECK: Failing Tests (26)
+# CHECK: Failing Tests (27)
diff --git a/llvm/utils/lit/tests/unit/TestRunner.py b/llvm/utils/lit/tests/unit/TestRunner.py
index 874bf275d4e..b3988b0c197 100644
--- a/llvm/utils/lit/tests/unit/TestRunner.py
+++ b/llvm/utils/lit/tests/unit/TestRunner.py
@@ -47,7 +47,7 @@ class TestIntegratedTestKeywordParser(unittest.TestCase):
 
     @staticmethod
     def make_parsers():
-        def custom_parse(line_number, line, output):
+        def custom_parse(line_number, line, output, keyword):
             if output is None:
                 output = []
             output += [part for part in line.split(' ') if part.strip()]
@@ -99,8 +99,8 @@ class TestIntegratedTestKeywordParser(unittest.TestCase):
         cmd_parser = self.get_parser(parsers, 'MY_RUN:')
         value = cmd_parser.getValue()
         self.assertEqual(len(value), 2)  # there are only two run lines
-        self.assertEqual(value[0].strip(), 'baz')
-        self.assertEqual(value[1].strip(), 'foo  bar')
+        self.assertEqual(value[0].strip(), ": 'MY_RUN: at line 4';  baz")
+        self.assertEqual(value[1].strip(), ": 'MY_RUN: at line 7';  foo  bar")
 
     def test_custom(self):
         parsers = self.make_parsers()