[lit] Report line number for failed RUN command

(Relands r333584, reverted in 333592.)

When debugging test failures with -vv (or -v in the case of the
internal shell), this makes it easier to locate the RUN line that
failed.  For example, clang's test/Driver/linux-ld.c has 892 total RUN
lines, and clang's test/Driver/arm-cortex-cpus.c has 424 RUN lines
after concatenation for line continuations.

When reading the generated shell script, this also makes it easier to
locate the RUN line that produced each command.

To support reporting RUN line numbers in the case of the internal
shell, this patch extends the internal shell to support the null
command, ":", except pipelines are not supported.

To support reporting RUN line numbers in the case of windows cmd.exe
as the external shell, this patch extends -vv to set "echo on" instead
of "echo off" in bat files.  (Support for windows cmd.exe as a lit
external shell will likely be dropped later, but I found out too
late.)

Reviewed By: delcypher,	asmith, stella.stamenova, jmorse, lebedev.ri, rnk

Differential Revision: https://reviews.llvm.org/D44598

llvm-svn: 333614

15 files changed, 140 insertions, 6 deletions

diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/basic.txt b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/basic.txt
new file mode 100644
index 00000000000..a359c996660
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/basic.txt
@@ -0,0 +1,6 @@
+# These commands must run under both bash and windows cmd.exe (with GnuWin32
+# tools).
+
+# RUN: true
+# RUN: false
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/line-continuation.txt b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/line-continuation.txt
new file mode 100644
index 00000000000..b11f0d759fe
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/line-continuation.txt
@@ -0,0 +1,12 @@
+# These commands must run under both bash and windows cmd.exe (with GnuWin32
+# tools).
+
+# RUN: echo 'foo bar' \
+# RUN: | FileCheck %s
+# RUN: echo \
+# RUN: 'foo baz' \
+# RUN: | FileCheck %s
+# RUN: echo 'foo bar' \
+# RUN: | FileCheck %s
+
+# CHECK: foo bar
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/lit.local.cfg b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/lit.local.cfg
new file mode 100644
index 00000000000..5e87c729919
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/lit.local.cfg
@@ -0,0 +1,2 @@
+import lit.formats
+config.test_format = lit.formats.ShTest(execute_external=True)
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/basic.txt b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/basic.txt
new file mode 100644
index 00000000000..ba2695431bd
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/basic.txt
@@ -0,0 +1,3 @@
+# RUN: true
+# RUN: false
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/line-continuation.txt b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/line-continuation.txt
new file mode 100644
index 00000000000..1e00bcb9616
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/line-continuation.txt
@@ -0,0 +1,11 @@
+# RUN: : first line continued \
+# RUN:   to second line
+# RUN: echo 'foo bar' \
+# RUN: | FileCheck %s
+# RUN: echo \
+# RUN: 'foo baz' \
+# RUN: | FileCheck %s
+# RUN: echo 'foo bar' \
+# RUN: | FileCheck %s
+
+# CHECK: foo bar
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/lit.local.cfg b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/lit.local.cfg
new file mode 100644
index 00000000000..b76b7a24c99
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/lit.local.cfg
@@ -0,0 +1,2 @@
+import lit.formats
+config.test_format = lit.formats.ShTest(execute_external=False)
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/lit.cfg
new file mode 100644
index 00000000000..f4c7921b732
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/lit.cfg
@@ -0,0 +1,2 @@
+config.name = 'shtest-run-at-line'
+config.suffixes = ['.txt']
diff --git a/llvm/utils/lit/tests/Inputs/shtest-shell/colon-error.txt b/llvm/utils/lit/tests/Inputs/shtest-shell/colon-error.txt
new file mode 100644
index 00000000000..8b84c08ce09
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-shell/colon-error.txt
@@ -0,0 +1,3 @@
+# Check error on an unsupported ":". (cannot be part of a pipeline)
+#
+# RUN: : | echo "hello"
diff --git a/llvm/utils/lit/tests/lit.cfg b/llvm/utils/lit/tests/lit.cfg
index 75d1b5eac85..a5ccc2c067e 100644
--- a/llvm/utils/lit/tests/lit.cfg
+++ b/llvm/utils/lit/tests/lit.cfg
@@ -71,3 +71,14 @@ for attribute in ('llvm_tools_dir', 'lit_tools_dir'):
     if directory:
         path = os.path.pathsep.join((directory, path))
 config.environment['PATH'] = path
+
+# These substitutions are needed only in tests where the external shell is used
+# and could be either bash or windows cmd.exe.  Substitutions are expected to
+# be expanded in double quotes.
+isWin32CMDEXE = lit_config.isWindows and not lit_config.getBashPath()
+if isWin32CMDEXE:
+    config.substitutions.append(('%{pdbg0}', "echo '"))
+    config.substitutions.append(('%{pdbg1}', "' > nul"))
+else:
+    config.substitutions.append(('%{pdbg0}', ": '"))
+    config.substitutions.append(('%{pdbg1}', "'"))
diff --git a/llvm/utils/lit/tests/max-failures.py b/llvm/utils/lit/tests/max-failures.py
index 1613eee8b5c..ca107bc29b9 100644
--- a/llvm/utils/lit/tests/max-failures.py
+++ b/llvm/utils/lit/tests/max-failures.py
@@ -8,7 +8,7 @@
 #
 # END.
 
-# CHECK: Failing Tests (26)
+# CHECK: Failing Tests (27)
 # CHECK: Failing Tests (1)
 # CHECK: Failing Tests (2)
 # CHECK: error: Setting --max-failures to 0 does not have any effect.
diff --git a/llvm/utils/lit/tests/shtest-format.py b/llvm/utils/lit/tests/shtest-format.py
index 94d74e3a920..33ed2fe44d6 100644
--- a/llvm/utils/lit/tests/shtest-format.py
+++ b/llvm/utils/lit/tests/shtest-format.py
@@ -39,6 +39,7 @@
 #
 # CHECK: Command Output (stdout):
 # CHECK-NEXT: --
+# CHECK-NEXT: $ ":" "RUN: at line 1"
 # CHECK-NEXT: $ "printf"
 # CHECK-NEXT: # command output:
 # CHECK-NEXT: line 1: failed test output on stdout
diff --git a/llvm/utils/lit/tests/shtest-output-printing.py b/llvm/utils/lit/tests/shtest-output-printing.py
index 2a85cf975c9..2344ef20036 100644
--- a/llvm/utils/lit/tests/shtest-output-printing.py
+++ b/llvm/utils/lit/tests/shtest-output-printing.py
@@ -16,12 +16,15 @@
 #
 # CHECK:      Command Output
 # CHECK-NEXT: --
+# CHECK-NEXT: $ ":" "RUN: at line 1"
 # CHECK-NEXT: $ "true"
+# CHECK-NEXT: $ ":" "RUN: at line 2"
 # CHECK-NEXT: $ "echo" "hi"
 # CHECK-NEXT: # command output:
 # CHECK-NEXT: hi
 #
-# CHECK:      $ "wc" "missing-file"
+# CHECK:      $ ":" "RUN: at line 3"
+# CHECK-NEXT: $ "wc" "missing-file"
 # CHECK-NEXT: # redirected output from '{{.*(/|\\\\)}}basic.txt.tmp.out':
 # CHECK-NEXT: missing-file{{.*}} No such file or directory
 # CHECK:      note: command had no output on stdout or stderr
diff --git a/llvm/utils/lit/tests/shtest-run-at-line.py b/llvm/utils/lit/tests/shtest-run-at-line.py
new file mode 100644
index 00000000000..adb7af81812
--- /dev/null
+++ b/llvm/utils/lit/tests/shtest-run-at-line.py
@@ -0,0 +1,70 @@
+# Check that -vv makes the line number of the failing RUN command clear.
+# (-v is actually sufficient in the case of the internal shell.)
+#
+# RUN: not %{lit} -j 1 -vv %{inputs}/shtest-run-at-line > %t.out
+# RUN: FileCheck --input-file %t.out -Dpdbg0="%{pdbg0}" -Dpdbg1="%{pdbg1}" %s
+#
+# END.
+
+
+# CHECK: Testing: 4 tests
+
+
+# In the case of the external shell, we check for only RUN lines in stderr in
+# case some shell implementations format "set -x" output differently.
+
+# CHECK-LABEL: FAIL: shtest-run-at-line :: external-shell/basic.txt
+
+# CHECK:      Script:
+# CHECK:      [[pdbg0]]RUN: at line 4[[pdbg1]] &&  true
+# CHECK-NEXT: [[pdbg0]]RUN: at line 5[[pdbg1]] &&  false
+# CHECK-NEXT: [[pdbg0]]RUN: at line 6[[pdbg1]] &&  true
+
+# CHECK:     RUN: at line 4
+# CHECK:     RUN: at line 5
+# CHECK-NOT: RUN
+
+# CHECK-LABEL: FAIL: shtest-run-at-line :: external-shell/line-continuation.txt
+
+# CHECK:      Script:
+# CHECK:      [[pdbg0]]RUN: at line 4[[pdbg1]] &&  echo 'foo bar'  | FileCheck
+# CHECK-NEXT: [[pdbg0]]RUN: at line 6[[pdbg1]] &&  echo  'foo baz'  | FileCheck
+# CHECK-NEXT: [[pdbg0]]RUN: at line 9[[pdbg1]] &&  echo 'foo bar'  | FileCheck
+
+# CHECK:     RUN: at line 4
+# CHECK:     RUN: at line 6
+# CHECK-NOT: RUN
+
+
+# CHECK-LABEL: FAIL: shtest-run-at-line :: internal-shell/basic.txt
+
+# CHECK:      Script:
+# CHECK:      : 'RUN: at line 1' &&  true
+# CHECK-NEXT: : 'RUN: at line 2' &&  false
+# CHECK-NEXT: : 'RUN: at line 3' &&  true
+
+# CHECK:      Command Output (stdout)
+# CHECK:      $ ":" "RUN: at line 1"
+# CHECK-NEXT: $ "true"
+# CHECK-NEXT: $ ":" "RUN: at line 2"
+# CHECK-NEXT: $ "false"
+# CHECK-NOT:  RUN
+
+# CHECK-LABEL: FAIL: shtest-run-at-line :: internal-shell/line-continuation.txt
+
+# CHECK:      Script:
+# CHECK:      : 'RUN: at line 1' &&  : first line continued to second line
+# CHECK-NEXT: : 'RUN: at line 3' &&  echo 'foo bar'  | FileCheck
+# CHECK-NEXT: : 'RUN: at line 5' &&  echo  'foo baz'  | FileCheck
+# CHECK-NEXT: : 'RUN: at line 8' &&  echo 'foo bar'  | FileCheck
+
+# CHECK:      Command Output (stdout)
+# CHECK:      $ ":" "RUN: at line 1"
+# CHECK-NEXT: $ ":" "first" "line" "continued" "to" "second" "line"
+# CHECK-NEXT: $ ":" "RUN: at line 3"
+# CHECK-NEXT: $ "echo" "foo bar"
+# CHECK-NEXT: $ "FileCheck" "{{.*}}"
+# CHECK-NEXT: $ ":" "RUN: at line 5"
+# CHECK-NEXT: $ "echo" "foo baz"
+# CHECK-NEXT: $ "FileCheck" "{{.*}}"
+# CHECK-NOT:  RUN
diff --git a/llvm/utils/lit/tests/shtest-shell.py b/llvm/utils/lit/tests/shtest-shell.py
index b3d55a0c6a9..016d1f6f108 100644
--- a/llvm/utils/lit/tests/shtest-shell.py
+++ b/llvm/utils/lit/tests/shtest-shell.py
@@ -26,6 +26,14 @@
 # CHECK: error: command failed with exit status: 1
 # CHECK: ***
 
+# CHECK: FAIL: shtest-shell :: colon-error.txt
+# CHECK: *** TEST 'shtest-shell :: colon-error.txt' FAILED ***
+# CHECK: $ ":"
+# CHECK: # command stderr:
+# CHECK: Unsupported: ':' cannot be part of a pipeline
+# CHECK: error: command failed with exit status: 127
+# CHECK: ***
+
 # CHECK: FAIL: shtest-shell :: diff-error-0.txt
 # CHECK: *** TEST 'shtest-shell :: diff-error-0.txt' FAILED ***
 # CHECK: $ "diff" "diff-error-0.txt" "diff-error-0.txt"
@@ -153,7 +161,7 @@
 #
 # CHECK: FAIL: shtest-shell :: error-1.txt
 # CHECK: *** TEST 'shtest-shell :: error-1.txt' FAILED ***
-# CHECK: shell parser error on: 'echo "missing quote'
+# CHECK: shell parser error on: ': \'RUN: at line 3\' && echo "missing quote'
 # CHECK: ***
 
 # CHECK: FAIL: shtest-shell :: error-2.txt
@@ -219,4 +227,4 @@
 # CHECK: PASS: shtest-shell :: sequencing-0.txt
 # CHECK: XFAIL: shtest-shell :: sequencing-1.txt
 # CHECK: PASS: shtest-shell :: valid-shell.txt
-# CHECK: Failing Tests (26)
+# CHECK: Failing Tests (27)
diff --git a/llvm/utils/lit/tests/unit/TestRunner.py b/llvm/utils/lit/tests/unit/TestRunner.py
index 874bf275d4e..4ade359790c 100644
--- a/llvm/utils/lit/tests/unit/TestRunner.py
+++ b/llvm/utils/lit/tests/unit/TestRunner.py
@@ -99,8 +99,8 @@ class TestIntegratedTestKeywordParser(unittest.TestCase):
         cmd_parser = self.get_parser(parsers, 'MY_RUN:')
         value = cmd_parser.getValue()
         self.assertEqual(len(value), 2)  # there are only two run lines
-        self.assertEqual(value[0].strip(), 'baz')
-        self.assertEqual(value[1].strip(), 'foo  bar')
+        self.assertEqual(value[0].strip(), "%dbg(MY_RUN: at line 4) &&  baz")
+        self.assertEqual(value[1].strip(), "%dbg(MY_RUN: at line 7) &&  foo  bar")
 
     def test_custom(self):
         parsers = self.make_parsers()