[lit] Fix `not` calling internal commands

Without this patch, when using lit's internal shell, if `not` on a lit
RUN line calls `env`, `diff`, or any of the other in-process shell
builtins that lit implements, lit accidentally searches for the latter
as an external executable.  What's worse is that works fine when a
developer is testing on a platform where those executables are
available and behave as expected, but it then breaks on other
platforms.

`not` seems useful for some builtins, such as `diff`, so this patch
supports such uses.  `not --crash` does not seem useful for builtins,
so this patch diagnoses such uses.  In all cases, this patch ensures
shell builtins are found behind any sequence of `env` and `not`
commands.

`not` calling `env` calling an external command appears useful when
the `env` and external command are part of a lit substitution, as in
D65156.  This patch supports that by looking through any sequence of
`env` and `not` commands, building the environment from the `env`s,
and storing the `not`s.  The `not`s are then added back to the command
line without the `env`s to execute externally.  This avoids the need
to replicate the `not` implementation, in particular the `--crash`
option, in lit.

Reviewed By: rnk

Differential Revision: https://reviews.llvm.org/D66531

20 files changed, 262 insertions, 2 deletions

diff --git a/llvm/utils/lit/tests/Inputs/shtest-env/env-calls-not-builtin.txt b/llvm/utils/lit/tests/Inputs/shtest-env/env-calls-not-builtin.txt
new file mode 100644
index 00000000000..b8aa57120b8
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-env/env-calls-not-builtin.txt
@@ -0,0 +1,4 @@
+# Other tests thoroughly check that 'env' cannot call various builtin commands.
+# Pick one and make sure it fails even if there's a 'not' in the way.
+
+# RUN: env -u FOO BAR=3 not rm %t.no-such-file
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/fail.py b/llvm/utils/lit/tests/Inputs/shtest-not/fail.py
new file mode 100644
index 00000000000..d64b8cb0f89
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/fail.py
@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+
+import print_environment
+import sys
+
+print_environment.execute()
+sys.exit(1)
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-not/lit.cfg
new file mode 100644
index 00000000000..8609fd2c513
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/lit.cfg
@@ -0,0 +1,7 @@
+import lit.formats
+config.name = 'shtest-not'
+config.suffixes = ['.txt']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+config.substitutions.append(('%{python}', '"%s"' % (sys.executable)))
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/not-args-last-is-crash.txt b/llvm/utils/lit/tests/Inputs/shtest-not/not-args-last-is-crash.txt
new file mode 100644
index 00000000000..acb4e91b74c
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/not-args-last-is-crash.txt
@@ -0,0 +1 @@
+# RUN: not --crash
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/not-args-nested-none.txt b/llvm/utils/lit/tests/Inputs/shtest-not/not-args-nested-none.txt
new file mode 100644
index 00000000000..20fbc5f23c5
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/not-args-nested-none.txt
@@ -0,0 +1 @@
+# RUN: not not not
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/not-args-none.txt b/llvm/utils/lit/tests/Inputs/shtest-not/not-args-none.txt
new file mode 100644
index 00000000000..101d2aecc5d
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/not-args-none.txt
@@ -0,0 +1 @@
+# RUN: not
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-cd.txt b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-cd.txt
new file mode 100644
index 00000000000..331fb51ec5d
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-cd.txt
@@ -0,0 +1,3 @@
+# Internal cd always succeeds.
+# RUN: not not cd foobar
+# RUN: not --crash cd foobar
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-colon.txt b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-colon.txt
new file mode 100644
index 00000000000..83bd4083d3e
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-colon.txt
@@ -0,0 +1,3 @@
+# ":" always succeeds.
+# RUN: not not : foobar
+# RUN: not --crash :
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-diff-with-crash.txt b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-diff-with-crash.txt
new file mode 100644
index 00000000000..df721275cb0
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-diff-with-crash.txt
@@ -0,0 +1,6 @@
+# Lit's diff is out-of-process, so check that "not --crash diff" fails because
+# diff doesn't crash rather than because "not --crash diff" isn't supported.
+
+# RUN: echo 'foo' > %t.foo
+# RUN: echo 'bar' > %t.bar
+# RUN: not --crash diff -u %t.foo %t.bar
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-diff.txt b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-diff.txt
new file mode 100644
index 00000000000..d16ce75acbe
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-diff.txt
@@ -0,0 +1,16 @@
+# RUN: echo 'foo' > %t.foo0
+# RUN: echo 'foo' > %t.foo1
+# RUN: echo 'bar' > %t.bar
+
+# diff fails.
+# RUN: not diff %t.foo0 %t.bar
+# RUN: not not not diff %t.foo0 %t.bar
+# RUN: not not not not not diff %t.foo0 %t.bar
+
+# diff succeeds.
+# RUN: diff %t.foo0 %t.foo1
+# RUN: not not diff %t.foo0 %t.foo1
+# RUN: not not not not diff %t.foo0 %t.foo1
+
+# diff succeeds but we expect failure.
+# RUN: not diff %t.foo0 %t.foo1
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-echo.txt b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-echo.txt
new file mode 100644
index 00000000000..b96da29d483
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-echo.txt
@@ -0,0 +1,3 @@
+# Internal "echo" always succeeds.
+# RUN: not not echo hello world
+# RUN: not --crash echo hello world
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-env-builtin.txt b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-env-builtin.txt
new file mode 100644
index 00000000000..68aa2974bd5
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-env-builtin.txt
@@ -0,0 +1,4 @@
+# Other tests thoroughly check that 'not' cannot call various builtin commands.
+# Pick one and make sure it fails even if there's an 'env' in the way.
+
+# RUN: not --crash env -u FOO BAR=3 rm %t.no-such-file
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-export.txt b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-export.txt
new file mode 100644
index 00000000000..13d0fe06615
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-export.txt
@@ -0,0 +1,3 @@
+# Internal "export" always succeeds.
+# RUN: not not export FOO=1
+# RUN: not --crash export BAZ=3
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-external.txt b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-external.txt
new file mode 100644
index 00000000000..9209da83d7d
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-external.txt
@@ -0,0 +1,66 @@
+# RUN: echo foo > %t.in
+# RUN: echo bar >> %t.in
+
+
+# Simple uses of 'not'
+
+# RUN: not %{python} fail.py
+# RUN: not not %{python} pass.py
+# RUN: not not not %{python} fail.py
+# RUN: not not not not %{python} pass.py
+
+
+# Simple uses of 'not --crash'
+
+# RUN: not not --crash %{python} pass.py
+# RUN: not not --crash %{python} fail.py
+# RUN: not not --crash not %{python} pass.py
+# RUN: not not --crash not  %{python} fail.py
+
+
+# Various patterns of 'not' and 'env'
+#
+# There's no particular pattern to the 'env' arguments except we try not to
+# do the same thing every time.
+
+# RUN: env not %{python} fail.py | FileCheck -check-prefixes=FOO-NO,BAR-NO %s
+# RUN: not env %{python} fail.py | FileCheck -check-prefixes=FOO-NO,BAR-NO %s
+
+# RUN: env FOO=1 not %{python} fail.py \
+# RUN: | FileCheck -check-prefixes=FOO1,BAR-NO %s
+
+# RUN: not env FOO=1 BAR=1 %{python} fail.py \
+# RUN: | FileCheck -check-prefixes=FOO1,BAR1 %s
+
+# RUN: env FOO=1 BAR=1 not env -u FOO BAR=2 %{python} fail.py \
+# RUN: | FileCheck -check-prefixes=FOO-NO,BAR2 %s
+
+# RUN: not env FOO=1 BAR=1 not env -u FOO -u BAR %{python} pass.py \
+# RUN: | FileCheck -check-prefixes=FOO-NO,BAR-NO %s
+
+# RUN: not not env FOO=1 env FOO=2 BAR=1 %{python} pass.py \
+# RUN: | FileCheck -check-prefixes=FOO2,BAR1 %s
+
+# RUN: env FOO=1 -u BAR env -u FOO BAR=1 not not %{python} pass.py \
+# RUN: | FileCheck -check-prefixes=FOO-NO,BAR1 %s
+
+
+# Various patterns of 'not', 'not --crash', and 'env'
+
+# RUN: not env FOO=1 BAR=1 env FOO=2 BAR=2 not --crash %{python} pass.py \
+# RUN: | FileCheck -check-prefixes=FOO2,BAR2 %s
+
+# RUN: not env FOO=1 BAR=1 not --crash not %{python} pass.py \
+# RUN: | FileCheck -check-prefixes=FOO1,BAR1 %s
+
+# RUN: not not --crash env -u BAR not env -u FOO BAR=1 %{python} pass.py \
+# RUN: | FileCheck -check-prefixes=FOO-NO,BAR1 %s
+
+
+# FOO-NO: FOO = [undefined]
+# FOO1:   FOO = 1
+# FOO2:   FOO = 2
+
+# BAR-NO: BAR = [undefined]
+# BAR1:   BAR = 1
+# BAR2:   BAR = 2
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-mkdir.txt b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-mkdir.txt
new file mode 100644
index 00000000000..262d0a71c2d
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-mkdir.txt
@@ -0,0 +1,2 @@
+# RUN: not mkdir %t/foobar
+# RUN: not --crash mkdir foobar
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-rm.txt b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-rm.txt
new file mode 100644
index 00000000000..20660ddd3c3
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/not-calls-rm.txt
@@ -0,0 +1,2 @@
+# RUN: not rm %t
+# RUN: not --crash rm foobar
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/pass.py b/llvm/utils/lit/tests/Inputs/shtest-not/pass.py
new file mode 100644
index 00000000000..bd113596b7d
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/pass.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+
+import print_environment
+
+print_environment.execute()
diff --git a/llvm/utils/lit/tests/Inputs/shtest-not/print_environment.py b/llvm/utils/lit/tests/Inputs/shtest-not/print_environment.py
new file mode 100644
index 00000000000..7397889d5e7
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-not/print_environment.py
@@ -0,0 +1,6 @@
+from __future__ import print_function
+import os
+
+def execute():
+    for name in ['FOO', 'BAR']:
+        print(name, '=', os.environ.get(name, '[undefined]'))
diff --git a/llvm/utils/lit/tests/shtest-env.py b/llvm/utils/lit/tests/shtest-env.py
index 9eee63abe0e..28a7639c3b0 100644
--- a/llvm/utils/lit/tests/shtest-env.py
+++ b/llvm/utils/lit/tests/shtest-env.py
@@ -7,7 +7,7 @@
 
 # Make sure env commands are included in printed commands.
 
-# CHECK: -- Testing: 15 tests{{.*}}
+# CHECK: -- Testing: 16 tests{{.*}}
 
 # CHECK: FAIL: shtest-env :: env-args-last-is-assign.txt ({{[^)]*}})
 # CHECK: $ "env" "FOO=1"
@@ -67,6 +67,11 @@
 # CHECK: Error: 'env' cannot call 'mkdir'
 # CHECK: error: command failed with exit status: {{.*}}
 
+# CHECK: FAIL: shtest-env :: env-calls-not-builtin.txt ({{[^)]*}})
+# CHECK: $ "env" "-u" "FOO" "BAR=3" "not" "rm" "{{.*}}.no-such-file"
+# CHECK: Error: 'env' cannot call 'rm'
+# CHECK: error: command failed with exit status: {{.*}}
+
 # CHECK: FAIL: shtest-env :: env-calls-rm.txt ({{[^)]*}})
 # CHECK: $ "env" "-u" "FOO" "BAR=3" "rm" "foobar"
 # CHECK: Error: 'env' cannot call 'rm'
@@ -89,5 +94,5 @@
 # CHECK-NOT: ${{.*}}print_environment.py
 
 # CHECK: Expected Passes : 4
-# CHECK: Unexpected Failures: 11
+# CHECK: Unexpected Failures: 12
 # CHECK-NOT: {{.}}
diff --git a/llvm/utils/lit/tests/shtest-not.py b/llvm/utils/lit/tests/shtest-not.py
new file mode 100644
index 00000000000..bd204685ebc
--- /dev/null
+++ b/llvm/utils/lit/tests/shtest-not.py
@@ -0,0 +1,115 @@
+# Check the not command
+#
+# RUN: not %{lit} -j 1 -a -v %{inputs}/shtest-not \
+# RUN: | FileCheck -match-full-lines %s
+#
+# END.
+
+# Make sure not and env commands are included in printed commands.
+
+# CHECK: -- Testing: 13 tests{{.*}}
+
+# CHECK: FAIL: shtest-not :: not-args-last-is-crash.txt {{.*}}
+# CHECK: $ "not" "--crash"
+# CHECK: Error: 'not' requires a subcommand
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-args-nested-none.txt {{.*}}
+# CHECK: $ "not" "not" "not"
+# CHECK: Error: 'not' requires a subcommand
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-args-none.txt {{.*}}
+# CHECK: $ "not"
+# CHECK: Error: 'not' requires a subcommand
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-calls-cd.txt {{.*}}
+# CHECK: $ "not" "not" "cd" "foobar"
+# CHECK: $ "not" "--crash" "cd" "foobar"
+# CHECK: Error: 'not --crash' cannot call 'cd'
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-calls-colon.txt {{.*}}
+# CHECK: $ "not" "not" ":" "foobar"
+# CHECK: $ "not" "--crash" ":"
+# CHECK: Error: 'not --crash' cannot call ':'
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-calls-diff-with-crash.txt {{.*}}
+# CHECK: $ "not" "--crash" "diff" "-u" {{.*}}
+# CHECK-NOT: "$"
+# CHECK-NOT: {{[Ee]rror}}
+# CHECK: error: command failed with exit status: {{.*}}
+# CHECK-NOT: {{[Ee]rror}}
+# CHECK-NOT: "$"
+
+# CHECK: FAIL: shtest-not :: not-calls-diff.txt {{.*}}
+# CHECK: $ "not" "diff" {{.*}}
+# CHECK: $ "not" "not" "not" "diff" {{.*}}
+# CHECK: $ "not" "not" "not" "not" "not" "diff" {{.*}}
+# CHECK: $ "diff" {{.*}}
+# CHECK: $ "not" "not" "diff" {{.*}}
+# CHECK: $ "not" "not" "not" "not" "diff" {{.*}}
+# CHECK: $ "not" "diff" {{.*}}
+# CHECK-NOT: "$"
+
+# CHECK: FAIL: shtest-not :: not-calls-echo.txt {{.*}}
+# CHECK: $ "not" "not" "echo" "hello" "world"
+# CHECK: $ "not" "--crash" "echo" "hello" "world"
+# CHECK: Error: 'not --crash' cannot call 'echo'
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-calls-env-builtin.txt {{.*}}
+# CHECK: $ "not" "--crash" "env" "-u" "FOO" "BAR=3" "rm" "{{.*}}.no-such-file"
+# CHECK: Error: 'env' cannot call 'rm'
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-calls-export.txt {{.*}}
+# CHECK: $ "not" "not" "export" "FOO=1"
+# CHECK: $ "not" "--crash" "export" "BAZ=3"
+# CHECK: Error: 'not --crash' cannot call 'export'
+# CHECK: error: command failed with exit status: {{.*}}
+
+
+# CHECK: PASS: shtest-not :: not-calls-external.txt {{.*}}
+
+# CHECK: $ "not" "{{[^"]*}}" "fail.py"
+# CHECK: $ "not" "not" "{{[^"]*}}" "pass.py"
+# CHECK: $ "not" "not" "not" "{{[^"]*}}" "fail.py"
+# CHECK: $ "not" "not" "not" "not" "{{[^"]*}}" "pass.py"
+
+# CHECK: $ "not" "not" "--crash" "{{[^"]*}}" "pass.py"
+# CHECK: $ "not" "not" "--crash" "{{[^"]*}}" "fail.py"
+# CHECK: $ "not" "not" "--crash" "not" "{{[^"]*}}" "pass.py"
+# CHECK: $ "not" "not" "--crash" "not" "{{[^"]*}}" "fail.py"
+
+# CHECK: $ "env" "not" "{{[^"]*}}" "fail.py"
+# CHECK: $ "not" "env" "{{[^"]*}}" "fail.py"
+# CHECK: $ "env" "FOO=1" "not" "{{[^"]*}}" "fail.py"
+# CHECK: $ "not" "env" "FOO=1" "BAR=1" "{{[^"]*}}" "fail.py"
+# CHECK: $ "env" "FOO=1" "BAR=1" "not" "env" "-u" "FOO" "BAR=2" "{{[^"]*}}" "fail.py"
+# CHECK: $ "not" "env" "FOO=1" "BAR=1" "not" "env" "-u" "FOO" "-u" "BAR" "{{[^"]*}}" "pass.py"
+# CHECK: $ "not" "not" "env" "FOO=1" "env" "FOO=2" "BAR=1" "{{[^"]*}}" "pass.py"
+# CHECK: $ "env" "FOO=1" "-u" "BAR" "env" "-u" "FOO" "BAR=1" "not" "not" "{{[^"]*}}" "pass.py"
+
+# CHECK: $ "not" "env" "FOO=1" "BAR=1" "env" "FOO=2" "BAR=2" "not" "--crash" "{{[^"]*}}" "pass.py"
+# CHECK: $ "not" "env" "FOO=1" "BAR=1" "not" "--crash" "not" "{{[^"]*}}" "pass.py"
+# CHECK: $ "not" "not" "--crash" "env" "-u" "BAR" "not" "env" "-u" "FOO" "BAR=1" "{{[^"]*}}" "pass.py"
+
+
+# CHECK: FAIL: shtest-not :: not-calls-mkdir.txt {{.*}}
+# CHECK: $ "not" "mkdir" {{.*}}
+# CHECK: $ "not" "--crash" "mkdir" "foobar"
+# CHECK: Error: 'not --crash' cannot call 'mkdir'
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: FAIL: shtest-not :: not-calls-rm.txt {{.*}}
+# CHECK: $ "not" "rm" {{.*}}
+# CHECK: $ "not" "--crash" "rm" "foobar"
+# CHECK: Error: 'not --crash' cannot call 'rm'
+# CHECK: error: command failed with exit status: {{.*}}
+
+# CHECK: Expected Passes : 1
+# CHECK: Unexpected Failures: 12
+# CHECK-NOT: {{.}}