Re-land "[test] Split LLDB tests into API, Shell & Unit"

The original patch got reverted because it broke `check-lldb` on a clean
build. This fixes that.

llvm-svn: 374201

8 files changed, 415 insertions, 0 deletions

diff --git a/lldb/test/API/CMakeLists.txt b/lldb/test/API/CMakeLists.txt
new file mode 100644
index 00000000000..4f787d21fd3
--- /dev/null
+++ b/lldb/test/API/CMakeLists.txt
@@ -0,0 +1,133 @@
+function(add_python_test_target name test_script args comment)
+  set(PYTHON_TEST_COMMAND
+    ${PYTHON_EXECUTABLE}
+    ${test_script}
+    ${args}
+    )
+
+  add_custom_target(${name}
+    COMMAND ${PYTHON_TEST_COMMAND} ${ARG_DEFAULT_ARGS}
+    COMMENT "${comment}"
+    USES_TERMINAL
+    )
+  add_dependencies(${name} lldb-test-deps)
+endfunction()
+
+# The default architecture with which to compile test executables is the default LLVM target
+# architecture, which itself defaults to the host architecture.
+string(TOLOWER "${LLVM_TARGET_ARCH}" LLDB_DEFAULT_TEST_ARCH)
+if( LLDB_DEFAULT_TEST_ARCH STREQUAL "host" )
+  string(REGEX MATCH "^[^-]*" LLDB_DEFAULT_TEST_ARCH ${LLVM_HOST_TRIPLE})
+endif ()
+
+# Allow the user to override the default by setting LLDB_TEST_ARCH
+set(LLDB_TEST_ARCH
+  ${LLDB_DEFAULT_TEST_ARCH}
+  CACHE STRING "Specify the architecture to run LLDB tests as (x86|x64).  Determines whether tests are compiled with -m32 or -m64")
+
+# Users can override LLDB_TEST_USER_ARGS to specify arbitrary arguments to pass to the script
+set(LLDB_TEST_USER_ARGS
+  ""
+  CACHE STRING "Specify additional arguments to pass to test runner. For example: '-C gcc -C clang -A i386 -A x86_64'")
+
+# The .noindex suffix is a marker for Spotlight to never index the
+# build directory.  LLDB queries Spotlight to locate .dSYM bundles
+# based on the UUID embedded in a binary, and because the UUID is a
+# hash of filename and .text section, there *will* be conflicts inside
+# the build directory.
+set(LLDB_TEST_COMMON_ARGS
+  --arch=${LLDB_TEST_ARCH}
+  -s
+  ${CMAKE_BINARY_DIR}/lldb-test-traces
+  -S nm
+  -u CXXFLAGS
+  -u CFLAGS
+  )
+
+list(APPEND LLDB_TEST_COMMON_ARGS
+  --executable ${LLDB_TEST_EXECUTABLE}
+  --dsymutil ${LLDB_TEST_DSYMUTIL}
+  --filecheck ${LLDB_TEST_FILECHECK}
+  -C ${LLDB_TEST_C_COMPILER}
+  )
+
+if ( CMAKE_SYSTEM_NAME MATCHES "Windows" )
+  # All tests are currently flaky on Windows, so rerun them all once when they fail.
+  set(LLDB_TEST_COMMON_ARGS ${LLDB_TEST_COMMON_ARGS} --rerun-all-issues)
+
+  set(LLDB_TEST_DEBUG_TEST_CRASHES
+    0
+    CACHE BOOL "(Windows only) Enables debugging of tests in the test suite by showing the crash dialog when lldb crashes")
+
+  set(LLDB_TEST_HIDE_CONSOLE_WINDOWS
+    1
+    CACHE BOOL "(Windows only) Hides the console window for an inferior when it is launched through the test suite")
+
+  if (LLDB_TEST_DEBUG_TEST_CRASHES)
+    set(LLDB_TEST_COMMON_ARGS ${LLDB_TEST_COMMON_ARGS} --enable-crash-dialog)
+  endif()
+
+  if (NOT LLDB_TEST_HIDE_CONSOLE_WINDOWS)
+    set(LLDB_TEST_COMMON_ARGS ${LLDB_TEST_COMMON_ARGS} --show-inferior-console)
+  endif()
+endif()
+
+if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows|Darwin")
+  list(APPEND LLDB_TEST_COMMON_ARGS
+    --env ARCHIVER=${CMAKE_AR} --env OBJCOPY=${CMAKE_OBJCOPY})
+endif()
+
+if (NOT "${LLDB_LIT_TOOLS_DIR}" STREQUAL "")
+  if (NOT EXISTS "${LLDB_LIT_TOOLS_DIR}")
+    message(WARNING "LLDB_LIT_TOOLS_DIR ${LLDB_LIT_TOOLS_DIR} does not exist.")
+  endif()
+endif()
+
+if(CMAKE_HOST_APPLE)
+  if(LLDB_BUILD_FRAMEWORK)
+    get_target_property(framework_build_dir liblldb LIBRARY_OUTPUT_DIRECTORY)
+    list(APPEND LLDB_TEST_COMMON_ARGS --framework ${framework_build_dir}/LLDB.framework)
+  endif()
+
+  # Use the same identity for testing
+  get_property(code_sign_identity_used GLOBAL PROPERTY LLDB_DEBUGSERVER_CODESIGN_IDENTITY)
+  if(code_sign_identity_used)
+    list(APPEND LLDB_TEST_COMMON_ARGS --codesign-identity "${code_sign_identity_used}")
+  endif()
+
+  if(LLDB_USE_SYSTEM_DEBUGSERVER)
+    lldb_find_system_debugserver(system_debugserver_path)
+    add_custom_target(debugserver
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      ${system_debugserver_path} ${LLVM_RUNTIME_OUTPUT_INTDIR}
+      COMMENT "Copying the system debugserver to LLDB's binaries directory for testing.")
+    # The custom target for the system debugserver has no install target, so we
+    # need to remove it from the LLVM_DISTRIBUTION_COMPONENTS list.
+    if (LLVM_DISTRIBUTION_COMPONENTS)
+      list(REMOVE_ITEM LLVM_DISTRIBUTION_COMPONENTS debugserver)
+      set(LLVM_DISTRIBUTION_COMPONENTS ${LLVM_DISTRIBUTION_COMPONENTS} PARENT_SCOPE)
+    endif()
+    message(STATUS "LLDB tests use out-of-tree debugserver: ${system_debugserver_path}")
+    list(APPEND LLDB_TEST_COMMON_ARGS --out-of-tree-debugserver)
+    add_lldb_test_dependency(debugserver)
+  elseif(TARGET debugserver)
+    set(debugserver_path ${LLVM_RUNTIME_OUTPUT_INTDIR}/debugserver)
+    message(STATUS "LLDB Tests use just-built debugserver: ${debugserver_path}")
+    list(APPEND LLDB_TEST_COMMON_ARGS --server ${debugserver_path})
+    add_lldb_test_dependency(debugserver)
+  elseif(TARGET lldb-server)
+    set(lldb_server_path ${LLVM_RUNTIME_OUTPUT_INTDIR}/lldb-server)
+    message(STATUS "LLDB Tests use just-built lldb-server: ${lldb_server_path}")
+    list(APPEND LLDB_TEST_COMMON_ARGS --server ${lldb_server_path})
+    add_lldb_test_dependency(lldb-server)
+  else()
+    message(WARNING "LLDB Tests enabled, but no server available")
+  endif()
+endif()
+
+set(LLDB_DOTEST_ARGS ${LLDB_TEST_COMMON_ARGS};${LLDB_TEST_USER_ARGS})
+set_property(GLOBAL PROPERTY LLDB_DOTEST_ARGS_PROPERTY ${LLDB_DOTEST_ARGS})
+
+# This will add LLDB's test dependencies to the dependencies for check-all and
+# include them in the test-depends target.
+set_property(GLOBAL APPEND PROPERTY LLVM_LIT_DEPENDS ${ARG_DEPENDS})
diff --git a/lldb/test/API/README.md b/lldb/test/API/README.md
new file mode 100644
index 00000000000..fbb37136e26
--- /dev/null
+++ b/lldb/test/API/README.md
@@ -0,0 +1,4 @@
+# LLDB API Tests
+
+This directory only exists for the lit test driver. The actual tests live in
+the `tests` directory in top level LLDB directory.
diff --git a/lldb/test/API/dotest.py b/lldb/test/API/dotest.py
new file mode 100755
index 00000000000..1f9d52354f5
--- /dev/null
+++ b/lldb/test/API/dotest.py
@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+
+if __name__ == "__main__":
+    import use_lldb_suite
+
+    import lldbsuite.test
+    lldbsuite.test.run_suite()
diff --git a/lldb/test/API/lit.cfg b/lldb/test/API/lit.cfg
new file mode 100644
index 00000000000..bb9e3aaaaa4
--- /dev/null
+++ b/lldb/test/API/lit.cfg
@@ -0,0 +1,80 @@
+# -*- Python -*-
+
+# Configuration file for the 'lit' test runner.
+
+import os
+import platform
+import shlex
+
+import lit.formats
+
+# name: The name of this test suite.
+config.name = 'lldb-api'
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = ['.py']
+
+# test_source_root: The root path where tests are located.
+# test_exec_root: The root path where tests should be run.
+config.test_source_root = os.path.join(config.lldb_src_root, 'packages',
+                                       'Python', 'lldbsuite', 'test')
+config.test_exec_root = config.test_source_root
+
+if 'Address' in config.llvm_use_sanitizer:
+  config.environment['ASAN_OPTIONS'] = 'detect_stack_use_after_return=1'
+  # macOS flags needed for LLDB built with address sanitizer.
+  if 'Darwin' in config.host_os and 'x86' in config.host_triple:
+    import subprocess
+    resource_dir = subprocess.check_output(
+        [config.cmake_cxx_compiler, '-print-resource-dir']).strip()
+    runtime = os.path.join(resource_dir, 'lib', 'darwin',
+                           'libclang_rt.asan_osx_dynamic.dylib')
+    config.environment['DYLD_INSERT_LIBRARIES'] = runtime
+
+def find_shlibpath_var():
+  if platform.system() in ['Linux', 'FreeBSD', 'NetBSD', 'SunOS']:
+    yield 'LD_LIBRARY_PATH'
+  elif platform.system() == 'Darwin':
+    yield 'DYLD_LIBRARY_PATH'
+  elif platform.system() == 'Windows':
+    yield 'PATH'
+
+# Shared library build of LLVM may require LD_LIBRARY_PATH or equivalent.
+if config.shared_libs:
+  for shlibpath_var in find_shlibpath_var():
+    # In stand-alone build llvm_shlib_dir specifies LLDB's lib directory while
+    # llvm_libs_dir specifies LLVM's lib directory.
+    shlibpath = os.path.pathsep.join(
+        (config.llvm_shlib_dir, config.llvm_libs_dir,
+         config.environment.get(shlibpath_var, '')))
+    config.environment[shlibpath_var] = shlibpath
+  else:
+    lit_config.warning("unable to inject shared library path on '{}'".format(
+        platform.system()))
+
+# Build dotest command.
+dotest_cmd = [config.dotest_path]
+dotest_cmd.extend(config.dotest_args_str.split(';'))
+
+# We don't want to force users passing arguments to lit to use `;` as a
+# separator. We use Python's simple lexical analyzer to turn the args into a
+# list.
+if config.dotest_lit_args_str:
+  dotest_cmd.extend(shlex.split(config.dotest_lit_args_str))
+
+# Library path may be needed to locate just-built clang.
+if config.llvm_libs_dir:
+  dotest_cmd += ['--env', 'LLVM_LIBS_DIR=' + config.llvm_libs_dir]
+
+if config.lldb_build_directory:
+  dotest_cmd += ['--build-dir', config.lldb_build_directory]
+
+if config.lldb_module_cache:
+  dotest_cmd += ['--module-cache-dir', config.lldb_module_cache]
+
+# Load LLDB test format.
+sys.path.append(os.path.join(config.lldb_src_root, "test", "API"))
+import lldbtest
+
+# testFormat: The test format to use to interpret tests.
+config.test_format = lldbtest.LLDBTest(dotest_cmd)
diff --git a/lldb/test/API/lit.site.cfg.in b/lldb/test/API/lit.site.cfg.in
new file mode 100644
index 00000000000..883bc403300
--- /dev/null
+++ b/lldb/test/API/lit.site.cfg.in
@@ -0,0 +1,46 @@
+@LIT_SITE_CFG_IN_HEADER@
+
+config.test_exec_root = "@LLDB_BINARY_DIR@"
+config.llvm_src_root = "@LLVM_SOURCE_DIR@"
+config.llvm_obj_root = "@LLVM_BINARY_DIR@"
+config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
+config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
+config.llvm_shlib_dir = "@SHLIBDIR@"
+config.llvm_build_mode = "@LLVM_BUILD_MODE@"
+config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
+config.lldb_obj_root = "@LLDB_BINARY_DIR@"
+config.lldb_src_root = "@LLDB_SOURCE_DIR@"
+config.cmake_cxx_compiler = "@CMAKE_CXX_COMPILER@"
+config.host_os = "@HOST_OS@"
+config.host_triple = "@LLVM_HOST_TRIPLE@"
+config.shared_libs = @LLVM_ENABLE_SHARED_LIBS@
+config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
+config.target_triple = "@TARGET_TRIPLE@"
+config.lldb_build_directory = "@LLDB_TEST_BUILD_DIRECTORY@"
+config.lldb_module_cache = "@LLDB_TEST_MODULE_CACHE_LLDB@"
+config.clang_module_cache = "@LLDB_TEST_MODULE_CACHE_CLANG@"
+config.python_executable = "@PYTHON_EXECUTABLE@"
+config.dotest_path = "@LLDB_SOURCE_DIR@/test/API/dotest.py"
+config.dotest_args_str = "@LLDB_DOTEST_ARGS@"
+config.lldb_disable_python = @LLDB_DISABLE_PYTHON@
+config.dotest_lit_args_str = None
+
+# Additional dotest arguments can be passed to lit by providing a
+# semicolon-separates list: --param dotest-args="arg;arg".
+dotest_lit_args_str = lit_config.params.get('dotest-args', None)
+if dotest_lit_args_str:
+    config.dotest_lit_args_str = dotest_lit_args_str
+
+# Support substitution of the tools and libs dirs with user parameters. This is
+# used when we can't determine the tool dir at configuration time.
+try:
+    config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
+    config.llvm_libs_dir = config.llvm_libs_dir % lit_config.params
+    config.dotest_args_str = config.dotest_args_str % lit_config.params
+    config.llvm_build_mode = config.llvm_build_mode % lit_config.params
+except KeyError as e:
+    key, = e.args
+    lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
+
+# Let the main config do the real work.
+lit_config.load_config(config, "@LLDB_SOURCE_DIR@/test/API/lit.cfg")
diff --git a/lldb/test/API/lldbtest.py b/lldb/test/API/lldbtest.py
new file mode 100644
index 00000000000..99eb1457259
--- /dev/null
+++ b/lldb/test/API/lldbtest.py
@@ -0,0 +1,116 @@
+from __future__ import absolute_import
+import os
+
+import subprocess
+import sys
+
+import lit.Test
+import lit.TestRunner
+import lit.util
+from lit.formats.base import TestFormat
+
+def getBuildDir(cmd):
+    found = False
+    for arg in cmd:
+        if found:
+            return arg
+        if arg == '--build-dir':
+            found = True
+    return None
+
+def mkdir_p(path):
+    import errno
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+    if not os.path.isdir(path):
+        raise OSError(errno.ENOTDIR, "%s is not a directory"%path)
+
+class LLDBTest(TestFormat):
+    def __init__(self, dotest_cmd):
+        self.dotest_cmd = dotest_cmd
+
+    def getTestsInDirectory(self, testSuite, path_in_suite, litConfig,
+                            localConfig):
+        source_path = testSuite.getSourcePath(path_in_suite)
+        for filename in os.listdir(source_path):
+            # Ignore dot files and excluded tests.
+            if (filename.startswith('.') or filename in localConfig.excludes):
+                continue
+
+            # Ignore files that don't start with 'Test'.
+            if not filename.startswith('Test'):
+                continue
+
+            filepath = os.path.join(source_path, filename)
+            if not os.path.isdir(filepath):
+                base, ext = os.path.splitext(filename)
+                if ext in localConfig.suffixes:
+                    yield lit.Test.Test(testSuite, path_in_suite +
+                                        (filename, ), localConfig)
+
+    def execute(self, test, litConfig):
+        if litConfig.noExecute:
+            return lit.Test.PASS, ''
+
+        if test.config.lldb_disable_python:
+            return (lit.Test.UNSUPPORTED, 'Python module disabled')
+
+        if test.config.unsupported:
+            return (lit.Test.UNSUPPORTED, 'Test is unsupported')
+
+        testPath, testFile = os.path.split(test.getSourcePath())
+        # On Windows, the system does not always correctly interpret
+        # shebang lines.  To make sure we can execute the tests, add
+        # python exe as the first parameter of the command.
+        cmd = [sys.executable] + self.dotest_cmd + [testPath, '-p', testFile]
+
+        # The macOS system integrity protection (SIP) doesn't allow injecting
+        # libraries into system binaries, but this can be worked around by
+        # copying the binary into a different location.
+        if 'DYLD_INSERT_LIBRARIES' in test.config.environment and \
+                (sys.executable.startswith('/System/') or \
+                sys.executable.startswith('/usr/')):
+            builddir = getBuildDir(cmd)
+            mkdir_p(builddir)
+            copied_python = os.path.join(builddir, 'copied-system-python')
+            if not os.path.isfile(copied_python):
+                import shutil, subprocess
+                python = subprocess.check_output([
+                    '/usr/bin/python2.7', '-c',
+                    'import sys; print sys.executable']).strip()
+                shutil.copy(python, copied_python)
+            cmd[0] = copied_python
+
+        try:
+            out, err, exitCode = lit.util.executeCommand(
+                cmd,
+                env=test.config.environment,
+                timeout=litConfig.maxIndividualTestTime)
+        except lit.util.ExecuteCommandTimeoutException:
+            return (lit.Test.TIMEOUT, 'Reached timeout of {} seconds'.format(
+                litConfig.maxIndividualTestTime))
+
+        if exitCode:
+            # Match FAIL but not XFAIL.
+            for line in out.splitlines() + err.splitlines():
+                if line.startswith('FAIL:'):
+                    return lit.Test.FAIL, out + err
+
+            if 'XPASS:' in out or 'XPASS:' in err:
+                return lit.Test.XPASS, out + err
+
+        has_unsupported_tests = 'UNSUPPORTED:' in out or 'UNSUPPORTED:' in err
+        has_passing_tests = 'PASS:' in out or 'PASS:' in err
+        if has_unsupported_tests and not has_passing_tests:
+            return lit.Test.UNSUPPORTED, out + err
+
+        passing_test_line = 'RESULT: PASSED'
+        if passing_test_line not in out and passing_test_line not in err:
+            msg = ('Unable to find %r in dotest output (exit code %d):\n\n%s%s'
+                   % (passing_test_line, exitCode, out, err))
+            return lit.Test.UNRESOLVED, msg
+
+        return lit.Test.PASS, ''
diff --git a/lldb/test/API/testcases b/lldb/test/API/testcases
new file mode 120000
index 00000000000..f4a13dceb79
--- /dev/null
+++ b/lldb/test/API/testcases
@@ -0,0 +1 @@
+../packages/Python/lldbsuite/test
\ No newline at end of file
diff --git a/lldb/test/API/use_lldb_suite.py b/lldb/test/API/use_lldb_suite.py
new file mode 100644
index 00000000000..6a8c12d8189
--- /dev/null
+++ b/lldb/test/API/use_lldb_suite.py
@@ -0,0 +1,28 @@
+import inspect
+import os
+import sys
+
+
+def find_lldb_root():
+    lldb_root = os.path.dirname(
+        os.path.abspath(inspect.getfile(inspect.currentframe()))
+    )
+    while True:
+        lldb_root = os.path.dirname(lldb_root)
+        if lldb_root is None:
+            return None
+
+        test_path = os.path.join(lldb_root, "use_lldb_suite_root.py")
+        if os.path.isfile(test_path):
+            return lldb_root
+    return None
+
+lldb_root = find_lldb_root()
+if lldb_root is not None:
+    import imp
+    fp, pathname, desc = imp.find_module("use_lldb_suite_root", [lldb_root])
+    try:
+        imp.load_module("use_lldb_suite_root", fp, pathname, desc)
+    finally:
+        if fp:
+            fp.close()