[lldb][tests] Make it possible to expect failure for a whole category

There already are decorators and "--excluded" option to mark test-cases/files
as expected to fail. However, when a new test file is added and it which relates
to a feature that a target doesn't support, this requires either adding decorators
to that file or modifying the file provided as "--excluded" option value.

The purpose of this patch is to avoid any modifications in such cases.
E.g. if a target doesn't support "watchpoints" and passes "--xfail-category watchpoint"
to dotest, a testing job will not fail after a new watchpoint-related test file is added.

Differential Revision: https://reviews.llvm.org/D71906

4 files changed, 24 insertions, 4 deletions

diff --git a/lldb/packages/Python/lldbsuite/test/configuration.py b/lldb/packages/Python/lldbsuite/test/configuration.py
index 21d527a9047..0fc831d1173 100644
--- a/lldb/packages/Python/lldbsuite/test/configuration.py
+++ b/lldb/packages/Python/lldbsuite/test/configuration.py
@@ -30,6 +30,8 @@ categories_list = None
 use_categories = False
 # Categories we want to skip
 skip_categories = ["darwin-log"]
+# Categories we expect to fail
+xfail_categories = []
 # use this to track per-category failures
 failures_per_category = {}
 
diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py
index e3f20482827..46600559286 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest.py
@@ -330,6 +330,10 @@ def parseOptionsAndInitTestdirs():
         configuration.skip_categories += test_categories.validate(
             args.skip_categories, False)
 
+    if args.xfail_categories:
+        configuration.xfail_categories += test_categories.validate(
+            args.xfail_categories, False)
+
     if args.E:
         os.environ['CFLAGS_EXTRAS'] = args.E
 
diff --git a/lldb/packages/Python/lldbsuite/test/dotest_args.py b/lldb/packages/Python/lldbsuite/test/dotest_args.py
index 9bcebc58cd2..41c96e3979c 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest_args.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest_args.py
@@ -83,6 +83,12 @@ def create_parser():
         action='append',
         dest='skip_categories',
         help=textwrap.dedent('''Specify categories of test cases to skip. Takes precedence over -G. Can be specified more than once.'''))
+    group.add_argument(
+        '--xfail-category',
+        metavar='category',
+        action='append',
+        dest='xfail_categories',
+        help=textwrap.dedent('''Specify categories of test cases that are expected to fail. Can be specified more than once.'''))
 
     # Configuration options
     group = parser.add_argument_group('Configuration options')
diff --git a/lldb/packages/Python/lldbsuite/test/test_result.py b/lldb/packages/Python/lldbsuite/test/test_result.py
index 193a64c5168..7e13e09d9bf 100644
--- a/lldb/packages/Python/lldbsuite/test/test_result.py
+++ b/lldb/packages/Python/lldbsuite/test/test_result.py
@@ -164,6 +164,10 @@ class LLDBTestResult(unittest2.TextTestResult):
                     return True
         return False
 
+    def checkCategoryExclusion(self, exclusion_list, test):
+        return not set(exclusion_list).isdisjoint(
+            self.getCategoriesForTest(test))
+
     def startTest(self, test):
         if configuration.shouldSkipBecauseOfCategories(
                 self.getCategoriesForTest(test)):
@@ -182,8 +186,10 @@ class LLDBTestResult(unittest2.TextTestResult):
                 EventBuilder.event_for_start(test))
 
     def addSuccess(self, test):
-        if self.checkExclusion(
-                configuration.xfail_tests, test.id()):
+        if (self.checkExclusion(
+                configuration.xfail_tests, test.id()) or
+            self.checkCategoryExclusion(
+                configuration.xfail_categories, test)):
             self.addUnexpectedSuccess(test, None)
             return
 
@@ -245,8 +251,10 @@ class LLDBTestResult(unittest2.TextTestResult):
                     test, err))
 
     def addFailure(self, test, err):
-        if self.checkExclusion(
-                configuration.xfail_tests, test.id()):
+        if (self.checkExclusion(
+                configuration.xfail_tests, test.id()) or
+            self.checkCategoryExclusion(
+                configuration.xfail_categories, test)):
             self.addExpectedFailure(test, err, None)
             return