[clang-tidy] check_clang_tidy.py: support CHECK-NOTES prefix

Summary:
Currently, there is two configured prefixes: `CHECK-FIXES` and `CHECK-MESSAGES`
`CHECK-MESSAGES` checks that there are no test output lines with `warning:|error:`, which are not explicitly handled in lit tests.
However there does not seem to be a nice way to enforce for all the `note:` to be checked.
This was useful for me when developing D36836.

Reviewers: alexfh, klimek, aaron.ballman, hokein

Reviewed By: alexfh, aaron.ballman

Subscribers: JonasToth, JDevlieghere, xazax.hun, cfe-commits

Tags: #clang-tools-extra

Differential Revision: https://reviews.llvm.org/D36892

llvm-svn: 339437

3 files changed, 48 insertions, 28 deletions

diff --git a/clang-tools-extra/docs/clang-tidy/index.rst b/clang-tools-extra/docs/clang-tidy/index.rst
index ec6c24ff554..0ed350c7bcb 100644
--- a/clang-tools-extra/docs/clang-tidy/index.rst
+++ b/clang-tools-extra/docs/clang-tidy/index.rst
@@ -650,7 +650,8 @@ clang-tidy tests.
 
 An additional check enabled by ``check_clang_tidy.py`` ensures that
 if `CHECK-MESSAGES:` is used in a file then every warning or error
-must have an associated CHECK in that file.
+must have an associated CHECK in that file. Or, you can use ``CHECK-NOTES:``
+instead, if you want to **also** ensure that all the notes are checked.
 
 To use the ``check_clang_tidy.py`` script, put a .cpp file with the
 appropriate ``RUN`` line in the ``test/clang-tidy`` directory. Use
diff --git a/clang-tools-extra/test/clang-tidy/check_clang_tidy.py b/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
index bb2c5a26052..dadb84d5790 100755
--- a/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
+++ b/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
@@ -78,6 +78,7 @@ def main():
   file_check_suffix = ('-' + args.check_suffix) if args.check_suffix else ''
   check_fixes_prefix = 'CHECK-FIXES' + file_check_suffix
   check_messages_prefix = 'CHECK-MESSAGES' + file_check_suffix
+  check_notes_prefix = 'CHECK-NOTES' + file_check_suffix
 
   # Tests should not rely on STL being available, and instead provide mock
   # implementations of relevant APIs.
@@ -91,9 +92,11 @@ def main():
 
   has_check_fixes = check_fixes_prefix in input_text
   has_check_messages = check_messages_prefix in input_text
+  has_check_notes = check_notes_prefix in input_text
 
-  if not has_check_fixes and not has_check_messages:
-    sys.exit('Neither %s nor %s found in the input' % (check_fixes_prefix, check_messages_prefix) )
+  if not has_check_fixes and not has_check_messages and not has_check_notes:
+    sys.exit('%s, %s or %s not found in the input' % (check_fixes_prefix,
+             check_messages_prefix, check_notes_prefix) )
 
   # Remove the contents of the CHECK lines to avoid CHECKs matching on
   # themselves.  We need to keep the comments to preserve line numbers while
@@ -156,5 +159,18 @@ def main():
       print('FileCheck failed:\n' + e.output.decode())
       raise
 
+  if has_check_notes:
+    notes_file = temp_file_name + '.notes'
+    write_file(notes_file, clang_tidy_output)
+    try:
+      subprocess.check_output(
+          ['FileCheck', '-input-file=' + notes_file, input_file_name,
+           '-check-prefix=' + check_notes_prefix,
+           '-implicit-check-not={{note|warning|error}}:'],
+          stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+      print('FileCheck failed:\n' + e.output.decode())
+      raise
+
 if __name__ == '__main__':
   main()
diff --git a/clang-tools-extra/test/clang-tidy/hicpp-exception-baseclass.cpp b/clang-tools-extra/test/clang-tidy/hicpp-exception-baseclass.cpp
index 9a0638b3a45..fdf8093b7d6 100644
--- a/clang-tools-extra/test/clang-tidy/hicpp-exception-baseclass.cpp
+++ b/clang-tools-extra/test/clang-tidy/hicpp-exception-baseclass.cpp
@@ -20,28 +20,28 @@ class really_creative : public non_derived_exception, private std::exception {};
 void problematic() {
   try {
     throw int(42);
-    // CHECK-MESSAGES: [[@LINE-1]]:11: warning: throwing an exception whose type 'int' is not derived from 'std::exception'
+    // CHECK-NOTES: [[@LINE-1]]:11: warning: throwing an exception whose type 'int' is not derived from 'std::exception'
   } catch (int e) {
   }
   throw int(42);
-  // CHECK-MESSAGES: [[@LINE-1]]:9: warning: throwing an exception whose type 'int' is not derived from 'std::exception'
+  // CHECK-NOTES: [[@LINE-1]]:9: warning: throwing an exception whose type 'int' is not derived from 'std::exception'
 
   try {
     throw 12;
-    // CHECK-MESSAGES: [[@LINE-1]]:11: warning: throwing an exception whose type 'int' is not derived from 'std::exception'
+    // CHECK-NOTES: [[@LINE-1]]:11: warning: throwing an exception whose type 'int' is not derived from 'std::exception'
   } catch (...) {
     throw; // Ok, even if the type is not known, conforming code can never rethrow a non-std::exception object.
   }
 
   try {
     throw non_derived_exception();
-    // CHECK-MESSAGES: [[@LINE-1]]:11: warning: throwing an exception whose type 'non_derived_exception' is not derived from 'std::exception'
-    // CHECK-MESSAGES: 9:1: note: type defined here
+    // CHECK-NOTES: [[@LINE-1]]:11: warning: throwing an exception whose type 'non_derived_exception' is not derived from 'std::exception'
+    // CHECK-NOTES: 9:1: note: type defined here
   } catch (non_derived_exception &e) {
   }
   throw non_derived_exception();
-  // CHECK-MESSAGES: [[@LINE-1]]:9: warning: throwing an exception whose type 'non_derived_exception' is not derived from 'std::exception'
-  // CHECK-MESSAGES: 9:1: note: type defined here
+  // CHECK-NOTES: [[@LINE-1]]:9: warning: throwing an exception whose type 'non_derived_exception' is not derived from 'std::exception'
+  // CHECK-NOTES: 9:1: note: type defined here
 
 // FIXME: More complicated kinds of inheritance should be checked later, but there is
 // currently no way use ASTMatchers for this kind of task.
@@ -100,15 +100,15 @@ void allowed_throws() {
 // Templated function that throws exception based on template type
 template <typename T>
 void ThrowException() { throw T(); }
-// CHECK-MESSAGES: [[@LINE-1]]:31: warning: throwing an exception whose type 'bad_generic_exception<int>' is not derived from 'std::exception'
-// CHECK-MESSAGES: 120:1: note: type defined here
-// CHECK-MESSAGES: [[@LINE-3]]:31: warning: throwing an exception whose type 'bad_generic_exception<std::exception>' is not derived from 'std::exception'
-// CHECK-MESSAGES: 120:1: note: type defined here
-// CHECK-MESSAGES: [[@LINE-5]]:31: warning: throwing an exception whose type 'exotic_exception<non_derived_exception>' is not derived from 'std::exception'
-// CHECK-MESSAGES: 123:1: note: type defined here
-// CHECK-MESSAGES: [[@LINE-7]]:31: warning: throwing an exception whose type 'int' is not derived from 'std::exception'
-// CHECK-MESSAGES: [[@LINE-8]]:31: warning: throwing an exception whose type 'non_derived_exception' is not derived from 'std::exception'
-// CHECK-MESSAGES: 9:1: note: type defined here
+// CHECK-NOTES: [[@LINE-1]]:31: warning: throwing an exception whose type 'bad_generic_exception<int>' is not derived from 'std::exception'
+// CHECK-NOTES: 120:1: note: type defined here
+// CHECK-NOTES: [[@LINE-3]]:31: warning: throwing an exception whose type 'bad_generic_exception<std::exception>' is not derived from 'std::exception'
+// CHECK-NOTES: 120:1: note: type defined here
+// CHECK-NOTES: [[@LINE-5]]:31: warning: throwing an exception whose type 'exotic_exception<non_derived_exception>' is not derived from 'std::exception'
+// CHECK-NOTES: 123:1: note: type defined here
+// CHECK-NOTES: [[@LINE-7]]:31: warning: throwing an exception whose type 'int' is not derived from 'std::exception'
+// CHECK-NOTES: [[@LINE-8]]:31: warning: throwing an exception whose type 'non_derived_exception' is not derived from 'std::exception'
+// CHECK-NOTES: 9:1: note: type defined here
 #define THROW_EXCEPTION(CLASS) ThrowException<CLASS>()
 #define THROW_BAD_EXCEPTION throw int(42);
 #define THROW_GOOD_EXCEPTION throw std::exception();
@@ -134,8 +134,8 @@ void generic_exceptions() {
   THROW_EXCEPTION(deep_hierarchy);    // Ok
 
   THROW_BAD_EXCEPTION;
-  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: throwing an exception whose type 'int' is not derived from 'std::exception'
-  // CHECK-MESSAGES: [[@LINE-25]]:35: note: expanded from macro 'THROW_BAD_EXCEPTION'
+  // CHECK-NOTES: [[@LINE-1]]:3: warning: throwing an exception whose type 'int' is not derived from 'std::exception'
+  // CHECK-NOTES: [[@LINE-25]]:35: note: expanded from macro 'THROW_BAD_EXCEPTION'
   THROW_GOOD_EXCEPTION;
   THROW_DERIVED_EXCEPTION;
 
@@ -143,16 +143,19 @@ void generic_exceptions() {
   THROW_EXCEPTION(generic_exception<float>); // Ok
 
   throw bad_generic_exception<int>();
-  // CHECK-MESSAGES: [[@LINE-1]]:9: warning: throwing an exception whose type 'bad_generic_exception<int>' is not derived from 'std::exception'
+  // CHECK-NOTES: [[@LINE-1]]:9: warning: throwing an exception whose type 'bad_generic_exception<int>' is not derived from 'std::exception'
+  // CHECK-NOTES: 120:1: note: type defined here
   throw bad_generic_exception<std::exception>();
-  // CHECK-MESSAGES: [[@LINE-1]]:9: warning: throwing an exception whose type 'bad_generic_exception<std::exception>' is not derived from 'std::exception'
+  // CHECK-NOTES: [[@LINE-1]]:9: warning: throwing an exception whose type 'bad_generic_exception<std::exception>' is not derived from 'std::exception'
+  // CHECK-NOTES: 120:1: note: type defined here
   THROW_EXCEPTION(bad_generic_exception<int>);
   // CHECK MESSAGES: [[@LINE-1]]:3: warning: throwing an exception whose type 'bad_generic_exception<int>' is not derived from 'std::exception'
   THROW_EXCEPTION(bad_generic_exception<std::exception>);
   // CHECK MESSAGES: [[@LINE-1]]:3: warning: throwing an exception whose type 'bad_generic_exception<std::exception>' is not derived from 'std::exception'
 
   throw exotic_exception<non_derived_exception>();
-  // CHECK-MESSAGES: [[@LINE-1]]:9: warning: throwing an exception whose type 'exotic_exception<non_derived_exception>' is not derived from 'std::exception'
+  // CHECK-NOTES: [[@LINE-1]]:9: warning: throwing an exception whose type 'exotic_exception<non_derived_exception>' is not derived from 'std::exception'
+  // CHECK-NOTES: 123:1: note: type defined here
   THROW_EXCEPTION(exotic_exception<non_derived_exception>);
   // CHECK MESSAGES: [[@LINE-1]]:3: warning: throwing an exception whose type 'exotic_exception<non_derived_exception>' is not derived from 'std::exception'
 
@@ -168,12 +171,12 @@ using UsingGood = deep_hierarchy;
 
 void typedefed() {
   throw TypedefedBad();
-  // CHECK-MESSAGES: [[@LINE-1]]:9: warning: throwing an exception whose type 'TypedefedBad' (aka 'int') is not derived from 'std::exception'
-  // CHECK-MESSAGES: 164:1: note: type defined here
+  // CHECK-NOTES: [[@LINE-1]]:9: warning: throwing an exception whose type 'TypedefedBad' (aka 'int') is not derived from 'std::exception'
+  // CHECK-NOTES: 167:1: note: type defined here
   throw TypedefedGood(); // Ok
 
   throw UsingBad();
-  // CHECK-MESSAGES: [[@LINE-1]]:9: warning: throwing an exception whose type 'UsingBad' (aka 'int') is not derived from 'std::exception'
-  // CHECK-MESSAGES: 166:1: note: type defined here
+  // CHECK-NOTES: [[@LINE-1]]:9: warning: throwing an exception whose type 'UsingBad' (aka 'int') is not derived from 'std::exception'
+  // CHECK-NOTES: 169:1: note: type defined here
   throw UsingGood(); // Ok
 }