Fix for PR21254 - Assertion in comment parser

The size of the ID field in CommandInfo was narrow, leading to potential 
wrap-around of command IDs, causing misinterpretation later on.
The patch does the following:
- It extends the ID bitfield from 8 to 20 bits.
- It provides a DRY definition of the number of bits for the field to 
  avoid using literal numbers in different files.
- It introduces a new assertion that checks for the wrap-around.
- It adds the testcase from PR21254.

llvm-svn: 219802

4 files changed, 310 insertions, 5 deletions

diff --git a/clang/include/clang/AST/Comment.h b/clang/include/clang/AST/Comment.h
index e18fe9ab86a..94470cbf305 100644
--- a/clang/include/clang/AST/Comment.h
+++ b/clang/include/clang/AST/Comment.h
@@ -96,9 +96,10 @@ protected:
     unsigned : NumInlineContentCommentBits;
 
     unsigned RenderKind : 2;
-    unsigned CommandID : 8;
+    unsigned CommandID : CommandInfo::NumCommandIDBits;
   };
-  enum { NumInlineCommandCommentBits = NumInlineContentCommentBits + 10 };
+  enum { NumInlineCommandCommentBits = NumInlineContentCommentBits + 2 + 
+                                       CommandInfo::NumCommandIDBits };
 
   class HTMLTagCommentBitfields {
     friend class HTMLTagComment;
@@ -139,13 +140,14 @@ protected:
 
     unsigned : NumCommentBits;
 
-    unsigned CommandID : 8;
+    unsigned CommandID : CommandInfo::NumCommandIDBits;
 
     /// Describes the syntax that was used in a documentation command.
     /// Contains values from CommandMarkerKind enum.
     unsigned CommandMarker : 1;
   };
-  enum { NumBlockCommandCommentBits = NumCommentBits + 9 };
+  enum { NumBlockCommandCommentBits = NumCommentBits + 
+                                      CommandInfo::NumCommandIDBits + 1 };
 
   class ParamCommandCommentBitfields {
     friend class ParamCommandComment;
diff --git a/clang/include/clang/AST/CommentCommandTraits.h b/clang/include/clang/AST/CommentCommandTraits.h
index 3a9e875d8d3..ec6d83c0302 100644
--- a/clang/include/clang/AST/CommentCommandTraits.h
+++ b/clang/include/clang/AST/CommentCommandTraits.h
@@ -40,7 +40,11 @@ struct CommandInfo {
   /// Name of the command that ends the verbatim block.
   const char *EndCommandName;
 
-  unsigned ID : 8;
+  /// DRY definition of the number of bits used for a command ID.
+  enum { NumCommandIDBits = 20 };
+
+  /// The ID of the command.
+  unsigned ID : NumCommandIDBits;
 
   /// Number of word-like arguments for a given block command, except for
   /// \\param and \\tparam commands -- these have special argument parsers.
diff --git a/clang/lib/AST/CommentCommandTraits.cpp b/clang/lib/AST/CommentCommandTraits.cpp
index a7b07a40c98..7378a7c3ac0 100644
--- a/clang/lib/AST/CommentCommandTraits.cpp
+++ b/clang/lib/AST/CommentCommandTraits.cpp
@@ -89,6 +89,10 @@ CommandInfo *CommandTraits::createCommandInfoWithName(StringRef CommandName) {
   // Value-initialize (=zero-initialize in this case) a new CommandInfo.
   CommandInfo *Info = new (Allocator) CommandInfo();
   Info->Name = Name;
+  // We only have a limited number of bits to encode command IDs in the
+  // CommandInfo structure, so the ID numbers can potentially wrap around.
+  assert((NextID < (1 << CommandInfo::NumCommandIDBits))
+         && "Too many commands. We have limited bits for the command ID.");
   Info->ID = NextID++;
 
   RegisteredCommands.push_back(Info);
diff --git a/clang/test/Index/comment-lots-of-unknown-commands.c b/clang/test/Index/comment-lots-of-unknown-commands.c
new file mode 100644
index 00000000000..f0640effebc
--- /dev/null
+++ b/clang/test/Index/comment-lots-of-unknown-commands.c
@@ -0,0 +1,295 @@
+// RUN: c-index-test -test-load-source-reparse 1 local %s | FileCheck %s
+
+// See PR 21254. We had too few bits to encode command IDs so if you created
+// enough of them the ID codes would wrap around. This test creates commands up
+// to an ID of 258. Ideally we should check for large numbers, but that would
+// require a test source file which is megabytes in size. This is the test case
+// from the PR.
+
+/**
+@s
+@tr
+@y
+@tt
+@tg
+@alu
+@U
+@I
+@r
+@t0
+@t1
+@ur
+@S
+@E
+@pb
+@f
+@pe
+@lue
+@re
+@oa
+@l
+@x
+@R
+@ute
+@am
+@ei
+@oun
+@ou
+@nl
+@ien
+@fr
+@en
+@tet
+@le
+@L
+@os
+@A
+@ro
+@o
+@ho
+@ca
+@Tie
+@tl
+@g
+@hr
+@et
+@fro
+@ast
+@ae
+@nN
+@pc
+@tae
+@ws
+@ia
+@N
+@lc
+@psg
+@ta
+@t2
+@D
+@str
+@ra
+@t3
+@t
+@xt
+@eN
+@fe
+@rU
+@ar
+@eD
+@iE
+@se
+@st1
+@rr
+@ime
+@ft
+@lm
+@wD
+@wne
+@h
+@otn
+@use
+@roi
+@ldc
+@ln
+@d
+@ee
+@ep
+@us
+@ut
+@u
+@n
+@Nme
+@min
+@ma
+@pct
+@hd
+@be
+@It
+@id
+@cm
+@ua
+@fs
+@Al
+@axn
+@rt
+@to
+@is
+@fo
+@i
+@an
+@de
+@tel
+@nd
+@dic
+@Lo
+@il
+@tle
+@axt
+@ba
+@ust
+@ac
+@tpe
+@tpl
+@ctG
+@ru
+@m
+@tG
+@it
+@rh
+@G
+@rpc
+@el
+@er
+@w
+@eo
+@tx
+@oo
+@dD
+@dD
+*/
+void f();
+
+// CHECK:  CommentAST=[
+// CHECK:    (CXComment_FullComment
+// CHECK:       (CXComment_Paragraph
+// CHECK:         (CXComment_InlineCommand CommandName=[s] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[tr] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[y] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[tt] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[tg] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[alu] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[U] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[I] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[r] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[t0] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[t1] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ur] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[S] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[E] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[pb] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[f] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[pe] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[lue] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[re] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[oa] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[l] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[x] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[R] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ute] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[am] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ei] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[oun] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ou] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[nl] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ien] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[fr] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[en] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[tet] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[le] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[L] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[os] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[A] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ro] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[o] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ho] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ca] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[Tie] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[tl] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[g] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[hr] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[et] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[fro] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ast] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ae] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[nN] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[pc] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[tae] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ws] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ia] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[N] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[lc] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[psg] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ta] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[t2] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[D] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[str] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ra] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[t3] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[t] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[xt] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[eN] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[fe] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[rU] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ar] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[eD] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[iE] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[se] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[st1] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[rr] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ime] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ft] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[lm] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[wD] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[wne] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[h] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[otn] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[use] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[roi] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ldc] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ln] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[d] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ee] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ep] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[us] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ut] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[u] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[n] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[Nme] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[min] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ma] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[pct] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[hd] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[be] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[It] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[id] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[cm] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ua] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[fs] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[Al] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[axn] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[rt] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[to] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[is] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[fo] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[i] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[an] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[de] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[tel] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[nd] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[dic] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[Lo] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[il] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[tle] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[axt] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ba] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ust] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ac] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[tpe] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[tpl] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ctG] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[ru] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[m] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[tG] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[it] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[rh] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[G] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[rpc] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[el] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[er] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[w] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[eo] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[tx] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[oo] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[dD] RenderNormal HasTrailingNewline)
+// CHECK:         (CXComment_InlineCommand CommandName=[dD] RenderNormal)))]