Make the Preprocessor more memory efficient and improve macro instantiation diagnostics.

When a macro instantiation occurs, reserve a SLocEntry chunk with length the
full length of the macro definition source. Set the spelling location of this chunk
to point to the start of the macro definition and any tokens that are lexed directly
from the macro definition will get a location from this chunk with the appropriate offset.

For any tokens that come from argument expansion, '##' paste operator, etc. have their
instantiation location point at the appropriate place in the instantiated macro definition
(the argument identifier and the '##' token respectively).
This improves macro instantiation diagnostics:

Before:

t.c:5:9: error: invalid operands to binary expression ('struct S' and 'int')
int y = M(/);
        ^~~~
t.c:5:11: note: instantiated from:
int y = M(/);
          ^

After:

t.c:5:9: error: invalid operands to binary expression ('struct S' and 'int')
int y = M(/);
        ^~~~
t.c:3:20: note: instantiated from:
\#define M(op) (foo op 3);
                ~~~ ^  ~
t.c:5:11: note: instantiated from:
int y = M(/);
          ^

The memory savings for a candidate boost library that abuses the preprocessor are:

- 32% less SLocEntries (37M -> 25M)
- 30% reduction in PCH file size (900M -> 635M)
- 50% reduction in memory usage for the SLocEntry table (1.6G -> 800M)

llvm-svn: 134587

1 files changed, 34 insertions, 52 deletions

diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index d6c4c16cec7..137da0d87ad 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/Lex/Lexer.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/SourceManagerInternals.h"
 #include "clang/Basic/Diagnostic.h"
@@ -1216,73 +1217,56 @@ PresumedLoc SourceManager::getPresumedLoc(SourceLocation Loc) const {
 
 /// \brief Returns true if the given MacroID location points at the first
 /// token of the macro instantiation.
-bool SourceManager::isAtStartOfMacroInstantiation(SourceLocation loc) const {
+bool SourceManager::isAtStartOfMacroInstantiation(SourceLocation loc,
+                                            const LangOptions &LangOpts) const {
   assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
 
   std::pair<FileID, unsigned> infoLoc = getDecomposedLoc(loc);
+  // FIXME: If the token comes from the macro token paste operator ('##')
+  // this function will always return false;
   if (infoLoc.second > 0)
     return false; // Does not point at the start of token.
 
-  unsigned FID = infoLoc.first.ID;
-  assert(FID > 1);
-  std::pair<SourceLocation, SourceLocation>
-    instRange = getImmediateInstantiationRange(loc);
-
-  bool invalid = false;
-  const SrcMgr::SLocEntry &Entry = getSLocEntry(FID-1, &invalid);
-  if (invalid)
-    return false;
-
-  // If the FileID immediately before it is a file then this is the first token
-  // in the macro.
-  if (Entry.isFile())
-    return true;
+  SourceLocation instLoc = 
+      getSLocEntry(infoLoc.first).getInstantiation().getInstantiationLocStart();
+  if (instLoc.isFileID())
+    return true; // No other macro instantiations, this is the first.
 
-  // If the FileID immediately before it (which is a macro token) is the
-  // immediate instantiated macro, check this macro token's location.
-  if (getFileID(instRange.second).ID == FID-1)
-    return isAtStartOfMacroInstantiation(instRange.first);
-
-  // If the FileID immediately before it (which is a macro token) came from a
-  // different instantiation, then this is the first token in the macro.
-  if (getInstantiationLoc(Entry.getInstantiation().getInstantiationLocStart())
-        != getInstantiationLoc(loc))
-    return true;
-
-  // It is inside the macro or the last token in the macro.
-  return false;
+  return isAtStartOfMacroInstantiation(instLoc, LangOpts);
 }
 
 /// \brief Returns true if the given MacroID location points at the last
 /// token of the macro instantiation.
-bool SourceManager::isAtEndOfMacroInstantiation(SourceLocation loc) const {
+bool SourceManager::isAtEndOfMacroInstantiation(SourceLocation loc,
+                                            const LangOptions &LangOpts) const {
   assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
 
-  unsigned FID = getFileID(loc).ID;
-  assert(FID > 1);
-  std::pair<SourceLocation, SourceLocation>
-    instRange = getInstantiationRange(loc);
+  SourceLocation spellLoc = getSpellingLoc(loc);
+  unsigned tokLen = Lexer::MeasureTokenLength(spellLoc, *this, LangOpts);
+  if (tokLen == 0)
+    return false;
+
+  std::pair<FileID, unsigned> infoLoc = getDecomposedLoc(loc);
+  unsigned FID = infoLoc.first.ID;
 
-  // If there's no FileID after it, it is the last token in the macro.
+  unsigned NextOffset;
   if (FID+1 == sloc_entry_size())
-    return true;
-
-  bool invalid = false;
-  const SrcMgr::SLocEntry &Entry = getSLocEntry(FID+1, &invalid);
-  if (invalid)
-    return false;
+    NextOffset = getNextOffset();
+  else
+    NextOffset = getSLocEntry(FID+1).getOffset();
 
-  // If the FileID immediately after it is a file or a macro token which
-  // came from a different instantiation, then this is the last token in the
-  // macro.
-  if (Entry.isFile())
-    return true;
-  if (getInstantiationLoc(Entry.getInstantiation().getInstantiationLocStart())
-        != instRange.first)
-    return true;
+  // FIXME: If the token comes from the macro token paste operator ('##')
+  // or the stringify operator ('#') this function will always return false;
+  assert(loc.getOffset() + tokLen < NextOffset);
+  if (loc.getOffset() + tokLen < NextOffset-1)
+    return false; // Does not point to the last token.
+  
+  SourceLocation instLoc = 
+      getSLocEntry(infoLoc.first).getInstantiation().getInstantiationLocEnd();
+  if (instLoc.isFileID())
+    return true; // No other macro instantiations.
 
-  // It is inside the macro or the first token in the macro.
-  return false;
+  return isAtEndOfMacroInstantiation(instLoc, LangOpts);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1479,8 +1463,6 @@ bool SourceManager::isBeforeInTranslationUnit(SourceLocation LHS,
   // reflect the order that the tokens, pointed to by these locations, were
   // instantiated (during parsing each token that is instantiated by a macro,
   // expands the SLocEntries).
-  if (LHS.isMacroID() && RHS.isMacroID())
-    return LHS.getOffset() < RHS.getOffset();
 
   std::pair<FileID, unsigned> LOffs = getDecomposedLoc(LHS);
   std::pair<FileID, unsigned> ROffs = getDecomposedLoc(RHS);