Rework the detailed preprocessing record to separate preprocessing

entities generated directly by the preprocessor from those loaded from
the external source (e.g., the ASTReader). By separating these two
sets of entities into different vectors, we allow both to grow
independently, and eliminate the need for preallocating all of the
loaded preprocessing entities. This is similar to the way the recent
SourceManager refactoring treats FileIDs and the source location
address space.

As part of this, switch over to building a continuous range map to
track preprocessing entities.

llvm-svn: 135646

2 files changed, 42 insertions, 30 deletions

diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 884cc1556bf..4465b6b801f 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1561,7 +1561,7 @@ PreprocessedEntity *ASTReader::LoadPreprocessedEntity(PerFileData &F) {
                                              Code, Record, BlobStart, BlobLen);
   switch (RecType) {
   case PPD_MACRO_EXPANSION: {
-    if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
+    if (PreprocessedEntity *PE = PPRec.getLoadedPreprocessedEntity(Record[0]))
       return PE;
     
     MacroExpansion *ME =
@@ -1569,12 +1569,12 @@ PreprocessedEntity *ASTReader::LoadPreprocessedEntity(PerFileData &F) {
                                  SourceRange(ReadSourceLocation(F, Record[1]),
                                              ReadSourceLocation(F, Record[2])),
                                  getMacroDefinition(Record[4]));
-    PPRec.SetPreallocatedEntity(Record[0], ME);
+    PPRec.setLoadedPreallocatedEntity(Record[0], ME);
     return ME;
   }
       
   case PPD_MACRO_DEFINITION: {
-    if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
+    if (PreprocessedEntity *PE = PPRec.getLoadedPreprocessedEntity(Record[0]))
       return PE;
     
     if (Record[1] > MacroDefinitionsLoaded.size()) {
@@ -1593,7 +1593,7 @@ PreprocessedEntity *ASTReader::LoadPreprocessedEntity(PerFileData &F) {
                                             ReadSourceLocation(F, Record[2]),
                                             ReadSourceLocation(F, Record[3])));
       
-      PPRec.SetPreallocatedEntity(Record[0], MD);
+      PPRec.setLoadedPreallocatedEntity(Record[0], MD);
       MacroDefinitionsLoaded[Record[1] - 1] = MD;
       
       if (DeserializationListener)
@@ -1604,7 +1604,7 @@ PreprocessedEntity *ASTReader::LoadPreprocessedEntity(PerFileData &F) {
   }
       
   case PPD_INCLUSION_DIRECTIVE: {
-    if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
+    if (PreprocessedEntity *PE = PPRec.getLoadedPreprocessedEntity(Record[0]))
       return PE;
     
     const char *FullFileNameStart = BlobStart + Record[3];
@@ -1622,7 +1622,7 @@ PreprocessedEntity *ASTReader::LoadPreprocessedEntity(PerFileData &F) {
                                        File,
                                  SourceRange(ReadSourceLocation(F, Record[1]),
                                              ReadSourceLocation(F, Record[2])));
-    PPRec.SetPreallocatedEntity(Record[0], ID);
+    PPRec.setLoadedPreallocatedEntity(Record[0], ID);
     return ID;
   }
   }
@@ -2356,22 +2356,43 @@ ASTReader::ReadASTBlock(PerFileData &F) {
       break;
     }
 
-    case MACRO_DEFINITION_OFFSETS:
+    case MACRO_DEFINITION_OFFSETS: {
       F.MacroDefinitionOffsets = (const uint32_t *)BlobStart;
       F.NumPreallocatedPreprocessingEntities = Record[0];
       F.LocalNumMacroDefinitions = Record[1];
-        
-      // Introduce the global -> local mapping for identifiers within this AST
-      // file
+
+      // Introduce the global -> local mapping for preprocessed entities within 
+      // this AST file.
+      unsigned StartingID;
+      if (PP) {
+        if (!PP->getPreprocessingRecord())
+          PP->createPreprocessingRecord(true);
+        if (!PP->getPreprocessingRecord()->getExternalSource())
+          PP->getPreprocessingRecord()->SetExternalSource(*this);
+        StartingID 
+          = PP->getPreprocessingRecord()
+              ->allocateLoadedEntities(F.NumPreallocatedPreprocessingEntities);
+      } else {
+        // FIXME: We'll eventually want to kill this path, since it assumes
+        // a particular allocation strategy in the preprocessing record.
+        StartingID = getTotalNumPreprocessedEntities();
+      }
+      
+      GlobalPreprocessedEntityMap.insert(
+                        std::make_pair(StartingID,
+                                         std::make_pair(&F, -(int)StartingID)));
+
+      // Introduce the global -> local mapping for macro definitions within 
+      // this AST file.
       GlobalMacroDefinitionMap.insert(
                std::make_pair(getTotalNumMacroDefinitions() + 1, 
                               std::make_pair(&F, 
                                              -getTotalNumMacroDefinitions())));
       MacroDefinitionsLoaded.resize(
                     MacroDefinitionsLoaded.size() + F.LocalNumMacroDefinitions);
-
       break;
-
+    }
+        
     case DECL_UPDATE_OFFSETS: {
       if (Record.size() % 2 != 0) {
         Error("invalid DECL_UPDATE_OFFSETS block in AST file");
@@ -2558,21 +2579,12 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName,
   }
 
   // Allocate space for loaded slocentries, identifiers, decls and types.
-  unsigned TotalNumPreallocatedPreprocessingEntities = 0;
   for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
     TotalNumSLocEntries += Chain[I]->LocalNumSLocEntries;
-    TotalNumPreallocatedPreprocessingEntities +=
-        Chain[I]->NumPreallocatedPreprocessingEntities;
   }
-  if (PP) {
+  if (PP)
     PP->getHeaderSearchInfo().SetExternalLookup(this);
-    if (TotalNumPreallocatedPreprocessingEntities > 0) {
-      if (!PP->getPreprocessingRecord())
-        PP->createPreprocessingRecord(true);
-      PP->getPreprocessingRecord()->SetExternalSource(*this,
-                                     TotalNumPreallocatedPreprocessingEntities);
-    }
-  }
+
   // Preload SLocEntries.
   for (unsigned I = 0, N = PreloadSLocEntries.size(); I != N; ++I) {
     ASTReadResult Result = ReadSLocEntryRecord(PreloadSLocEntries[I]);
@@ -2768,15 +2780,15 @@ ASTReader::ASTReadResult ASTReader::ReadASTCore(llvm::StringRef FileName,
 
 void ASTReader::setPreprocessor(Preprocessor &pp) {
   PP = &pp;
-
-  unsigned TotalNum = 0;
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I)
-    TotalNum += Chain[I]->NumPreallocatedPreprocessingEntities;
-  if (TotalNum) {
+  
+  if (unsigned N = getTotalNumPreprocessedEntities()) {
     if (!PP->getPreprocessingRecord())
       PP->createPreprocessingRecord(true);
-    PP->getPreprocessingRecord()->SetExternalSource(*this, TotalNum);
+    PP->getPreprocessingRecord()->SetExternalSource(*this);
+    PP->getPreprocessingRecord()->allocateLoadedEntities(N);
   }
+  
+  PP->getHeaderSearchInfo().SetExternalLookup(this);
 }
 
 void ASTReader::InitializeContext(ASTContext &Ctx) {
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index a3a03a5b1f1..aaf4678ba29 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -1801,7 +1801,7 @@ void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) {
     InclusionAbbrev = Stream.EmitAbbrev(Abbrev);
   }
   
-  unsigned IndexBase = Chain ? PPRec.getNumPreallocatedEntities() : 0;
+  unsigned IndexBase = Chain ? PPRec.getNumLoadedPreprocessedEntities() : 0;
   RecordData Record;
   for (PreprocessingRecord::iterator E = PPRec.begin(Chain),
                                   EEnd = PPRec.end(Chain);