Rework the detailed preprocessing record to separate preprocessing

entities generated directly by the preprocessor from those loaded from
the external source (e.g., the ASTReader). By separating these two
sets of entities into different vectors, we allow both to grow
independently, and eliminate the need for preallocating all of the
loaded preprocessing entities. This is similar to the way the recent
SourceManager refactoring treats FileIDs and the source location
address space.

As part of this, switch over to building a continuous range map to
track preprocessing entities.

llvm-svn: 135646

4 files changed, 61 insertions, 70 deletions

diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp
index f7c6695d71d..26a880829ec 100644
--- a/clang/lib/Frontend/ASTUnit.cpp
+++ b/clang/lib/Frontend/ASTUnit.cpp
@@ -1545,10 +1545,6 @@ ASTUnit::pp_entity_iterator ASTUnit::pp_entity_begin() {
       PreprocessedEntities.empty())
     RealizePreprocessedEntitiesFromPreamble();
   
-  if (PreprocessedEntities.empty())
-    if (PreprocessingRecord *PPRec = PP->getPreprocessingRecord())
-      return PPRec->begin(true);
-  
   return PreprocessedEntities.begin();
 }
 
@@ -1556,11 +1552,7 @@ ASTUnit::pp_entity_iterator ASTUnit::pp_entity_end() {
   if (!PreprocessedEntitiesInPreamble.empty() &&
       PreprocessedEntities.empty())
     RealizePreprocessedEntitiesFromPreamble();
-  
-  if (PreprocessedEntities.empty())
-    if (PreprocessingRecord *PPRec = PP->getPreprocessingRecord())
-      return PPRec->end(true);
-  
+    
   return PreprocessedEntities.end();
 }
 
diff --git a/clang/lib/Lex/PreprocessingRecord.cpp b/clang/lib/Lex/PreprocessingRecord.cpp
index 9f93ab04502..6e2216ae867 100644
--- a/clang/lib/Lex/PreprocessingRecord.cpp
+++ b/clang/lib/Lex/PreprocessingRecord.cpp
@@ -47,42 +47,24 @@ void PreprocessingRecord::MaybeLoadPreallocatedEntities() const {
 
 PreprocessingRecord::PreprocessingRecord(bool IncludeNestedMacroExpansions)
   : IncludeNestedMacroExpansions(IncludeNestedMacroExpansions),
-    ExternalSource(0), NumPreallocatedEntities(0), 
-    LoadedPreallocatedEntities(false)
+    ExternalSource(0), LoadedPreallocatedEntities(false)
 {
 }
 
 PreprocessingRecord::iterator 
 PreprocessingRecord::begin(bool OnlyLocalEntities) {
   if (OnlyLocalEntities)
-    return PreprocessedEntities.begin() + NumPreallocatedEntities;
+    return iterator(this, 0);
   
   MaybeLoadPreallocatedEntities();
-  return PreprocessedEntities.begin();
+  return iterator(this, -(int)LoadedPreprocessedEntities.size());
 }
 
 PreprocessingRecord::iterator PreprocessingRecord::end(bool OnlyLocalEntities) {
   if (!OnlyLocalEntities)
     MaybeLoadPreallocatedEntities();
   
-  return PreprocessedEntities.end();
-}
-
-PreprocessingRecord::const_iterator 
-PreprocessingRecord::begin(bool OnlyLocalEntities) const {
-  if (OnlyLocalEntities)
-    return PreprocessedEntities.begin() + NumPreallocatedEntities;
-  
-  MaybeLoadPreallocatedEntities();
-  return PreprocessedEntities.begin();
-}
-
-PreprocessingRecord::const_iterator 
-PreprocessingRecord::end(bool OnlyLocalEntities) const {
-  if (!OnlyLocalEntities)
-    MaybeLoadPreallocatedEntities();
-  
-  return PreprocessedEntities.end();
+  return iterator(this, PreprocessedEntities.size());
 }
 
 void PreprocessingRecord::addPreprocessedEntity(PreprocessedEntity *Entity) {
@@ -90,20 +72,25 @@ void PreprocessingRecord::addPreprocessedEntity(PreprocessedEntity *Entity) {
 }
 
 void PreprocessingRecord::SetExternalSource(
-                                    ExternalPreprocessingRecordSource &Source,
-                                            unsigned NumPreallocatedEntities) {
+                                    ExternalPreprocessingRecordSource &Source) {
   assert(!ExternalSource &&
          "Preprocessing record already has an external source");
   ExternalSource = &Source;
-  this->NumPreallocatedEntities = NumPreallocatedEntities;
-  PreprocessedEntities.insert(PreprocessedEntities.begin(), 
-                              NumPreallocatedEntities, 0);
 }
 
-void PreprocessingRecord::SetPreallocatedEntity(unsigned Index, 
-                                                PreprocessedEntity *Entity) {
-  assert(Index < NumPreallocatedEntities &&"Out-of-bounds preallocated entity");
-  PreprocessedEntities[Index] = Entity;
+unsigned PreprocessingRecord::allocateLoadedEntities(unsigned NumEntities) {
+  unsigned Result = LoadedPreprocessedEntities.size();
+  LoadedPreprocessedEntities.resize(LoadedPreprocessedEntities.size() 
+                                    + NumEntities);
+  return Result;
+}
+
+void 
+PreprocessingRecord::setLoadedPreallocatedEntity(unsigned Index, 
+                                                 PreprocessedEntity *Entity) {
+  assert(Index < LoadedPreprocessedEntities.size() &&
+         "Out-of-bounds preallocated entity");
+  LoadedPreprocessedEntities[Index] = Entity;
 }
 
 void PreprocessingRecord::RegisterMacroDefinition(MacroInfo *Macro, 
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 884cc1556bf..4465b6b801f 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1561,7 +1561,7 @@ PreprocessedEntity *ASTReader::LoadPreprocessedEntity(PerFileData &F) {
                                              Code, Record, BlobStart, BlobLen);
   switch (RecType) {
   case PPD_MACRO_EXPANSION: {
-    if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
+    if (PreprocessedEntity *PE = PPRec.getLoadedPreprocessedEntity(Record[0]))
       return PE;
     
     MacroExpansion *ME =
@@ -1569,12 +1569,12 @@ PreprocessedEntity *ASTReader::LoadPreprocessedEntity(PerFileData &F) {
                                  SourceRange(ReadSourceLocation(F, Record[1]),
                                              ReadSourceLocation(F, Record[2])),
                                  getMacroDefinition(Record[4]));
-    PPRec.SetPreallocatedEntity(Record[0], ME);
+    PPRec.setLoadedPreallocatedEntity(Record[0], ME);
     return ME;
   }
       
   case PPD_MACRO_DEFINITION: {
-    if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
+    if (PreprocessedEntity *PE = PPRec.getLoadedPreprocessedEntity(Record[0]))
       return PE;
     
     if (Record[1] > MacroDefinitionsLoaded.size()) {
@@ -1593,7 +1593,7 @@ PreprocessedEntity *ASTReader::LoadPreprocessedEntity(PerFileData &F) {
                                             ReadSourceLocation(F, Record[2]),
                                             ReadSourceLocation(F, Record[3])));
       
-      PPRec.SetPreallocatedEntity(Record[0], MD);
+      PPRec.setLoadedPreallocatedEntity(Record[0], MD);
       MacroDefinitionsLoaded[Record[1] - 1] = MD;
       
       if (DeserializationListener)
@@ -1604,7 +1604,7 @@ PreprocessedEntity *ASTReader::LoadPreprocessedEntity(PerFileData &F) {
   }
       
   case PPD_INCLUSION_DIRECTIVE: {
-    if (PreprocessedEntity *PE = PPRec.getPreprocessedEntity(Record[0]))
+    if (PreprocessedEntity *PE = PPRec.getLoadedPreprocessedEntity(Record[0]))
       return PE;
     
     const char *FullFileNameStart = BlobStart + Record[3];
@@ -1622,7 +1622,7 @@ PreprocessedEntity *ASTReader::LoadPreprocessedEntity(PerFileData &F) {
                                        File,
                                  SourceRange(ReadSourceLocation(F, Record[1]),
                                              ReadSourceLocation(F, Record[2])));
-    PPRec.SetPreallocatedEntity(Record[0], ID);
+    PPRec.setLoadedPreallocatedEntity(Record[0], ID);
     return ID;
   }
   }
@@ -2356,22 +2356,43 @@ ASTReader::ReadASTBlock(PerFileData &F) {
       break;
     }
 
-    case MACRO_DEFINITION_OFFSETS:
+    case MACRO_DEFINITION_OFFSETS: {
       F.MacroDefinitionOffsets = (const uint32_t *)BlobStart;
       F.NumPreallocatedPreprocessingEntities = Record[0];
       F.LocalNumMacroDefinitions = Record[1];
-        
-      // Introduce the global -> local mapping for identifiers within this AST
-      // file
+
+      // Introduce the global -> local mapping for preprocessed entities within 
+      // this AST file.
+      unsigned StartingID;
+      if (PP) {
+        if (!PP->getPreprocessingRecord())
+          PP->createPreprocessingRecord(true);
+        if (!PP->getPreprocessingRecord()->getExternalSource())
+          PP->getPreprocessingRecord()->SetExternalSource(*this);
+        StartingID 
+          = PP->getPreprocessingRecord()
+              ->allocateLoadedEntities(F.NumPreallocatedPreprocessingEntities);
+      } else {
+        // FIXME: We'll eventually want to kill this path, since it assumes
+        // a particular allocation strategy in the preprocessing record.
+        StartingID = getTotalNumPreprocessedEntities();
+      }
+      
+      GlobalPreprocessedEntityMap.insert(
+                        std::make_pair(StartingID,
+                                         std::make_pair(&F, -(int)StartingID)));
+
+      // Introduce the global -> local mapping for macro definitions within 
+      // this AST file.
       GlobalMacroDefinitionMap.insert(
                std::make_pair(getTotalNumMacroDefinitions() + 1, 
                               std::make_pair(&F, 
                                              -getTotalNumMacroDefinitions())));
       MacroDefinitionsLoaded.resize(
                     MacroDefinitionsLoaded.size() + F.LocalNumMacroDefinitions);
-
       break;
-
+    }
+        
     case DECL_UPDATE_OFFSETS: {
       if (Record.size() % 2 != 0) {
         Error("invalid DECL_UPDATE_OFFSETS block in AST file");
@@ -2558,21 +2579,12 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName,
   }
 
   // Allocate space for loaded slocentries, identifiers, decls and types.
-  unsigned TotalNumPreallocatedPreprocessingEntities = 0;
   for (unsigned I = 0, N = Chain.size(); I != N; ++I) {
     TotalNumSLocEntries += Chain[I]->LocalNumSLocEntries;
-    TotalNumPreallocatedPreprocessingEntities +=
-        Chain[I]->NumPreallocatedPreprocessingEntities;
   }
-  if (PP) {
+  if (PP)
     PP->getHeaderSearchInfo().SetExternalLookup(this);
-    if (TotalNumPreallocatedPreprocessingEntities > 0) {
-      if (!PP->getPreprocessingRecord())
-        PP->createPreprocessingRecord(true);
-      PP->getPreprocessingRecord()->SetExternalSource(*this,
-                                     TotalNumPreallocatedPreprocessingEntities);
-    }
-  }
+
   // Preload SLocEntries.
   for (unsigned I = 0, N = PreloadSLocEntries.size(); I != N; ++I) {
     ASTReadResult Result = ReadSLocEntryRecord(PreloadSLocEntries[I]);
@@ -2768,15 +2780,15 @@ ASTReader::ASTReadResult ASTReader::ReadASTCore(llvm::StringRef FileName,
 
 void ASTReader::setPreprocessor(Preprocessor &pp) {
   PP = &pp;
-
-  unsigned TotalNum = 0;
-  for (unsigned I = 0, N = Chain.size(); I != N; ++I)
-    TotalNum += Chain[I]->NumPreallocatedPreprocessingEntities;
-  if (TotalNum) {
+  
+  if (unsigned N = getTotalNumPreprocessedEntities()) {
     if (!PP->getPreprocessingRecord())
       PP->createPreprocessingRecord(true);
-    PP->getPreprocessingRecord()->SetExternalSource(*this, TotalNum);
+    PP->getPreprocessingRecord()->SetExternalSource(*this);
+    PP->getPreprocessingRecord()->allocateLoadedEntities(N);
   }
+  
+  PP->getHeaderSearchInfo().SetExternalLookup(this);
 }
 
 void ASTReader::InitializeContext(ASTContext &Ctx) {
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index a3a03a5b1f1..aaf4678ba29 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -1801,7 +1801,7 @@ void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) {
     InclusionAbbrev = Stream.EmitAbbrev(Abbrev);
   }
   
-  unsigned IndexBase = Chain ? PPRec.getNumPreallocatedEntities() : 0;
+  unsigned IndexBase = Chain ? PPRec.getNumLoadedPreprocessedEntities() : 0;
   RecordData Record;
   for (PreprocessingRecord::iterator E = PPRec.begin(Chain),
                                   EEnd = PPRec.end(Chain);