Implement a proper local -> global type ID remapping scheme in the AST

reader. This scheme permits an AST file to be loaded with its type IDs
shifted anywhere in the type ID space. 

At present, the type indices are still allocated in the same boring
way they always have been, just by adding up the number of types in
each PCH file within the chain. However, I've done testing with this
patch by randomly sliding the base indices at load time, to ensure
that remapping is occurring as expected. I may eventually formalize
this in some testing flag, but loading multiple (non-chained) AST
files at once will eventually exercise the same code.

There is one known problem with this patch, which involves name lookup
of operator names (e.g., "x.operator int*()") in cases where multiple
PCH files in the chain. The hash function itself depends on having a
stable type ID, which doesn't happen with chained PCH and *certainly*
doesn't happen when sliding type IDs around. We'll need another
approach. I'll tackle that next.

llvm-svn: 136693

5 files changed, 139 insertions, 49 deletions

diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 97b4cfc0283..21fb8fa4edf 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -84,9 +84,15 @@ namespace clang {
 
       uint32_t getIndex() const { return Idx; }
       TypeID asTypeID(unsigned FastQuals) const {
+        if (Idx == uint32_t(-1))
+          return TypeID(-1);
+        
         return (Idx << Qualifiers::FastWidth) | FastQuals;
       }
       static TypeIdx fromTypeID(TypeID ID) {
+        if (ID == TypeID(-1))
+          return TypeIdx(-1);
+        
         return TypeIdx(ID >> Qualifiers::FastWidth);
       }
     };
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index e28179d3353..e167985e306 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -380,9 +380,26 @@ public:
   /// type ID, or the representation of a Type*.
   const uint32_t *TypeOffsets;
   
-  /// \brief Base type ID for types local to this module.
-  serialization::TypeID BaseTypeID;
-  
+  /// \brief Base type ID for types local to this module as represented in 
+  /// the global type ID space.
+  serialization::TypeID GlobalBaseTypeIndex;
+  
+  /// \brief Remapping table for type IDs in this module.
+  ContinuousRangeMap<uint32_t, int, 2> TypeRemap;
+
+  /// \brief Base type ID for types local to this module as represented in
+  /// the module's type ID space.
+  serialization::TypeID LocalBaseTypeIndex;
+
+  /// \brief Remapping table that maps from a type as represented as a module
+  /// and local type index to the index used within the current module to
+  /// refer to that same type.
+  /// 
+  /// This mapping is effectively the reverse of the normal \c TypeRemap, and
+  /// is used specifically by ASTReader::GetTypeIdx() to help map between
+  /// global type IDs and a module's view of the same type ID as a hash value.
+  llvm::DenseMap<Module *, int> ReverseTypeRemap;
+
   // === Miscellaneous ===
   
   /// \brief Diagnostic IDs and their mappings that the user changed.
@@ -1233,6 +1250,10 @@ public:
   /// \brief Map a local type ID within a given AST file into a global type ID.
   serialization::TypeID getGlobalTypeID(Module &F, unsigned LocalID) const;
   
+  /// \brief Map a global type ID to an ID as it would be locally expressed
+  /// in the given model.
+  unsigned getLocalTypeID(Module &M, serialization::TypeID GlobalID);
+  
   /// \brief Read a type from the current position in the given record, which 
   /// was read from the given AST file.
   QualType readType(Module &F, const RecordData &Record, unsigned &Idx) {
diff --git a/clang/include/clang/Serialization/ContinuousRangeMap.h b/clang/include/clang/Serialization/ContinuousRangeMap.h
index 820086e8206..42b8954b54a 100644
--- a/clang/include/clang/Serialization/ContinuousRangeMap.h
+++ b/clang/include/clang/Serialization/ContinuousRangeMap.h
@@ -34,7 +34,7 @@ namespace clang {
 template <typename Int, typename V, unsigned InitialCapacity>
 class ContinuousRangeMap {
 public:
-  typedef std::pair<const Int, V> value_type;
+  typedef std::pair<Int, V> value_type;
   typedef value_type &reference;
   typedef const value_type &const_reference;
   typedef value_type *pointer;
@@ -94,7 +94,6 @@ public:
   /// from a set of values.
   class Builder {
     ContinuousRangeMap &Self;
-    SmallVector<std::pair<Int, V>, InitialCapacity> Elements;
     
     Builder(const Builder&); // DO NOT IMPLEMENT
     Builder &operator=(const Builder&); // DO NOT IMPLEMENT
@@ -103,13 +102,11 @@ public:
     explicit Builder(ContinuousRangeMap &Self) : Self(Self) { }
     
     ~Builder() {
-      std::sort(Elements.begin(), Elements.end(), Compare());
-      for (unsigned I = 0, N = Elements.size(); I != N; ++I)
-        Self.insert(Elements[I]);
+      std::sort(Self.Rep.begin(), Self.Rep.end(), Compare());
     }
     
     void insert(const value_type &Val) {
-      Elements.push_back(Val);
+      Self.Rep.push_back(Val);
     }
   };
   friend class Builder;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 69fdf10a1a8..0241b22f9e3 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -785,7 +785,10 @@ public:
     case DeclarationName::CXXConstructorName:
     case DeclarationName::CXXDestructorName:
     case DeclarationName::CXXConversionFunctionName:
-      ID.AddInteger((TypeID)Key.Data);
+      if (TypeID(Key.Data) == TypeID(-1))
+        ID.AddInteger((TypeID)Key.Data);
+      else
+        ID.AddInteger(Reader.getLocalTypeID(F, (TypeID)Key.Data));
       break;
     case DeclarationName::CXXOperatorName:
       ID.AddInteger((OverloadedOperatorKind)Key.Data);
@@ -2049,21 +2052,29 @@ ASTReader::ReadASTBlock(Module &F) {
       break;
     }
 
-    case TYPE_OFFSET:
+    case TYPE_OFFSET: {
       if (F.LocalNumTypes != 0) {
         Error("duplicate TYPE_OFFSET record in AST file");
         return Failure;
       }
       F.TypeOffsets = (const uint32_t *)BlobStart;
       F.LocalNumTypes = Record[0];
-      F.BaseTypeID = getTotalNumTypes();
+      F.LocalBaseTypeIndex = Record[1];
+      F.GlobalBaseTypeIndex = getTotalNumTypes();
         
-      // Introduce the global -> local mapping for types within this
-      // AST file.
-      GlobalTypeMap.insert(std::make_pair(getTotalNumTypes() + 1, &F));
-      TypesLoaded.resize(TypesLoaded.size() + F.LocalNumTypes);
+      if (F.LocalNumTypes > 0) {
+        // Introduce the global -> local mapping for types within this module.
+        GlobalTypeMap.insert(std::make_pair(getTotalNumTypes(), &F));
+        
+        // Introduce the local -> global mapping for types within this module.
+        F.TypeRemap.insert(std::make_pair(F.LocalBaseTypeIndex, 
+                             F.GlobalBaseTypeIndex - F.LocalBaseTypeIndex));
+        
+        TypesLoaded.resize(TypesLoaded.size() + F.LocalNumTypes);
+      }
       break;
-
+    }
+        
     case DECL_OFFSET:
       if (F.LocalNumDecls != 0) {
         Error("duplicate DECL_OFFSET record in AST file");
@@ -2278,7 +2289,8 @@ ASTReader::ReadASTBlock(Module &F) {
       
       // Continuous range maps we may be updating in our module.
       ContinuousRangeMap<uint32_t, int, 2>::Builder SLocRemap(F.SLocRemap);
-      
+      ContinuousRangeMap<uint32_t, int, 2>::Builder TypeRemap(F.TypeRemap);
+
       while(Data < DataEnd) {
         uint16_t Len = io::ReadUnalignedLE16(Data);
         StringRef Name = StringRef((const char*)Data, Len);
@@ -2296,7 +2308,7 @@ ASTReader::ReadASTBlock(Module &F) {
         uint32_t SelectorIDOffset = io::ReadUnalignedLE32(Data);
         uint32_t DeclIDOffset = io::ReadUnalignedLE32(Data);
         uint32_t CXXBaseSpecifiersIDOffset = io::ReadUnalignedLE32(Data);
-        uint32_t TypeIDOffset = io::ReadUnalignedLE32(Data);
+        uint32_t TypeIndexOffset = io::ReadUnalignedLE32(Data);
         
         // Source location offset is mapped to OM->SLocEntryBaseOffset.
         SLocRemap.insert(std::make_pair(SLocOffset,
@@ -2309,7 +2321,10 @@ ASTReader::ReadASTBlock(Module &F) {
         (void)SelectorIDOffset;
         (void)DeclIDOffset;
         (void)CXXBaseSpecifiersIDOffset;
-        (void)TypeIDOffset;
+        
+        TypeRemap.insert(std::make_pair(TypeIndexOffset, 
+                                    OM->GlobalBaseTypeIndex - TypeIndexOffset));
+        F.ReverseTypeRemap[OM] = TypeIndexOffset - OM->GlobalBaseTypeIndex;
       }
       break;
     }
@@ -3231,10 +3246,10 @@ void ASTReader::ReadPragmaDiagnosticMappings(Diagnostic &Diag) {
 
 /// \brief Get the correct cursor and offset for loading a type.
 ASTReader::RecordLocation ASTReader::TypeCursorForIndex(unsigned Index) {
-  GlobalTypeMapType::iterator I = GlobalTypeMap.find(Index+1);
+  GlobalTypeMapType::iterator I = GlobalTypeMap.find(Index);
   assert(I != GlobalTypeMap.end() && "Corrupted global type map");
   Module *M = I->second;
-  return RecordLocation(M, M->TypeOffsets[Index - M->BaseTypeID]);
+  return RecordLocation(M, M->TypeOffsets[Index - M->GlobalBaseTypeIndex]);
 }
 
 /// \brief Read and return the type with the given index..
@@ -3979,8 +3994,49 @@ QualType ASTReader::getLocalType(Module &F, unsigned LocalID) {
 
 serialization::TypeID 
 ASTReader::getGlobalTypeID(Module &F, unsigned LocalID) const {
-  // FIXME: Map from local type ID to global type ID.
-  return LocalID;
+  unsigned FastQuals = LocalID & Qualifiers::FastMask;
+  unsigned LocalIndex = LocalID >> Qualifiers::FastWidth;
+  
+  if (LocalIndex < NUM_PREDEF_TYPE_IDS)
+    return LocalID;
+
+  ContinuousRangeMap<uint32_t, int, 2>::iterator I
+    = F.TypeRemap.find(LocalIndex - NUM_PREDEF_TYPE_IDS);
+  assert(I != F.TypeRemap.end() && "Invalid index into type index remap");
+  
+  unsigned GlobalIndex = LocalIndex + I->second;
+  return (GlobalIndex << Qualifiers::FastWidth) | FastQuals;
+}
+
+unsigned ASTReader::getLocalTypeID(Module &M, serialization::TypeID GlobalID) {
+  unsigned FastQuals = GlobalID & Qualifiers::FastMask;
+  unsigned GlobalIndex = GlobalID >> Qualifiers::FastWidth;
+
+  if (GlobalIndex < NUM_PREDEF_TYPE_IDS)
+    return GlobalID;
+  
+  GlobalIndex -= NUM_PREDEF_TYPE_IDS;
+  RecordLocation Loc = TypeCursorForIndex(GlobalIndex);
+  
+  if (Loc.F == &M) {
+    // Simple case: the type ID came from the module we're asked to provide a
+    // type ID for. Shift the index appropriately;
+    unsigned LocalIndex 
+      = GlobalIndex - M.GlobalBaseTypeIndex + M.LocalBaseTypeIndex 
+      + NUM_PREDEF_TYPE_IDS ;
+    return (LocalIndex << Qualifiers::FastWidth) | FastQuals;
+  }
+
+  // Complex case: the type ID came from a module that M depends on, which may
+  // have had some remapping between the IDs used to store it in M and its
+  // location in the global space.
+  llvm::DenseMap<Module *, int>::iterator R = Loc.F->ReverseTypeRemap.find(&M);
+  if (R == Loc.F->ReverseTypeRemap.end())
+    return TypeID(-1); // FIXME: This is a terrible failure case
+  
+  unsigned LocalIndex = GlobalIndex - Loc.F->GlobalBaseTypeIndex 
+                      + R->second + NUM_PREDEF_TYPE_IDS;
+  return (LocalIndex << Qualifiers::FastWidth) | FastQuals;
 }
 
 TypeID ASTReader::GetTypeID(QualType T) const {
@@ -3993,6 +4049,10 @@ TypeIdx ASTReader::GetTypeIdx(QualType T) const {
     return TypeIdx();
   assert(!T.getLocalFastQualifiers());
 
+  // FIXME: Modules can't handle this. It's even dubious with chained PCH,
+  // because the same type (say, int*) can be serialized into different
+  // PCH files within the chain, and there's no way to know which of the
+  // ID numbers we actually want.
   TypeIdxMap::const_iterator I = TypeIdxs.find(T);
   // GetTypeIdx is mostly used for computing the hash of DeclarationNames and
   // comparing keys of ASTDeclContextNameLookupTable.
@@ -5473,7 +5533,8 @@ Module::Module(ModuleKind Kind)
     DeclOffsets(0), BaseDeclID(0),
     LocalNumCXXBaseSpecifiers(0), CXXBaseSpecifiersOffsets(0),
     BaseCXXBaseSpecifiersID(0),
-    LocalNumTypes(0), TypeOffsets(0), BaseTypeID(0), StatCache(0),
+    LocalNumTypes(0), TypeOffsets(0), GlobalBaseTypeIndex(0), 
+    LocalBaseTypeIndex(0), StatCache(0),
     NumPreallocatedPreprocessingEntities(0)
 {}
 
@@ -5515,6 +5576,9 @@ void Module::dump() {
   llvm::errs() << "  Base source location offset: " << SLocEntryBaseOffset 
                << '\n';
   dumpLocalRemap("Source location offset map", SLocRemap);
+  llvm::errs() << "  Base type ID: " << GlobalBaseTypeIndex << '\n'
+               << "  Number of types: " << LocalNumTypes << '\n';
+  dumpLocalRemap("Type ID map", TypeRemap);
 }
 
 Module *ModuleManager::lookup(StringRef Name) {
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index fb24f851887..197690171d5 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -2020,11 +2020,13 @@ void ASTWriter::WriteTypeDeclOffsets() {
   BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
   Abbrev->Add(BitCodeAbbrevOp(TYPE_OFFSET));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of types
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // base type index
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // types block
   unsigned TypeOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
   Record.clear();
   Record.push_back(TYPE_OFFSET);
   Record.push_back(TypeOffsets.size());
+  Record.push_back(FirstTypeID - NUM_PREDEF_TYPE_IDS);
   Stream.EmitRecordWithBlob(TypeOffsetAbbrev, Record, data(TypeOffsets));
 
   // Write the declaration offsets array
@@ -2907,29 +2909,27 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
     WriteStatCache(*StatCalls);
   WriteSourceManagerBlock(Context.getSourceManager(), PP, isysroot);
   
-  // Write the record of special types.
-  Record.clear();
-
-  AddTypeRef(Context.getBuiltinVaListType(), Record);
-  AddTypeRef(Context.getObjCIdType(), Record);
-  AddTypeRef(Context.getObjCSelType(), Record);
-  AddTypeRef(Context.getObjCProtoType(), Record);
-  AddTypeRef(Context.getObjCClassType(), Record);
-  AddTypeRef(Context.getRawCFConstantStringType(), Record);
-  AddTypeRef(Context.getRawObjCFastEnumerationStateType(), Record);
-  AddTypeRef(Context.getFILEType(), Record);
-  AddTypeRef(Context.getjmp_bufType(), Record);
-  AddTypeRef(Context.getsigjmp_bufType(), Record);
-  AddTypeRef(Context.ObjCIdRedefinitionType, Record);
-  AddTypeRef(Context.ObjCClassRedefinitionType, Record);
-  AddTypeRef(Context.getRawBlockdescriptorType(), Record);
-  AddTypeRef(Context.getRawBlockdescriptorExtendedType(), Record);
-  AddTypeRef(Context.ObjCSelRedefinitionType, Record);
-  AddTypeRef(Context.getRawNSConstantStringType(), Record);
-  Record.push_back(Context.isInt128Installed());
-  AddTypeRef(Context.AutoDeductTy, Record);
-  AddTypeRef(Context.AutoRRefDeductTy, Record);
-  Stream.EmitRecord(SPECIAL_TYPES, Record);
+  // Form the record of special types.
+  RecordData SpecialTypes;
+  AddTypeRef(Context.getBuiltinVaListType(), SpecialTypes);
+  AddTypeRef(Context.getObjCIdType(), SpecialTypes);
+  AddTypeRef(Context.getObjCSelType(), SpecialTypes);
+  AddTypeRef(Context.getObjCProtoType(), SpecialTypes);
+  AddTypeRef(Context.getObjCClassType(), SpecialTypes);
+  AddTypeRef(Context.getRawCFConstantStringType(), SpecialTypes);
+  AddTypeRef(Context.getRawObjCFastEnumerationStateType(), SpecialTypes);
+  AddTypeRef(Context.getFILEType(), SpecialTypes);
+  AddTypeRef(Context.getjmp_bufType(), SpecialTypes);
+  AddTypeRef(Context.getsigjmp_bufType(), SpecialTypes);
+  AddTypeRef(Context.ObjCIdRedefinitionType, SpecialTypes);
+  AddTypeRef(Context.ObjCClassRedefinitionType, SpecialTypes);
+  AddTypeRef(Context.getRawBlockdescriptorType(), SpecialTypes);
+  AddTypeRef(Context.getRawBlockdescriptorExtendedType(), SpecialTypes);
+  AddTypeRef(Context.ObjCSelRedefinitionType, SpecialTypes);
+  AddTypeRef(Context.getRawNSConstantStringType(), SpecialTypes);
+  SpecialTypes.push_back(Context.isInt128Installed());
+  AddTypeRef(Context.AutoDeductTy, SpecialTypes);
+  AddTypeRef(Context.AutoRRefDeductTy, SpecialTypes);
 
   // Keep writing types and declarations until all types and
   // declarations have been written.
@@ -2958,6 +2958,8 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
 
   WriteCXXBaseSpecifiersOffsets();
   
+  Stream.EmitRecord(SPECIAL_TYPES, SpecialTypes);
+
   // Write the record containing external, unnamed definitions.
   if (!ExternalDefinitions.empty())
     Stream.EmitRecord(EXTERNAL_DEFINITIONS, ExternalDefinitions);
@@ -3072,7 +3074,7 @@ void ASTWriter::WriteASTChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
       io::Emit32(Out, (*M)->BaseSelectorID);
       io::Emit32(Out, (*M)->BaseDeclID);
       io::Emit32(Out, (*M)->BaseCXXBaseSpecifiersID);
-      io::Emit32(Out, (*M)->BaseTypeID);
+      io::Emit32(Out, (*M)->GlobalBaseTypeIndex);
     }
   }
   Record.clear();