[LLD][COFF] Early dependency detection

We introduce a new class hierarchy for debug types merging (in DebugTypes.h). The end-goal is to parallelize the type merging - please see the plan in D59226.

Previously, dependency discovery was done on the fly, much later, during the type merging loop. Unfortunately, parallelizing the type merging requires the dependencies to be merged in first, before any dependent ObjFile, thus this early discovery.

The overall intention for this path is to discover debug information dependencies at a much earlier stage, when processing input files. Currently, two types of dependency are supported: PDB type servers (when compiling with MSVC /Zi) and precompiled headers OBJs (when compiling with MSVC /Yc and /Yu). Once discovered, an explicit link is added into the dependent ObjFile, through the new debug types class hierarchy introduced in DebugTypes.h.

Differential Revision: https://reviews.llvm.org/D59053

llvm-svn: 357383

6 files changed, 238 insertions, 59 deletions

diff --git a/lld/COFF/CMakeLists.txt b/lld/COFF/CMakeLists.txt
index bb241e788c1..c7ef7c47fea 100644
--- a/lld/COFF/CMakeLists.txt
+++ b/lld/COFF/CMakeLists.txt
@@ -8,6 +8,7 @@ endif()
 
 add_lld_library(lldCOFF
   Chunks.cpp
+  DebugTypes.cpp
   DLL.cpp
   Driver.cpp
   DriverUtils.cpp
diff --git a/lld/COFF/DebugTypes.cpp b/lld/COFF/DebugTypes.cpp
new file mode 100644
index 00000000000..b815665875f
--- /dev/null
+++ b/lld/COFF/DebugTypes.cpp
@@ -0,0 +1,84 @@
+//===- DebugTypes.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DebugTypes.h"
+#include "InputFiles.h"
+#include "llvm/DebugInfo/CodeView/TypeRecord.h"
+
+using namespace lld;
+using namespace lld::coff;
+using namespace llvm;
+using namespace llvm::codeview;
+
+namespace {
+class TypeServerSource : public TpiSource {
+public:
+  TypeServerSource(ObjFile *F) : TpiSource(PDB, F) {}
+};
+
+class UseTypeServerSource : public TpiSource {
+public:
+  UseTypeServerSource(ObjFile *F, TypeServer2Record *TS)
+      : TpiSource(UsingPDB, F), TypeServerDependency(*TS) {}
+
+  // Information about the PDB type server dependency, that needs to be loaded
+  // in before merging this OBJ.
+  TypeServer2Record TypeServerDependency;
+};
+
+class PrecompSource : public TpiSource {
+public:
+  PrecompSource(ObjFile *F) : TpiSource(PCH, F) {}
+};
+
+class UsePrecompSource : public TpiSource {
+public:
+  UsePrecompSource(ObjFile *F, PrecompRecord *Precomp)
+      : TpiSource(UsingPCH, F), PrecompDependency(*Precomp) {}
+
+  // Information about the Precomp OBJ dependency, that needs to be loaded in
+  // before merging this OBJ.
+  PrecompRecord PrecompDependency;
+};
+} // namespace
+
+static std::vector<std::unique_ptr<TpiSource>> GC;
+
+TpiSource::TpiSource(TpiKind K, ObjFile *F) : Kind(K), File(F) {
+  GC.push_back(std::unique_ptr<TpiSource>(this));
+}
+
+TpiSource *coff::makeTpiSource(ObjFile *F) {
+  return new TpiSource(TpiSource::Regular, F);
+}
+
+TpiSource *coff::makeTypeServerSource(ObjFile *F) {
+  return new TypeServerSource(F);
+}
+
+TpiSource *coff::makeUseTypeServerSource(ObjFile *F, TypeServer2Record *TS) {
+  return new UseTypeServerSource(F, TS);
+}
+
+TpiSource *coff::makePrecompSource(ObjFile *F) { return new PrecompSource(F); }
+
+TpiSource *coff::makeUsePrecompSource(ObjFile *F, PrecompRecord *Precomp) {
+  return new UsePrecompSource(F, Precomp);
+}
+
+template <>
+const PrecompRecord &coff::retrieveDependencyInfo(TpiSource *Source) {
+  assert(Source->Kind == TpiSource::UsingPCH);
+  return ((UsePrecompSource *)Source)->PrecompDependency;
+}
+
+template <>
+const TypeServer2Record &coff::retrieveDependencyInfo(TpiSource *Source) {
+  assert(Source->Kind == TpiSource::UsingPDB);
+  return ((UseTypeServerSource *)Source)->TypeServerDependency;
+}
\ No newline at end of file
diff --git a/lld/COFF/DebugTypes.h b/lld/COFF/DebugTypes.h
new file mode 100644
index 00000000000..0505a354257
--- /dev/null
+++ b/lld/COFF/DebugTypes.h
@@ -0,0 +1,51 @@
+//===- DebugTypes.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_COFF_DEBUGTYPES_H
+#define LLD_COFF_DEBUGTYPES_H
+
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace codeview {
+class PrecompRecord;
+class TypeServer2Record;
+} // namespace codeview
+} // namespace llvm
+
+namespace lld {
+namespace coff {
+
+class ObjFile;
+
+class TpiSource {
+public:
+  enum TpiKind { Regular, PCH, UsingPCH, PDB, UsingPDB };
+
+  TpiSource(TpiKind K, ObjFile *F);
+  virtual ~TpiSource() {}
+
+  const TpiKind Kind;
+  ObjFile *File;
+};
+
+TpiSource *makeTpiSource(ObjFile *F);
+TpiSource *makeTypeServerSource(ObjFile *F);
+TpiSource *makeUseTypeServerSource(ObjFile *F,
+                                   llvm::codeview::TypeServer2Record *TS);
+TpiSource *makePrecompSource(ObjFile *F);
+TpiSource *makeUsePrecompSource(ObjFile *F,
+                                llvm::codeview::PrecompRecord *Precomp);
+
+// Temporary interface to get the dependency
+template <typename T> const T &retrieveDependencyInfo(TpiSource *Source);
+
+} // namespace coff
+} // namespace lld
+
+#endif
\ No newline at end of file
diff --git a/lld/COFF/InputFiles.cpp b/lld/COFF/InputFiles.cpp
index 10ebeab94f5..3c7f4b418a6 100644
--- a/lld/COFF/InputFiles.cpp
+++ b/lld/COFF/InputFiles.cpp
@@ -9,6 +9,7 @@
 #include "InputFiles.h"
 #include "Chunks.h"
 #include "Config.h"
+#include "DebugTypes.h"
 #include "Driver.h"
 #include "SymbolTable.h"
 #include "Symbols.h"
@@ -22,6 +23,7 @@
 #include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h"
 #include "llvm/DebugInfo/CodeView/SymbolDeserializer.h"
 #include "llvm/DebugInfo/CodeView/SymbolRecord.h"
+#include "llvm/DebugInfo/CodeView/TypeDeserializer.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/COFF.h"
 #include "llvm/Support/Casting.h"
@@ -129,6 +131,7 @@ void ObjFile::parse() {
   initializeChunks();
   initializeSymbols();
   initializeFlags();
+  initializeDependencies();
 }
 
 const coff_section* ObjFile::getSection(uint32_t I) {
@@ -656,6 +659,59 @@ void ObjFile::initializeFlags() {
   }
 }
 
+// Depending on the compilation flags, OBJs can refer to external files,
+// necessary to merge this OBJ into the final PDB. We currently support two
+// types of external files: Precomp/PCH OBJs, when compiling with /Yc and /Yu.
+// And PDB type servers, when compiling with /Zi. This function extracts these
+// dependencies and makes them available as a TpiSource interface (see
+// DebugTypes.h).
+void ObjFile::initializeDependencies() {
+  if (!Config->Debug)
+    return;
+
+  bool IsPCH = false;
+
+  ArrayRef<uint8_t> Data = getDebugSection(".debug$P");
+  if (!Data.empty())
+    IsPCH = true;
+  else
+    Data = getDebugSection(".debug$T");
+
+  if (Data.empty())
+    return;
+
+  CVTypeArray Types;
+  BinaryStreamReader Reader(Data, support::little);
+  cantFail(Reader.readArray(Types, Reader.getLength()));
+
+  CVTypeArray::Iterator FirstType = Types.begin();
+  if (FirstType == Types.end())
+    return;
+
+  DebugTypes.emplace(Types);
+
+  if (IsPCH) {
+    DebugTypesObj = makePrecompSource(this);
+    return;
+  }
+
+  if (FirstType->kind() == LF_TYPESERVER2) {
+    TypeServer2Record TS = cantFail(
+        TypeDeserializer::deserializeAs<TypeServer2Record>(FirstType->data()));
+    DebugTypesObj = makeUseTypeServerSource(this, &TS);
+    return;
+  }
+
+  if (FirstType->kind() == LF_PRECOMP) {
+    PrecompRecord Precomp = cantFail(
+        TypeDeserializer::deserializeAs<PrecompRecord>(FirstType->data()));
+    DebugTypesObj = makeUsePrecompSource(this, &Precomp);
+    return;
+  }
+
+  DebugTypesObj = makeTpiSource(this);
+}
+
 StringRef ltrim1(StringRef S, const char *Chars) {
   if (!S.empty() && strchr(Chars, S[0]))
     return S.substr(1);
diff --git a/lld/COFF/InputFiles.h b/lld/COFF/InputFiles.h
index 27e1025675b..8a81ea114f6 100644
--- a/lld/COFF/InputFiles.h
+++ b/lld/COFF/InputFiles.h
@@ -51,6 +51,7 @@ class Lazy;
 class SectionChunk;
 class Symbol;
 class Undefined;
+class TpiSource;
 
 // The root class of input files.
 class InputFile {
@@ -167,6 +168,12 @@ public:
   // Whether the object was already merged into the final PDB or not
   bool MergedIntoPDB = false;
 
+  // If the OBJ has a .debug$T stream, this tells how it will be handled.
+  TpiSource *DebugTypesObj = nullptr;
+
+  // The .debug$T stream if there's one.
+  llvm::Optional<llvm::codeview::CVTypeArray> DebugTypes;
+
 private:
   const coff_section* getSection(uint32_t I);
   const coff_section *getSection(COFFSymbolRef Sym) {
@@ -176,6 +183,7 @@ private:
   void initializeChunks();
   void initializeSymbols();
   void initializeFlags();
+  void initializeDependencies();
 
   SectionChunk *
   readSection(uint32_t SectionNumber,
diff --git a/lld/COFF/PDB.cpp b/lld/COFF/PDB.cpp
index c27212d7514..9948399d5ae 100644
--- a/lld/COFF/PDB.cpp
+++ b/lld/COFF/PDB.cpp
@@ -9,6 +9,7 @@
 #include "PDB.h"
 #include "Chunks.h"
 #include "Config.h"
+#include "DebugTypes.h"
 #include "Driver.h"
 #include "SymbolTable.h"
 #include "Symbols.h"
@@ -132,15 +133,12 @@ public:
                                            CVIndexMap *ObjectIndexMap);
 
   /// Reads and makes available a PDB.
-  Expected<const CVIndexMap &> maybeMergeTypeServerPDB(ObjFile *File,
-                                                       const CVType &FirstType);
+  Expected<const CVIndexMap &> maybeMergeTypeServerPDB(ObjFile *File);
 
   /// Merges a precompiled headers TPI map into the current TPI map. The
   /// precompiled headers object will also be loaded and remapped in the
   /// process.
-  Expected<const CVIndexMap &>
-  mergeInPrecompHeaderObj(ObjFile *File, const CVType &FirstType,
-                          CVIndexMap *ObjectIndexMap);
+  Error mergeInPrecompHeaderObj(ObjFile *File, CVIndexMap *ObjectIndexMap);
 
   /// Reads and makes available a precompiled headers object.
   ///
@@ -151,8 +149,7 @@ public:
   ///
   /// If the precompiled headers object was already loaded, this function will
   /// simply return its (remapped) TPI map.
-  Expected<const CVIndexMap &> aquirePrecompObj(ObjFile *File,
-                                                PrecompRecord Precomp);
+  Expected<const CVIndexMap &> aquirePrecompObj(ObjFile *File);
 
   /// Adds a precompiled headers object signature -> TPI mapping.
   std::pair<CVIndexMap &, bool /*already there*/>
@@ -373,21 +370,12 @@ Expected<const CVIndexMap &>
 PDBLinker::mergeDebugT(ObjFile *File, CVIndexMap *ObjectIndexMap) {
   ScopedTimer T(TypeMergingTimer);
 
-  bool IsPrecompiledHeader = false;
-
-  ArrayRef<uint8_t> Data = File->getDebugSection(".debug$T");
-  if (Data.empty()) {
-    // Try again, Microsoft precompiled headers use .debug$P instead of
-    // .debug$T
-    Data = File->getDebugSection(".debug$P");
-    IsPrecompiledHeader = true;
-  }
-  if (Data.empty())
-    return *ObjectIndexMap; // no debug info
+  if (!File->DebugTypesObj)
+      return *ObjectIndexMap; // no Types stream
 
   // Precompiled headers objects need to save the index map for further
   // reference by other objects which use the precompiled headers.
-  if (IsPrecompiledHeader) {
+  if (File->DebugTypesObj->Kind == TpiSource::PCH) {
     uint32_t PCHSignature = File->PCHSignature.getValueOr(0);
     if (PCHSignature == 0)
       fatal("No signature found for the precompiled headers OBJ (" +
@@ -409,33 +397,28 @@ PDBLinker::mergeDebugT(ObjFile *File, CVIndexMap *ObjectIndexMap) {
     }
   }
 
-  BinaryByteStream Stream(Data, support::little);
-  CVTypeArray Types;
-  BinaryStreamReader Reader(Stream);
-  if (auto EC = Reader.readArray(Types, Reader.getLength()))
-    fatal("Reader::readArray failed: " + toString(std::move(EC)));
-
-  auto FirstType = Types.begin();
-  if (FirstType == Types.end())
-    return *ObjectIndexMap;
-
-  if (FirstType->kind() == LF_TYPESERVER2) {
+  if (File->DebugTypesObj->Kind == TpiSource::UsingPDB) {
     // Look through type servers. If we've already seen this type server,
     // don't merge any type information.
-    return maybeMergeTypeServerPDB(File, *FirstType);
-  } else if (FirstType->kind() == LF_PRECOMP) {
+    return maybeMergeTypeServerPDB(File);
+  }
+  
+  CVTypeArray &Types = *File->DebugTypes;
+
+  if (File->DebugTypesObj->Kind == TpiSource::UsingPCH) {
     // This object was compiled with /Yu, so process the corresponding
     // precompiled headers object (/Yc) first. Some type indices in the current
     // object are referencing data in the precompiled headers object, so we need
     // both to be loaded.
-    auto E = mergeInPrecompHeaderObj(File, *FirstType, ObjectIndexMap);
-    if (!E)
-      return E.takeError();
-
-    // Drop LF_PRECOMP record from the input stream, as it needs to be replaced
-    // with the precompiled headers object type stream.
-    // Note that we can't just call Types.drop_front(), as we explicitly want to
-    // rebase the stream.
+    Error E = mergeInPrecompHeaderObj(File, ObjectIndexMap);
+    if (E)
+      return std::move(E);
+
+    // Drop LF_PRECOMP record from the input stream, as it has been replaced
+    // with the precompiled headers Type stream in the mergeInPrecompHeaderObj()
+    // call above. Note that we can't just call Types.drop_front(), as we
+    // explicitly want to rebase the stream.
+    CVTypeArray::Iterator FirstType = Types.begin();
     Types.setUnderlyingStream(
         Types.getUnderlyingStream().drop_front(FirstType->RecordData.size()));
   }
@@ -503,12 +486,9 @@ tryToLoadPDB(const codeview::GUID &GuidFromObj, StringRef TSPath) {
   return std::move(NS);
 }
 
-Expected<const CVIndexMap &>
-PDBLinker::maybeMergeTypeServerPDB(ObjFile *File, const CVType &FirstType) {
-  TypeServer2Record TS;
-  if (auto EC =
-          TypeDeserializer::deserializeAs(const_cast<CVType &>(FirstType), TS))
-    fatal("error reading record: " + toString(std::move(EC)));
+Expected<const CVIndexMap &> PDBLinker::maybeMergeTypeServerPDB(ObjFile *File) {
+  const TypeServer2Record &TS =
+      retrieveDependencyInfo<TypeServer2Record>(File->DebugTypesObj);
 
   const codeview::GUID &TSId = TS.getGuid();
   StringRef TSPath = TS.getName();
@@ -617,15 +597,12 @@ PDBLinker::maybeMergeTypeServerPDB(ObjFile *File, const CVType &FirstType) {
   return IndexMap;
 }
 
-Expected<const CVIndexMap &>
-PDBLinker::mergeInPrecompHeaderObj(ObjFile *File, const CVType &FirstType,
-                                   CVIndexMap *ObjectIndexMap) {
-  PrecompRecord Precomp;
-  if (auto EC = TypeDeserializer::deserializeAs(const_cast<CVType &>(FirstType),
-                                                Precomp))
-    fatal("error reading record: " + toString(std::move(EC)));
-
-  auto E = aquirePrecompObj(File, Precomp);
+Error PDBLinker::mergeInPrecompHeaderObj(ObjFile *File,
+                                         CVIndexMap *ObjectIndexMap) {
+  const PrecompRecord &Precomp =
+      retrieveDependencyInfo<PrecompRecord>(File->DebugTypesObj);
+
+  Expected<const CVIndexMap &> E = aquirePrecompObj(File);
   if (!E)
     return E.takeError();
 
@@ -633,7 +610,7 @@ PDBLinker::mergeInPrecompHeaderObj(ObjFile *File, const CVType &FirstType,
   assert(PrecompIndexMap.IsPrecompiledTypeMap);
 
   if (PrecompIndexMap.TPIMap.empty())
-    return PrecompIndexMap;
+    return Error::success();
 
   assert(Precomp.getStartTypeIndex() == TypeIndex::FirstNonSimpleIndex);
   assert(Precomp.getTypesCount() <= PrecompIndexMap.TPIMap.size());
@@ -641,7 +618,7 @@ PDBLinker::mergeInPrecompHeaderObj(ObjFile *File, const CVType &FirstType,
   ObjectIndexMap->TPIMap.append(PrecompIndexMap.TPIMap.begin(),
                                 PrecompIndexMap.TPIMap.begin() +
                                     Precomp.getTypesCount());
-  return *ObjectIndexMap;
+  return Error::success();
 }
 
 static bool equals_path(StringRef path1, StringRef path2) {
@@ -677,8 +654,10 @@ PDBLinker::registerPrecompiledHeaders(uint32_t Signature) {
   return {IndexMap, false};
 }
 
-Expected<const CVIndexMap &>
-PDBLinker::aquirePrecompObj(ObjFile *File, PrecompRecord Precomp) {
+Expected<const CVIndexMap &> PDBLinker::aquirePrecompObj(ObjFile *File) {
+  const PrecompRecord &Precomp =
+      retrieveDependencyInfo<PrecompRecord>(File->DebugTypesObj);
+
   // First, check if we already loaded the precompiled headers object with this
   // signature. Return the type index mapping if we've already seen it.
   auto R = registerPrecompiledHeaders(Precomp.getSignature());