[PDB] Don't build the entire source file list up front.

I tried to run llvm-pdbdump on a very large (~1.5GB) PDB to
try and identify show-stopping performance problems.  This
patch addresses the first such problem.

When loading the DBI stream, before anyone has even tried to
access a single record, we build an in memory map of every
source file for every module.  In the particular PDB I was
using, this was over 85 million files.  Specifically, the
complexity is O(m*n) where m is the number of modules and
n is the average number of source files (including headers)
per module.

The whole reason for doing this was so that we could have
constant time access to any module and any of its source
file lists.  However, we can still get O(1) access to the
source file list for a given module with a simple O(m)
precomputation, and access to the list of modules is
already O(1) anyway.

So this patches reduces the O(m*n) up-front precomputation
to an O(m) one, where n is ~6,500 and n*m is about 85 million
in my pathological test case.

Differential Revision: https://reviews.llvm.org/D32870

llvm-svn: 302205

3 files changed, 45 insertions, 36 deletions

diff --git a/llvm/tools/llvm-pdbdump/LLVMOutputStyle.cpp b/llvm/tools/llvm-pdbdump/LLVMOutputStyle.cpp
index 06d8ced128a..1af62b4e8df 100644
--- a/llvm/tools/llvm-pdbdump/LLVMOutputStyle.cpp
+++ b/llvm/tools/llvm-pdbdump/LLVMOutputStyle.cpp
@@ -775,37 +775,37 @@ Error LLVMOutputStyle::dumpDbiStream() {
 
   if (DumpModules) {
     ListScope L(P, "Modules");
-    for (auto &Modi : DS->modules()) {
+    const DbiModuleList &Modules = DS->modules();
+    for (uint32_t I = 0; I < Modules.getModuleCount(); ++I) {
+      const DbiModuleDescriptor &Modi = Modules.getModuleDescriptor(I);
       DictScope DD(P);
-      P.printString("Name", Modi.Info.getModuleName().str());
-      P.printNumber("Debug Stream Index", Modi.Info.getModuleStreamIndex());
-      P.printString("Object File Name", Modi.Info.getObjFileName().str());
-      P.printNumber("Num Files", Modi.Info.getNumberOfFiles());
-      P.printNumber("Source File Name Idx", Modi.Info.getSourceFileNameIndex());
-      P.printNumber("Pdb File Name Idx", Modi.Info.getPdbFilePathNameIndex());
-      P.printNumber("Line Info Byte Size", Modi.Info.getC11LineInfoByteSize());
-      P.printNumber("C13 Line Info Byte Size",
-                    Modi.Info.getC13LineInfoByteSize());
-      P.printNumber("Symbol Byte Size", Modi.Info.getSymbolDebugInfoByteSize());
-      P.printNumber("Type Server Index", Modi.Info.getTypeServerIndex());
-      P.printBoolean("Has EC Info", Modi.Info.hasECInfo());
+      P.printString("Name", Modi.getModuleName().str());
+      P.printNumber("Debug Stream Index", Modi.getModuleStreamIndex());
+      P.printString("Object File Name", Modi.getObjFileName().str());
+      P.printNumber("Num Files", Modi.getNumberOfFiles());
+      P.printNumber("Source File Name Idx", Modi.getSourceFileNameIndex());
+      P.printNumber("Pdb File Name Idx", Modi.getPdbFilePathNameIndex());
+      P.printNumber("Line Info Byte Size", Modi.getC11LineInfoByteSize());
+      P.printNumber("C13 Line Info Byte Size", Modi.getC13LineInfoByteSize());
+      P.printNumber("Symbol Byte Size", Modi.getSymbolDebugInfoByteSize());
+      P.printNumber("Type Server Index", Modi.getTypeServerIndex());
+      P.printBoolean("Has EC Info", Modi.hasECInfo());
       if (opts::raw::DumpModuleFiles) {
-        std::string FileListName =
-            to_string(Modi.SourceFiles.size()) + " Contributing Source Files";
+        std::string FileListName = to_string(Modules.getSourceFileCount(I)) +
+                                   " Contributing Source Files";
         ListScope LL(P, FileListName);
-        for (auto File : Modi.SourceFiles)
-          P.printString(File.str());
+        for (auto File : Modules.source_files(I))
+          P.printString(File);
       }
-      bool HasModuleDI =
-          (Modi.Info.getModuleStreamIndex() < File.getNumStreams());
+      bool HasModuleDI = (Modi.getModuleStreamIndex() < File.getNumStreams());
       bool ShouldDumpSymbols =
           (opts::raw::DumpModuleSyms || opts::raw::DumpSymRecordBytes);
       if (HasModuleDI && (ShouldDumpSymbols || opts::raw::DumpLineInfo)) {
         auto ModStreamData = MappedBlockStream::createIndexedStream(
             File.getMsfLayout(), File.getMsfBuffer(),
-            Modi.Info.getModuleStreamIndex());
+            Modi.getModuleStreamIndex());
 
-        ModuleDebugStreamRef ModS(Modi.Info, std::move(ModStreamData));
+        ModuleDebugStreamRef ModS(Modi, std::move(ModStreamData));
         if (auto EC = ModS.reload())
           return EC;
 
@@ -876,9 +876,10 @@ Error LLVMOutputStyle::dumpSectionContribs() {
       {
         DictScope DD(P, "Module");
         P.printNumber("Index", SC.Imod);
-        auto M = DS.modules();
-        if (M.size() > SC.Imod) {
-          P.printString("Name", M[SC.Imod].Info.getModuleName());
+        const DbiModuleList &Modules = DS.modules();
+        if (Modules.getModuleCount() > SC.Imod) {
+          P.printString("Name",
+                        Modules.getModuleDescriptor(SC.Imod).getModuleName());
         }
       }
       P.printNumber("Data CRC", SC.DataCrc);
diff --git a/llvm/tools/llvm-pdbdump/StreamUtil.cpp b/llvm/tools/llvm-pdbdump/StreamUtil.cpp
index 6577702adac..81aa256b500 100644
--- a/llvm/tools/llvm-pdbdump/StreamUtil.cpp
+++ b/llvm/tools/llvm-pdbdump/StreamUtil.cpp
@@ -12,6 +12,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h"
+#include "llvm/DebugInfo/PDB/Native/DbiModuleList.h"
 #include "llvm/DebugInfo/PDB/Native/DbiStream.h"
 #include "llvm/DebugInfo/PDB/Native/InfoStream.h"
 #include "llvm/DebugInfo/PDB/Native/PDBFile.h"
@@ -30,14 +31,16 @@ void discoverStreamPurposes(PDBFile &File,
   auto Info = File.getPDBInfoStream();
 
   uint32_t StreamCount = File.getNumStreams();
-  DenseMap<uint16_t, const ModuleInfoEx *> ModStreams;
+  DenseMap<uint16_t, DbiModuleDescriptor> ModStreams;
   DenseMap<uint16_t, std::string> NamedStreams;
 
   if (Dbi) {
-    for (auto &ModI : Dbi->modules()) {
-      uint16_t SN = ModI.Info.getModuleStreamIndex();
+    const DbiModuleList &Modules = Dbi->modules();
+    for (uint32_t I = 0; I < Modules.getModuleCount(); ++I) {
+      DbiModuleDescriptor Descriptor = Modules.getModuleDescriptor(I);
+      uint16_t SN = Descriptor.getModuleStreamIndex();
       if (SN != kInvalidStreamIndex)
-        ModStreams[SN] = &ModI;
+        ModStreams[SN] = Descriptor;
     }
   }
   if (Info) {
@@ -109,7 +112,7 @@ void discoverStreamPurposes(PDBFile &File,
       auto NSIter = NamedStreams.find(StreamIdx);
       if (ModIter != ModStreams.end()) {
         Value = "Module \"";
-        Value += ModIter->second->Info.getModuleName().str();
+        Value += ModIter->second.getModuleName();
         Value += "\"";
       } else if (NSIter != NamedStreams.end()) {
         Value = "Named Stream \"";
diff --git a/llvm/tools/llvm-pdbdump/YAMLOutputStyle.cpp b/llvm/tools/llvm-pdbdump/YAMLOutputStyle.cpp
index b94b5a4abf3..0573b23cdc7 100644
--- a/llvm/tools/llvm-pdbdump/YAMLOutputStyle.cpp
+++ b/llvm/tools/llvm-pdbdump/YAMLOutputStyle.cpp
@@ -305,23 +305,28 @@ Error YAMLOutputStyle::dumpDbiStream() {
   Obj.DbiStream->PdbDllVersion = DS.getPdbDllVersion();
   Obj.DbiStream->VerHeader = DS.getDbiVersion();
   if (opts::pdb2yaml::DbiModuleInfo) {
-    for (const auto &MI : DS.modules()) {
+    const auto &Modules = DS.modules();
+    for (uint32_t I = 0; I < Modules.getModuleCount(); ++I) {
+      DbiModuleDescriptor MI = Modules.getModuleDescriptor(I);
+
       Obj.DbiStream->ModInfos.emplace_back();
       yaml::PdbDbiModuleInfo &DMI = Obj.DbiStream->ModInfos.back();
 
-      DMI.Mod = MI.Info.getModuleName();
-      DMI.Obj = MI.Info.getObjFileName();
-      if (opts::pdb2yaml::DbiModuleSourceFileInfo)
-        DMI.SourceFiles = MI.SourceFiles;
+      DMI.Mod = MI.getModuleName();
+      DMI.Obj = MI.getObjFileName();
+      if (opts::pdb2yaml::DbiModuleSourceFileInfo) {
+        auto Files = Modules.source_files(I);
+        DMI.SourceFiles.assign(Files.begin(), Files.end());
+      }
 
-      uint16_t ModiStream = MI.Info.getModuleStreamIndex();
+      uint16_t ModiStream = MI.getModuleStreamIndex();
       if (ModiStream == kInvalidStreamIndex)
         continue;
 
       auto ModStreamData = msf::MappedBlockStream::createIndexedStream(
           File.getMsfLayout(), File.getMsfBuffer(), ModiStream);
 
-      pdb::ModuleDebugStreamRef ModS(MI.Info, std::move(ModStreamData));
+      pdb::ModuleDebugStreamRef ModS(MI, std::move(ModStreamData));
       if (auto EC = ModS.reload())
         return EC;