summaryrefslogtreecommitdiffstats
path: root/llvm/lib/Bitcode
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Bitcode')
-rw-r--r--llvm/lib/Bitcode/Reader/BitcodeReader.cpp461
-rw-r--r--llvm/lib/Bitcode/Writer/BitcodeWriter.cpp465
-rw-r--r--llvm/lib/Bitcode/Writer/LLVMBuild.txt2
3 files changed, 716 insertions, 212 deletions
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 06a120a5ff1..4b1ccf85231 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -416,7 +416,7 @@ private:
class FunctionIndexBitcodeReader {
DiagnosticHandlerFunction DiagnosticHandler;
- /// Eventually points to the function index built during parsing.
+ /// Eventually points to the module index built during parsing.
FunctionInfoIndex *TheIndex = nullptr;
std::unique_ptr<MemoryBuffer> Buffer;
@@ -432,29 +432,37 @@ class FunctionIndexBitcodeReader {
bool IsLazy = false;
/// Used to indicate whether caller only wants to check for the presence
- /// of the function summary bitcode section. All blocks are skipped,
- /// but the SeenFuncSummary boolean is set.
- bool CheckFuncSummaryPresenceOnly = false;
+ /// of the global value summary bitcode section. All blocks are skipped,
+ /// but the SeenGlobalValSummary boolean is set.
+ bool CheckGlobalValSummaryPresenceOnly = false;
- /// Indicates whether we have encountered a function summary section
- /// yet during parsing, used when checking if file contains function
+ /// Indicates whether we have encountered a global value summary section
+ /// yet during parsing, used when checking if file contains global value
/// summary section.
- bool SeenFuncSummary = false;
+ bool SeenGlobalValSummary = false;
- /// \brief Map populated during function summary section parsing, and
- /// consumed during ValueSymbolTable parsing.
- ///
- /// Used to correlate summary records with VST entries. For the per-module
- /// index this maps the ValueID to the parsed function summary, and
- /// for the combined index this maps the summary record's bitcode
- /// offset to the function summary (since in the combined index the
- /// VST records do not hold value IDs but rather hold the function
- /// summary record offset).
- DenseMap<uint64_t, std::unique_ptr<FunctionSummary>> SummaryMap;
+ /// Indicates whether we have already parsed the VST, used for error checking.
+ bool SeenValueSymbolTable = false;
+
+ /// Set to the offset of the VST recorded in the MODULE_CODE_VSTOFFSET record.
+ /// Used to enable on-demand parsing of the VST.
+ uint64_t VSTOffset = 0;
+
+ // Map to save ValueId to GUID association that was recorded in the
+ // ValueSymbolTable. It is used after the VST is parsed to convert
+ // call graph edges read from the function summary from referencing
+ // callees by their ValueId to using the GUID instead, which is how
+ // they are recorded in the function index being built.
+ DenseMap<unsigned, uint64_t> ValueIdToCallGraphGUIDMap;
+
+ /// Map to save the association between summary offset in the VST to the
+ /// GlobalValueInfo object created when parsing it. Used to access the
+ /// info object when parsing the summary section.
+ DenseMap<uint64_t, GlobalValueInfo *> SummaryOffsetToInfoMap;
/// Map populated during module path string table parsing, from the
/// module ID to a string reference owned by the index's module
- /// path string table, used to correlate with combined index function
+ /// path string table, used to correlate with combined index
/// summary records.
DenseMap<uint64_t, StringRef> ModuleIdMap;
@@ -469,37 +477,41 @@ public:
FunctionIndexBitcodeReader(MemoryBuffer *Buffer,
DiagnosticHandlerFunction DiagnosticHandler,
bool IsLazy = false,
- bool CheckFuncSummaryPresenceOnly = false);
+ bool CheckGlobalValSummaryPresenceOnly = false);
FunctionIndexBitcodeReader(DiagnosticHandlerFunction DiagnosticHandler,
bool IsLazy = false,
- bool CheckFuncSummaryPresenceOnly = false);
+ bool CheckGlobalValSummaryPresenceOnly = false);
~FunctionIndexBitcodeReader() { freeState(); }
void freeState();
void releaseBuffer();
- /// Check if the parser has encountered a function summary section.
- bool foundFuncSummary() { return SeenFuncSummary; }
+ /// Check if the parser has encountered a summary section.
+ bool foundGlobalValSummary() { return SeenGlobalValSummary; }
/// \brief Main interface to parsing a bitcode buffer.
/// \returns true if an error occurred.
std::error_code parseSummaryIndexInto(std::unique_ptr<DataStreamer> Streamer,
FunctionInfoIndex *I);
- /// \brief Interface for parsing a function summary lazily.
+ /// \brief Interface for parsing a summary lazily.
std::error_code parseFunctionSummary(std::unique_ptr<DataStreamer> Streamer,
FunctionInfoIndex *I,
size_t FunctionSummaryOffset);
private:
std::error_code parseModule();
- std::error_code parseValueSymbolTable();
+ std::error_code parseValueSymbolTable(
+ uint64_t Offset,
+ DenseMap<unsigned, GlobalValue::LinkageTypes> &ValueIdToLinkageMap);
std::error_code parseEntireSummary();
std::error_code parseModuleStringTable();
std::error_code initStream(std::unique_ptr<DataStreamer> Streamer);
std::error_code initStreamFromBuffer();
std::error_code initLazyStream(std::unique_ptr<DataStreamer> Streamer);
+ uint64_t getGUIDFromValueId(unsigned ValueId);
+ GlobalValueInfo *getInfoFromSummaryOffset(uint64_t Offset);
};
} // end anonymous namespace
@@ -1727,6 +1739,27 @@ ErrorOr<Value *> BitcodeReader::recordValue(SmallVectorImpl<uint64_t> &Record,
return V;
}
+/// Helper to note and return the current location, and jump to the given
+/// offset.
+static uint64_t jumpToValueSymbolTable(uint64_t Offset,
+ BitstreamCursor &Stream) {
+ // Save the current parsing location so we can jump back at the end
+ // of the VST read.
+ uint64_t CurrentBit = Stream.GetCurrentBitNo();
+ Stream.JumpToBit(Offset * 32);
+#ifndef NDEBUG
+ // Do some checking if we are in debug mode.
+ BitstreamEntry Entry = Stream.advance();
+ assert(Entry.Kind == BitstreamEntry::SubBlock);
+ assert(Entry.ID == bitc::VALUE_SYMTAB_BLOCK_ID);
+#else
+ // In NDEBUG mode ignore the output so we don't get an unused variable
+ // warning.
+ Stream.advance();
+#endif
+ return CurrentBit;
+}
+
/// Parse the value symbol table at either the current parsing location or
/// at the given bit offset if provided.
std::error_code BitcodeReader::parseValueSymbolTable(uint64_t Offset) {
@@ -1734,22 +1767,8 @@ std::error_code BitcodeReader::parseValueSymbolTable(uint64_t Offset) {
// Pass in the Offset to distinguish between calling for the module-level
// VST (where we want to jump to the VST offset) and the function-level
// VST (where we don't).
- if (Offset > 0) {
- // Save the current parsing location so we can jump back at the end
- // of the VST read.
- CurrentBit = Stream.GetCurrentBitNo();
- Stream.JumpToBit(Offset * 32);
-#ifndef NDEBUG
- // Do some checking if we are in debug mode.
- BitstreamEntry Entry = Stream.advance();
- assert(Entry.Kind == BitstreamEntry::SubBlock);
- assert(Entry.ID == bitc::VALUE_SYMTAB_BLOCK_ID);
-#else
- // In NDEBUG mode ignore the output so we don't get an unused variable
- // warning.
- Stream.advance();
-#endif
- }
+ if (Offset > 0)
+ CurrentBit = jumpToValueSymbolTable(Offset, Stream);
// Compute the delta between the bitcode indices in the VST (the word offset
// to the word-aligned ENTER_SUBBLOCK for the function block, and that
@@ -5411,27 +5430,45 @@ std::error_code FunctionIndexBitcodeReader::error(BitcodeError E) {
FunctionIndexBitcodeReader::FunctionIndexBitcodeReader(
MemoryBuffer *Buffer, DiagnosticHandlerFunction DiagnosticHandler,
- bool IsLazy, bool CheckFuncSummaryPresenceOnly)
+ bool IsLazy, bool CheckGlobalValSummaryPresenceOnly)
: DiagnosticHandler(DiagnosticHandler), Buffer(Buffer), IsLazy(IsLazy),
- CheckFuncSummaryPresenceOnly(CheckFuncSummaryPresenceOnly) {}
+ CheckGlobalValSummaryPresenceOnly(CheckGlobalValSummaryPresenceOnly) {}
FunctionIndexBitcodeReader::FunctionIndexBitcodeReader(
DiagnosticHandlerFunction DiagnosticHandler, bool IsLazy,
- bool CheckFuncSummaryPresenceOnly)
+ bool CheckGlobalValSummaryPresenceOnly)
: DiagnosticHandler(DiagnosticHandler), Buffer(nullptr), IsLazy(IsLazy),
- CheckFuncSummaryPresenceOnly(CheckFuncSummaryPresenceOnly) {}
+ CheckGlobalValSummaryPresenceOnly(CheckGlobalValSummaryPresenceOnly) {}
void FunctionIndexBitcodeReader::freeState() { Buffer = nullptr; }
void FunctionIndexBitcodeReader::releaseBuffer() { Buffer.release(); }
-// Specialized value symbol table parser used when reading function index
+uint64_t FunctionIndexBitcodeReader::getGUIDFromValueId(unsigned ValueId) {
+ auto VGI = ValueIdToCallGraphGUIDMap.find(ValueId);
+ assert(VGI != ValueIdToCallGraphGUIDMap.end());
+ return VGI->second;
+}
+
+GlobalValueInfo *
+FunctionIndexBitcodeReader::getInfoFromSummaryOffset(uint64_t Offset) {
+ auto I = SummaryOffsetToInfoMap.find(Offset);
+ assert(I != SummaryOffsetToInfoMap.end());
+ return I->second;
+}
+
+// Specialized value symbol table parser used when reading module index
// blocks where we don't actually create global values.
-// At the end of this routine the function index is populated with a map
-// from function name to FunctionInfo. The function info contains
-// the function block's bitcode offset as well as the offset into the
-// function summary section.
-std::error_code FunctionIndexBitcodeReader::parseValueSymbolTable() {
+// At the end of this routine the module index is populated with a map
+// from global value name to GlobalValueInfo. The global value info contains
+// the function block's bitcode offset (if applicable), or the offset into the
+// summary section for the combined index.
+std::error_code FunctionIndexBitcodeReader::parseValueSymbolTable(
+ uint64_t Offset,
+ DenseMap<unsigned, GlobalValue::LinkageTypes> &ValueIdToLinkageMap) {
+ assert(Offset > 0 && "Expected non-zero VST offset");
+ uint64_t CurrentBit = jumpToValueSymbolTable(Offset, Stream);
+
if (Stream.EnterSubBlock(bitc::VALUE_SYMTAB_BLOCK_ID))
return error("Invalid record");
@@ -5447,6 +5484,8 @@ std::error_code FunctionIndexBitcodeReader::parseValueSymbolTable() {
case BitstreamEntry::Error:
return error("Malformed block");
case BitstreamEntry::EndBlock:
+ // Done parsing VST, jump back to wherever we came from.
+ Stream.JumpToBit(CurrentBit);
return std::error_code();
case BitstreamEntry::Record:
// The interesting case.
@@ -5458,6 +5497,23 @@ std::error_code FunctionIndexBitcodeReader::parseValueSymbolTable() {
switch (Stream.readRecord(Entry.ID, Record)) {
default: // Default behavior: ignore (e.g. VST_CODE_BBENTRY records).
break;
+ case bitc::VST_CODE_ENTRY: { // VST_CODE_ENTRY: [valueid, namechar x N]
+ if (convertToString(Record, 1, ValueName))
+ return error("Invalid record");
+ unsigned ValueID = Record[0];
+ std::unique_ptr<GlobalValueInfo> GlobalValInfo =
+ llvm::make_unique<GlobalValueInfo>();
+ assert(!SourceFileName.empty());
+ auto VLI = ValueIdToLinkageMap.find(ValueID);
+ assert(VLI != ValueIdToLinkageMap.end() &&
+ "No linkage found for VST entry?");
+ std::string GlobalId =
+ Function::getGlobalIdentifier(ValueName, VLI->second, SourceFileName);
+ TheIndex->addGlobalValueInfo(GlobalId, std::move(GlobalValInfo));
+ ValueIdToCallGraphGUIDMap[ValueID] = Function::getGUID(GlobalId);
+ ValueName.clear();
+ break;
+ }
case bitc::VST_CODE_FNENTRY: {
// VST_CODE_FNENTRY: [valueid, offset, namechar x N]
if (convertToString(Record, 2, ValueName))
@@ -5465,59 +5521,58 @@ std::error_code FunctionIndexBitcodeReader::parseValueSymbolTable() {
unsigned ValueID = Record[0];
uint64_t FuncOffset = Record[1];
assert(!IsLazy && "Lazy summary read only supported for combined index");
- // Gracefully handle bitcode without a function summary section,
- // which will simply not populate the index.
- if (foundFuncSummary()) {
- DenseMap<uint64_t, std::unique_ptr<FunctionSummary>>::iterator SMI =
- SummaryMap.find(ValueID);
- assert(SMI != SummaryMap.end() && "Summary info not found");
- std::unique_ptr<FunctionInfo> FuncInfo =
- llvm::make_unique<FunctionInfo>(FuncOffset);
- FuncInfo->setFunctionSummary(std::move(SMI->second));
- assert(!SourceFileName.empty());
- std::string FunctionGlobalId = Function::getGlobalIdentifier(
- ValueName, FuncInfo->functionSummary()->getFunctionLinkage(),
- SourceFileName);
- TheIndex->addFunctionInfo(FunctionGlobalId, std::move(FuncInfo));
- }
+ std::unique_ptr<GlobalValueInfo> FuncInfo =
+ llvm::make_unique<GlobalValueInfo>(FuncOffset);
+ assert(!SourceFileName.empty());
+ auto VLI = ValueIdToLinkageMap.find(ValueID);
+ assert(VLI != ValueIdToLinkageMap.end() &&
+ "No linkage found for VST entry?");
+ std::string FunctionGlobalId =
+ Function::getGlobalIdentifier(ValueName, VLI->second, SourceFileName);
+ TheIndex->addGlobalValueInfo(FunctionGlobalId, std::move(FuncInfo));
+ ValueIdToCallGraphGUIDMap[ValueID] = Function::getGUID(FunctionGlobalId);
ValueName.clear();
break;
}
- case bitc::VST_CODE_COMBINED_FNENTRY: {
- // VST_CODE_COMBINED_FNENTRY: [offset, funcguid]
- uint64_t FuncSummaryOffset = Record[0];
- uint64_t FuncGUID = Record[1];
- std::unique_ptr<FunctionInfo> FuncInfo =
- llvm::make_unique<FunctionInfo>(FuncSummaryOffset);
- if (foundFuncSummary() && !IsLazy) {
- DenseMap<uint64_t, std::unique_ptr<FunctionSummary>>::iterator SMI =
- SummaryMap.find(FuncSummaryOffset);
- assert(SMI != SummaryMap.end() && "Summary info not found");
- FuncInfo->setFunctionSummary(std::move(SMI->second));
- }
- TheIndex->addFunctionInfo(FuncGUID, std::move(FuncInfo));
-
- ValueName.clear();
+ case bitc::VST_CODE_COMBINED_GVDEFENTRY: {
+ // VST_CODE_COMBINED_GVDEFENTRY: [valueid, offset, guid]
+ unsigned ValueID = Record[0];
+ uint64_t GlobalValSummaryOffset = Record[1];
+ uint64_t GlobalValGUID = Record[2];
+ std::unique_ptr<GlobalValueInfo> GlobalValInfo =
+ llvm::make_unique<GlobalValueInfo>(GlobalValSummaryOffset);
+ SummaryOffsetToInfoMap[GlobalValSummaryOffset] = GlobalValInfo.get();
+ TheIndex->addGlobalValueInfo(GlobalValGUID, std::move(GlobalValInfo));
+ ValueIdToCallGraphGUIDMap[ValueID] = GlobalValGUID;
+ break;
+ }
+ case bitc::VST_CODE_COMBINED_ENTRY: {
+ // VST_CODE_COMBINED_ENTRY: [valueid, refguid]
+ unsigned ValueID = Record[0];
+ uint64_t RefGUID = Record[1];
+ ValueIdToCallGraphGUIDMap[ValueID] = RefGUID;
break;
}
}
}
}
-// Parse just the blocks needed for function index building out of the module.
-// At the end of this routine the function Index is populated with a map
-// from function name to FunctionInfo. The function info contains
-// either the parsed function summary information (when parsing summaries
-// eagerly), or just to the function summary record's offset
+// Parse just the blocks needed for building the index out of the module.
+// At the end of this routine the module Index is populated with a map
+// from global value name to GlobalValueInfo. The global value info contains
+// either the parsed summary information (when parsing summaries
+// eagerly), or just to the summary record's offset
// if parsing lazily (IsLazy).
std::error_code FunctionIndexBitcodeReader::parseModule() {
if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID))
return error("Invalid record");
SmallVector<uint64_t, 64> Record;
+ DenseMap<unsigned, GlobalValue::LinkageTypes> ValueIdToLinkageMap;
+ unsigned ValueId = 0;
- // Read the function index for this module.
+ // Read the index for this module.
while (1) {
BitstreamEntry Entry = Stream.advance();
@@ -5528,9 +5583,9 @@ std::error_code FunctionIndexBitcodeReader::parseModule() {
return std::error_code();
case BitstreamEntry::SubBlock:
- if (CheckFuncSummaryPresenceOnly) {
- if (Entry.ID == bitc::FUNCTION_SUMMARY_BLOCK_ID) {
- SeenFuncSummary = true;
+ if (CheckGlobalValSummaryPresenceOnly) {
+ if (Entry.ID == bitc::GLOBALVAL_SUMMARY_BLOCK_ID) {
+ SeenGlobalValSummary = true;
// No need to parse the rest since we found the summary.
return std::error_code();
}
@@ -5549,11 +5604,23 @@ std::error_code FunctionIndexBitcodeReader::parseModule() {
return error("Malformed block");
break;
case bitc::VALUE_SYMTAB_BLOCK_ID:
- if (std::error_code EC = parseValueSymbolTable())
- return EC;
+ // Should have been parsed earlier via VSTOffset, unless there
+ // is no summary section.
+ assert(((SeenValueSymbolTable && VSTOffset > 0) ||
+ !SeenGlobalValSummary) &&
+ "Expected early VST parse via VSTOffset record");
+ if (Stream.SkipBlock())
+ return error("Invalid record");
break;
- case bitc::FUNCTION_SUMMARY_BLOCK_ID:
- SeenFuncSummary = true;
+ case bitc::GLOBALVAL_SUMMARY_BLOCK_ID:
+ assert(VSTOffset > 0 && "Expected non-zero VST offset");
+ assert(!SeenValueSymbolTable &&
+ "Already read VST when parsing summary block?");
+ if (std::error_code EC =
+ parseValueSymbolTable(VSTOffset, ValueIdToLinkageMap))
+ return EC;
+ SeenValueSymbolTable = true;
+ SeenGlobalValSummary = true;
if (IsLazy) {
// Lazy parsing of summary info, skip it.
if (Stream.SkipBlock())
@@ -5569,8 +5636,8 @@ std::error_code FunctionIndexBitcodeReader::parseModule() {
continue;
case BitstreamEntry::Record:
- // Once we find the single record of interest, skip the rest.
- if (!SourceFileName.empty())
+ // Once we find the last record of interest, skip the rest.
+ if (VSTOffset > 0)
Stream.skipRecord(Entry.ID);
else {
Record.clear();
@@ -5579,28 +5646,68 @@ std::error_code FunctionIndexBitcodeReader::parseModule() {
default:
break; // Default behavior, ignore unknown content.
/// MODULE_CODE_SOURCE_FILENAME: [namechar x N]
- case bitc::MODULE_CODE_SOURCE_FILENAME:
+ case bitc::MODULE_CODE_SOURCE_FILENAME: {
SmallString<128> ValueName;
if (convertToString(Record, 0, ValueName))
return error("Invalid record");
SourceFileName = ValueName.c_str();
break;
}
+ /// MODULE_CODE_VSTOFFSET: [offset]
+ case bitc::MODULE_CODE_VSTOFFSET:
+ if (Record.size() < 1)
+ return error("Invalid record");
+ VSTOffset = Record[0];
+ break;
+ // GLOBALVAR: [pointer type, isconst, initid,
+ // linkage, alignment, section, visibility, threadlocal,
+ // unnamed_addr, externally_initialized, dllstorageclass,
+ // comdat]
+ case bitc::MODULE_CODE_GLOBALVAR: {
+ if (Record.size() < 6)
+ return error("Invalid record");
+ uint64_t RawLinkage = Record[3];
+ GlobalValue::LinkageTypes Linkage = getDecodedLinkage(RawLinkage);
+ ValueIdToLinkageMap[ValueId++] = Linkage;
+ break;
+ }
+ // FUNCTION: [type, callingconv, isproto, linkage, paramattr,
+ // alignment, section, visibility, gc, unnamed_addr,
+ // prologuedata, dllstorageclass, comdat, prefixdata]
+ case bitc::MODULE_CODE_FUNCTION: {
+ if (Record.size() < 8)
+ return error("Invalid record");
+ uint64_t RawLinkage = Record[3];
+ GlobalValue::LinkageTypes Linkage = getDecodedLinkage(RawLinkage);
+ ValueIdToLinkageMap[ValueId++] = Linkage;
+ break;
+ }
+ // ALIAS: [alias type, addrspace, aliasee val#, linkage, visibility,
+ // dllstorageclass]
+ case bitc::MODULE_CODE_ALIAS: {
+ if (Record.size() < 6)
+ return error("Invalid record");
+ uint64_t RawLinkage = Record[3];
+ GlobalValue::LinkageTypes Linkage = getDecodedLinkage(RawLinkage);
+ ValueIdToLinkageMap[ValueId++] = Linkage;
+ break;
+ }
+ }
}
continue;
}
}
}
-// Eagerly parse the entire function summary block (i.e. for all functions
-// in the index). This populates the FunctionSummary objects in
-// the index.
+// Eagerly parse the entire summary block. This populates the GlobalValueSummary
+// objects in the index.
std::error_code FunctionIndexBitcodeReader::parseEntireSummary() {
- if (Stream.EnterSubBlock(bitc::FUNCTION_SUMMARY_BLOCK_ID))
+ if (Stream.EnterSubBlock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID))
return error("Invalid record");
SmallVector<uint64_t, 64> Record;
+ bool Combined = false;
while (1) {
BitstreamEntry Entry = Stream.advanceSkippingSubblocks();
@@ -5609,6 +5716,16 @@ std::error_code FunctionIndexBitcodeReader::parseEntireSummary() {
case BitstreamEntry::Error:
return error("Malformed block");
case BitstreamEntry::EndBlock:
+ // For a per-module index, remove any entries that still have empty
+ // summaries. The VST parsing creates entries eagerly for all symbols,
+ // but not all have associated summaries (e.g. it doesn't know how to
+ // distinguish between VST_CODE_ENTRY for function declarations vs global
+ // variables with initializers that end up with a summary). Remove those
+ // entries now so that we don't need to rely on the combined index merger
+ // to clean them up (especially since that may not run for the first
+ // module's index if we merge into that).
+ if (!Combined)
+ TheIndex->removeEmptySummaryEntries();
return std::error_code();
case BitstreamEntry::Record:
// The interesting case.
@@ -5624,17 +5741,23 @@ std::error_code FunctionIndexBitcodeReader::parseEntireSummary() {
// information used for ThinLTO renaming and importing.
Record.clear();
uint64_t CurRecordBit = Stream.GetCurrentBitNo();
- switch (Stream.readRecord(Entry.ID, Record)) {
+ auto BitCode = Stream.readRecord(Entry.ID, Record);
+ switch (BitCode) {
default: // Default behavior: ignore.
break;
- // FS_PERMODULE_ENTRY: [valueid, linkage, instcount]
- case bitc::FS_CODE_PERMODULE_ENTRY: {
+ // FS_PERMODULE: [valueid, linkage, instcount, numrefs, numrefs x valueid,
+ // n x (valueid, callsitecount)]
+ // FS_PERMODULE_PROFILE: [valueid, linkage, instcount, numrefs,
+ // numrefs x valueid,
+ // n x (valueid, callsitecount, profilecount)]
+ case bitc::FS_PERMODULE:
+ case bitc::FS_PERMODULE_PROFILE: {
unsigned ValueID = Record[0];
uint64_t RawLinkage = Record[1];
unsigned InstCount = Record[2];
- std::unique_ptr<FunctionSummary> FS =
- llvm::make_unique<FunctionSummary>(InstCount);
- FS->setFunctionLinkage(getDecodedLinkage(RawLinkage));
+ unsigned NumRefs = Record[3];
+ std::unique_ptr<FunctionSummary> FS = llvm::make_unique<FunctionSummary>(
+ getDecodedLinkage(RawLinkage), InstCount);
// The module path string ref set in the summary must be owned by the
// index's module string table. Since we don't have a module path
// string table section in the per-module index, we create a single
@@ -5642,19 +5765,115 @@ std::error_code FunctionIndexBitcodeReader::parseEntireSummary() {
// ownership.
FS->setModulePath(
TheIndex->addModulePath(Buffer->getBufferIdentifier(), 0));
- SummaryMap[ValueID] = std::move(FS);
+ static int RefListStartIndex = 4;
+ int CallGraphEdgeStartIndex = RefListStartIndex + NumRefs;
+ assert(Record.size() >= RefListStartIndex + NumRefs &&
+ "Record size inconsistent with number of references");
+ for (unsigned I = 4, E = CallGraphEdgeStartIndex; I != E; ++I) {
+ unsigned RefValueId = Record[I];
+ uint64_t RefGUID = getGUIDFromValueId(RefValueId);
+ FS->addRefEdge(RefGUID);
+ }
+ bool HasProfile = (BitCode == bitc::FS_PERMODULE_PROFILE);
+ for (unsigned I = CallGraphEdgeStartIndex, E = Record.size(); I != E;
+ ++I) {
+ unsigned CalleeValueId = Record[I];
+ unsigned CallsiteCount = Record[++I];
+ uint64_t ProfileCount = HasProfile ? Record[++I] : 0;
+ uint64_t CalleeGUID = getGUIDFromValueId(CalleeValueId);
+ FS->addCallGraphEdge(CalleeGUID,
+ CalleeInfo(CallsiteCount, ProfileCount));
+ }
+ uint64_t GUID = getGUIDFromValueId(ValueID);
+ auto InfoList = TheIndex->findGlobalValueInfoList(GUID);
+ assert(InfoList != TheIndex->end() &&
+ "Expected VST parse to create GlobalValueInfo entry");
+ assert(InfoList->second.size() == 1 &&
+ "Expected a single GlobalValueInfo per GUID in module");
+ auto &Info = InfoList->second[0];
+ assert(!Info->summary() && "Expected a single summary per VST entry");
+ Info->setSummary(std::move(FS));
+ break;
+ }
+ // FS_PERMODULE_GLOBALVAR_INIT_REFS: [valueid, linkage, n x valueid]
+ case bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS: {
+ unsigned ValueID = Record[0];
+ uint64_t RawLinkage = Record[1];
+ std::unique_ptr<GlobalVarSummary> FS =
+ llvm::make_unique<GlobalVarSummary>(getDecodedLinkage(RawLinkage));
+ FS->setModulePath(
+ TheIndex->addModulePath(Buffer->getBufferIdentifier(), 0));
+ for (unsigned I = 2, E = Record.size(); I != E; ++I) {
+ unsigned RefValueId = Record[I];
+ uint64_t RefGUID = getGUIDFromValueId(RefValueId);
+ FS->addRefEdge(RefGUID);
+ }
+ uint64_t GUID = getGUIDFromValueId(ValueID);
+ auto InfoList = TheIndex->findGlobalValueInfoList(GUID);
+ assert(InfoList != TheIndex->end() &&
+ "Expected VST parse to create GlobalValueInfo entry");
+ assert(InfoList->second.size() == 1 &&
+ "Expected a single GlobalValueInfo per GUID in module");
+ auto &Info = InfoList->second[0];
+ assert(!Info->summary() && "Expected a single summary per VST entry");
+ Info->setSummary(std::move(FS));
+ break;
+ }
+ // FS_COMBINED: [modid, linkage, instcount, numrefs, numrefs x valueid,
+ // n x (valueid, callsitecount)]
+ // FS_COMBINED_PROFILE: [modid, linkage, instcount, numrefs,
+ // numrefs x valueid,
+ // n x (valueid, callsitecount, profilecount)]
+ case bitc::FS_COMBINED:
+ case bitc::FS_COMBINED_PROFILE: {
+ uint64_t ModuleId = Record[0];
+ uint64_t RawLinkage = Record[1];
+ unsigned InstCount = Record[2];
+ unsigned NumRefs = Record[3];
+ std::unique_ptr<FunctionSummary> FS = llvm::make_unique<FunctionSummary>(
+ getDecodedLinkage(RawLinkage), InstCount);
+ FS->setModulePath(ModuleIdMap[ModuleId]);
+ static int RefListStartIndex = 4;
+ int CallGraphEdgeStartIndex = RefListStartIndex + NumRefs;
+ assert(Record.size() >= RefListStartIndex + NumRefs &&
+ "Record size inconsistent with number of references");
+ for (unsigned I = 4, E = CallGraphEdgeStartIndex; I != E; ++I) {
+ unsigned RefValueId = Record[I];
+ uint64_t RefGUID = getGUIDFromValueId(RefValueId);
+ FS->addRefEdge(RefGUID);
+ }
+ bool HasProfile = (BitCode == bitc::FS_COMBINED_PROFILE);
+ for (unsigned I = CallGraphEdgeStartIndex, E = Record.size(); I != E;
+ ++I) {
+ unsigned CalleeValueId = Record[I];
+ unsigned CallsiteCount = Record[++I];
+ uint64_t ProfileCount = HasProfile ? Record[++I] : 0;
+ uint64_t CalleeGUID = getGUIDFromValueId(CalleeValueId);
+ FS->addCallGraphEdge(CalleeGUID,
+ CalleeInfo(CallsiteCount, ProfileCount));
+ }
+ auto *Info = getInfoFromSummaryOffset(CurRecordBit);
+ assert(!Info->summary() && "Expected a single summary per VST entry");
+ Info->setSummary(std::move(FS));
+ Combined = true;
break;
}
- // FS_COMBINED_ENTRY: [modid, linkage, instcount]
- case bitc::FS_CODE_COMBINED_ENTRY: {
+ // FS_COMBINED_GLOBALVAR_INIT_REFS: [modid, linkage, n x valueid]
+ case bitc::FS_COMBINED_GLOBALVAR_INIT_REFS: {
uint64_t ModuleId = Record[0];
uint64_t RawLinkage = Record[1];
- unsigned InstCount = Record[2];
- std::unique_ptr<FunctionSummary> FS =
- llvm::make_unique<FunctionSummary>(InstCount);
- FS->setFunctionLinkage(getDecodedLinkage(RawLinkage));
+ std::unique_ptr<GlobalVarSummary> FS =
+ llvm::make_unique<GlobalVarSummary>(getDecodedLinkage(RawLinkage));
FS->setModulePath(ModuleIdMap[ModuleId]);
- SummaryMap[CurRecordBit] = std::move(FS);
+ for (unsigned I = 2, E = Record.size(); I != E; ++I) {
+ unsigned RefValueId = Record[I];
+ uint64_t RefGUID = getGUIDFromValueId(RefValueId);
+ FS->addRefEdge(RefGUID);
+ }
+ auto *Info = getInfoFromSummaryOffset(CurRecordBit);
+ assert(!Info->summary() && "Expected a single summary per VST entry");
+ Info->setSummary(std::move(FS));
+ Combined = true;
break;
}
}
@@ -5773,7 +5992,9 @@ std::error_code FunctionIndexBitcodeReader::parseFunctionSummary(
// importing is added so that it can be tested.
SmallVector<uint64_t, 64> Record;
switch (Stream.readRecord(Entry.ID, Record)) {
- case bitc::FS_CODE_COMBINED_ENTRY:
+ case bitc::FS_COMBINED:
+ case bitc::FS_COMBINED_PROFILE:
+ case bitc::FS_COMBINED_GLOBALVAR_INIT_REFS:
default:
return error("Invalid record");
}
@@ -5987,9 +6208,9 @@ llvm::getFunctionInfoIndex(MemoryBufferRef Buffer,
return std::move(Index);
}
-// Check if the given bitcode buffer contains a function summary block.
-bool llvm::hasFunctionSummary(MemoryBufferRef Buffer,
- DiagnosticHandlerFunction DiagnosticHandler) {
+// Check if the given bitcode buffer contains a global value summary block.
+bool llvm::hasGlobalValueSummary(MemoryBufferRef Buffer,
+ DiagnosticHandlerFunction DiagnosticHandler) {
std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Buffer, false);
FunctionIndexBitcodeReader R(Buf.get(), DiagnosticHandler, false, true);
@@ -6002,7 +6223,7 @@ bool llvm::hasFunctionSummary(MemoryBufferRef Buffer,
return cleanupOnError(EC);
Buf.release(); // The FunctionIndexBitcodeReader owns it now.
- return R.foundFuncSummary();
+ return R.foundGlobalValSummary();
}
// This method supports lazy reading of function summary data from the combined
@@ -6026,7 +6247,7 @@ std::error_code llvm::readFunctionSummary(
// contain a list of function infos in the case of a COMDAT. Walk through
// and parse each function summary info at the function summary offset
// recorded when parsing the value symbol table.
- for (const auto &FI : Index->getFunctionInfoList(FunctionName)) {
+ for (const auto &FI : Index->getGlobalValueInfoList(FunctionName)) {
size_t FunctionSummaryOffset = FI->bitcodeIndex();
if (std::error_code EC =
R.parseFunctionSummary(nullptr, Index.get(), FunctionSummaryOffset))
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 4b7b0d70044..84d09635d1c 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -11,24 +11,30 @@
//
//===----------------------------------------------------------------------===//
-#include "llvm/Bitcode/ReaderWriter.h"
#include "ValueEnumerator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Triple.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BlockFrequencyInfoImpl.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Bitcode/BitstreamWriter.h"
#include "llvm/Bitcode/LLVMBitCodes.h"
+#include "llvm/Bitcode/ReaderWriter.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Dominators.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Instructions.h"
-#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/UseListOrder.h"
#include "llvm/IR/ValueSymbolTable.h"
+#include "llvm/ProfileData/ProfileCommon.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
@@ -591,11 +597,7 @@ static void writeComdats(const ValueEnumerator &VE, BitstreamWriter &Stream) {
/// Write a record that will eventually hold the word offset of the
/// module-level VST. For now the offset is 0, which will be backpatched
/// after the real VST is written. Returns the bit offset to backpatch.
-static uint64_t WriteValueSymbolTableForwardDecl(const ValueSymbolTable &VST,
- BitstreamWriter &Stream) {
- if (VST.empty())
- return 0;
-
+static uint64_t WriteValueSymbolTableForwardDecl(BitstreamWriter &Stream) {
// Write a placeholder value in for the offset of the real VST,
// which is written after the function blocks so that it can include
// the offset of each function. The placeholder offset will be
@@ -844,9 +846,11 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE,
Vals.clear();
}
- uint64_t VSTOffsetPlaceholder =
- WriteValueSymbolTableForwardDecl(M->getValueSymbolTable(), Stream);
- return VSTOffsetPlaceholder;
+ // If we have a VST, write the VSTOFFSET record placeholder and return
+ // its offset.
+ if (M->getValueSymbolTable().empty())
+ return 0;
+ return WriteValueSymbolTableForwardDecl(Stream);
}
static uint64_t GetOptimizationFlags(const Value *V) {
@@ -2248,8 +2252,8 @@ static void WriteValueSymbolTable(
const ValueSymbolTable &VST, const ValueEnumerator &VE,
BitstreamWriter &Stream, uint64_t VSTOffsetPlaceholder = 0,
uint64_t BitcodeStartBit = 0,
- DenseMap<const Function *, std::unique_ptr<FunctionInfo>> *FunctionIndex =
- nullptr) {
+ DenseMap<const Function *, std::unique_ptr<GlobalValueInfo>>
+ *FunctionIndex = nullptr) {
if (VST.empty()) {
// WriteValueSymbolTableForwardDecl should have returned early as
// well. Ensure this handling remains in sync by asserting that
@@ -2375,33 +2379,61 @@ static void WriteValueSymbolTable(
/// Emit function names and summary offsets for the combined index
/// used by ThinLTO.
-static void WriteCombinedValueSymbolTable(const FunctionInfoIndex &Index,
- BitstreamWriter &Stream) {
+static void
+WriteCombinedValueSymbolTable(const FunctionInfoIndex &Index,
+ BitstreamWriter &Stream,
+ std::map<uint64_t, unsigned> &GUIDToValueIdMap,
+ uint64_t VSTOffsetPlaceholder) {
+ assert(VSTOffsetPlaceholder > 0 && "Expected non-zero VSTOffsetPlaceholder");
+ // Get the offset of the VST we are writing, and backpatch it into
+ // the VST forward declaration record.
+ uint64_t VSTOffset = Stream.GetCurrentBitNo();
+ assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned");
+ Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32);
+
Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4);
BitCodeAbbrev *Abbv = new BitCodeAbbrev();
- Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_FNENTRY));
- Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcsumoffset
- Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcguid
- unsigned FnEntryAbbrev = Stream.EmitAbbrev(Abbv);
+ Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_GVDEFENTRY));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // sumoffset
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // guid
+ unsigned DefEntryAbbrev = Stream.EmitAbbrev(Abbv);
+
+ Abbv = new BitCodeAbbrev();
+ Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid
+ unsigned EntryAbbrev = Stream.EmitAbbrev(Abbv);
SmallVector<uint64_t, 64> NameVals;
for (const auto &FII : Index) {
+ uint64_t FuncGUID = FII.first;
+ const auto &VMI = GUIDToValueIdMap.find(FuncGUID);
+ assert(VMI != GUIDToValueIdMap.end());
+
for (const auto &FI : FII.second) {
+ // VST_CODE_COMBINED_GVDEFENTRY: [valueid, sumoffset, guid]
+ NameVals.push_back(VMI->second);
NameVals.push_back(FI->bitcodeIndex());
-
- uint64_t FuncGUID = FII.first;
-
- // VST_CODE_COMBINED_FNENTRY: [funcsumoffset, funcguid]
- unsigned AbbrevToUse = FnEntryAbbrev;
-
NameVals.push_back(FuncGUID);
// Emit the finished record.
- Stream.EmitRecord(bitc::VST_CODE_COMBINED_FNENTRY, NameVals, AbbrevToUse);
+ Stream.EmitRecord(bitc::VST_CODE_COMBINED_GVDEFENTRY, NameVals,
+ DefEntryAbbrev);
NameVals.clear();
}
+ GUIDToValueIdMap.erase(VMI);
+ }
+ for (const auto &GVI : GUIDToValueIdMap) {
+ // VST_CODE_COMBINED_ENTRY: [valueid, refguid]
+ NameVals.push_back(GVI.second);
+ NameVals.push_back(GVI.first);
+
+ // Emit the finished record.
+ Stream.EmitRecord(bitc::VST_CODE_COMBINED_ENTRY, NameVals, EntryAbbrev);
+ NameVals.clear();
}
Stream.ExitBlock();
}
@@ -2440,34 +2472,61 @@ static void WriteUseListBlock(const Function *F, ValueEnumerator &VE,
Stream.ExitBlock();
}
-/// \brief Save information for the given function into the function index.
-///
-/// At a minimum this saves the bitcode index of the function record that
-/// was just written. However, if we are emitting function summary information,
-/// for example for ThinLTO, then a \a FunctionSummary object is created
-/// to hold the provided summary information.
-static void SaveFunctionInfo(
- const Function &F,
- DenseMap<const Function *, std::unique_ptr<FunctionInfo>> &FunctionIndex,
- unsigned NumInsts, uint64_t BitcodeIndex, bool EmitFunctionSummary) {
- std::unique_ptr<FunctionSummary> FuncSummary;
- if (EmitFunctionSummary) {
- FuncSummary = llvm::make_unique<FunctionSummary>(NumInsts);
- FuncSummary->setFunctionLinkage(F.getLinkage());
+// Walk through the operands of a given User via worklist iteration and populate
+// the set of GlobalValue references encountered. Invoked either on an
+// Instruction or a GlobalVariable (which walks its initializer).
+static void findRefEdges(const User *CurUser, const ValueEnumerator &VE,
+ DenseSet<unsigned> &RefEdges,
+ SmallPtrSet<const User *, 8> &Visited) {
+ SmallVector<const User *, 32> Worklist;
+ Worklist.push_back(CurUser);
+
+ while (!Worklist.empty()) {
+ const User *U = Worklist.pop_back_val();
+
+ if (!Visited.insert(U).second)
+ continue;
+
+ ImmutableCallSite CS(U);
+
+ for (const auto &OI : U->operands()) {
+ const User *Operand = dyn_cast<User>(OI);
+ if (!Operand)
+ continue;
+ if (isa<BlockAddress>(Operand))
+ continue;
+ if (isa<GlobalValue>(Operand)) {
+ // We have a reference to a global value. This should be added to
+ // the reference set unless it is a callee. Callees are handled
+ // specially by WriteFunction and are added to a separate list.
+ if (!(CS && CS.isCallee(&OI)))
+ RefEdges.insert(VE.getValueID(Operand));
+ continue;
+ }
+ Worklist.push_back(Operand);
+ }
}
- FunctionIndex[&F] =
- llvm::make_unique<FunctionInfo>(BitcodeIndex, std::move(FuncSummary));
}
/// Emit a function body to the module stream.
static void WriteFunction(
- const Function &F, ValueEnumerator &VE, BitstreamWriter &Stream,
- DenseMap<const Function *, std::unique_ptr<FunctionInfo>> &FunctionIndex,
- bool EmitFunctionSummary) {
+ const Function &F, const Module *M, ValueEnumerator &VE,
+ BitstreamWriter &Stream,
+ DenseMap<const Function *, std::unique_ptr<GlobalValueInfo>> &FunctionIndex,
+ bool EmitSummaryIndex) {
// Save the bitcode index of the start of this function block for recording
// in the VST.
uint64_t BitcodeIndex = Stream.GetCurrentBitNo();
+ bool HasProfileData = F.getEntryCount().hasValue();
+ std::unique_ptr<BlockFrequencyInfo> BFI;
+ if (EmitSummaryIndex && HasProfileData) {
+ Function &Func = const_cast<Function &>(F);
+ LoopInfo LI{DominatorTree(Func)};
+ BranchProbabilityInfo BPI{Func, LI};
+ BFI = llvm::make_unique<BlockFrequencyInfo>(Func, BPI, LI);
+ }
+
Stream.EnterSubblock(bitc::FUNCTION_BLOCK_ID, 4);
VE.incorporateFunction(F);
@@ -2494,7 +2553,12 @@ static void WriteFunction(
DILocation *LastDL = nullptr;
unsigned NumInsts = 0;
+ // Map from callee ValueId to profile count. Used to accumulate profile
+ // counts for all static calls to a given callee.
+ DenseMap<unsigned, CalleeInfo> CallGraphEdges;
+ DenseSet<unsigned> RefEdges;
+ SmallPtrSet<const User *, 8> Visited;
// Finally, emit all the instructions, in order.
for (Function::const_iterator BB = F.begin(), E = F.end(); BB != E; ++BB)
for (BasicBlock::const_iterator I = BB->begin(), E = BB->end();
@@ -2507,6 +2571,24 @@ static void WriteFunction(
if (!I->getType()->isVoidTy())
++InstID;
+ if (EmitSummaryIndex) {
+ if (auto CS = ImmutableCallSite(&*I)) {
+ auto *CalledFunction = CS.getCalledFunction();
+ if (CalledFunction && CalledFunction->hasName() &&
+ !CalledFunction->isIntrinsic()) {
+ uint64_t ScaledCount = 0;
+ if (HasProfileData)
+ ScaledCount = getBlockProfileCount(
+ BFI->getBlockFreq(&(*BB)).getFrequency(), BFI->getEntryFreq(),
+ F.getEntryCount().getValue());
+ unsigned CalleeId = VE.getValueID(
+ M->getValueSymbolTable().lookup(CalledFunction->getName()));
+ CallGraphEdges[CalleeId] += ScaledCount;
+ }
+ }
+ findRefEdges(&*I, VE, RefEdges, Visited);
+ }
+
// If the instruction has metadata, write a metadata attachment later.
NeedsMetadataAttachment |= I->hasMetadataOtherThanDebugLoc();
@@ -2531,6 +2613,15 @@ static void WriteFunction(
LastDL = DL;
}
+ std::unique_ptr<FunctionSummary> FuncSummary;
+ if (EmitSummaryIndex) {
+ FuncSummary = llvm::make_unique<FunctionSummary>(F.getLinkage(), NumInsts);
+ FuncSummary->addCallGraphEdges(CallGraphEdges);
+ FuncSummary->addRefEdges(RefEdges);
+ }
+ FunctionIndex[&F] =
+ llvm::make_unique<GlobalValueInfo>(BitcodeIndex, std::move(FuncSummary));
+
// Emit names for all the instructions etc.
WriteValueSymbolTable(F.getValueSymbolTable(), VE, Stream);
@@ -2540,9 +2631,6 @@ static void WriteFunction(
WriteUseListBlock(&F, VE, Stream);
VE.purgeFunction();
Stream.ExitBlock();
-
- SaveFunctionInfo(F, FunctionIndex, NumInsts, BitcodeIndex,
- EmitFunctionSummary);
}
// Emit blockinfo, which defines the standard abbreviations etc.
@@ -2776,34 +2864,103 @@ static void WriteModStrings(const FunctionInfoIndex &I,
// Helper to emit a single function summary record.
static void WritePerModuleFunctionSummaryRecord(
- SmallVector<unsigned, 64> &NameVals, FunctionSummary *FS, unsigned ValueID,
- unsigned FSAbbrev, BitstreamWriter &Stream) {
+ SmallVector<uint64_t, 64> &NameVals, FunctionSummary *FS, unsigned ValueID,
+ unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev,
+ BitstreamWriter &Stream, const Function &F) {
assert(FS);
NameVals.push_back(ValueID);
- NameVals.push_back(getEncodedLinkage(FS->getFunctionLinkage()));
+ NameVals.push_back(getEncodedLinkage(FS->linkage()));
NameVals.push_back(FS->instCount());
+ NameVals.push_back(FS->refs().size());
+
+ for (auto &RI : FS->refs())
+ NameVals.push_back(RI);
+
+ bool HasProfileData = F.getEntryCount().hasValue();
+ for (auto &ECI : FS->edges()) {
+ NameVals.push_back(ECI.first);
+ assert(ECI.second.CallsiteCount > 0 && "Expected at least one callsite");
+ NameVals.push_back(ECI.second.CallsiteCount);
+ if (HasProfileData)
+ NameVals.push_back(ECI.second.ProfileCount);
+ }
+
+ unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev);
+ unsigned Code =
+ (HasProfileData ? bitc::FS_PERMODULE_PROFILE : bitc::FS_PERMODULE);
// Emit the finished record.
- Stream.EmitRecord(bitc::FS_CODE_PERMODULE_ENTRY, NameVals, FSAbbrev);
+ Stream.EmitRecord(Code, NameVals, FSAbbrev);
NameVals.clear();
}
-/// Emit the per-module function summary section alongside the rest of
+// Collect the global value references in the given variable's initializer,
+// and emit them in a summary record.
+static void WriteModuleLevelReferences(const GlobalVariable &V,
+ const ValueEnumerator &VE,
+ SmallVector<uint64_t, 64> &NameVals,
+ unsigned FSModRefsAbbrev,
+ BitstreamWriter &Stream) {
+ DenseSet<unsigned> RefEdges;
+ SmallPtrSet<const User *, 8> Visited;
+ findRefEdges(&V, VE, RefEdges, Visited);
+ unsigned RefCount = RefEdges.size();
+ if (RefCount) {
+ NameVals.push_back(VE.getValueID(&V));
+ NameVals.push_back(getEncodedLinkage(V.getLinkage()));
+ for (auto RefId : RefEdges) {
+ NameVals.push_back(RefId);
+ }
+ Stream.EmitRecord(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, NameVals,
+ FSModRefsAbbrev);
+ NameVals.clear();
+ }
+}
+
+/// Emit the per-module summary section alongside the rest of
/// the module's bitcode.
-static void WritePerModuleFunctionSummary(
- DenseMap<const Function *, std::unique_ptr<FunctionInfo>> &FunctionIndex,
+static void WritePerModuleGlobalValueSummary(
+ DenseMap<const Function *, std::unique_ptr<GlobalValueInfo>> &FunctionIndex,
const Module *M, const ValueEnumerator &VE, BitstreamWriter &Stream) {
- Stream.EnterSubblock(bitc::FUNCTION_SUMMARY_BLOCK_ID, 3);
+ if (M->empty())
+ return;
+
+ Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3);
- // Abbrev for FS_CODE_PERMODULE_ENTRY.
+ // Abbrev for FS_PERMODULE.
BitCodeAbbrev *Abbv = new BitCodeAbbrev();
- Abbv->Add(BitCodeAbbrevOp(bitc::FS_CODE_PERMODULE_ENTRY));
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE));
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
- unsigned FSAbbrev = Stream.EmitAbbrev(Abbv);
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
+ // numrefs x valueid, n x (valueid, callsitecount)
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+ unsigned FSCallsAbbrev = Stream.EmitAbbrev(Abbv);
- SmallVector<unsigned, 64> NameVals;
+ // Abbrev for FS_PERMODULE_PROFILE.
+ Abbv = new BitCodeAbbrev();
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
+ // numrefs x valueid, n x (valueid, callsitecount, profilecount)
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+ unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv);
+
+ // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
+ Abbv = new BitCodeAbbrev();
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+ unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv);
+
+ SmallVector<uint64_t, 64> NameVals;
// Iterate over the list of functions instead of the FunctionIndex map to
// ensure the ordering is stable.
for (const Function &F : *M) {
@@ -2817,9 +2974,9 @@ static void WritePerModuleFunctionSummary(
assert(FunctionIndex.count(&F) == 1);
WritePerModuleFunctionSummaryRecord(
- NameVals, FunctionIndex[&F]->functionSummary(),
- VE.getValueID(M->getValueSymbolTable().lookup(F.getName())), FSAbbrev,
- Stream);
+ NameVals, dyn_cast<FunctionSummary>(FunctionIndex[&F]->summary()),
+ VE.getValueID(M->getValueSymbolTable().lookup(F.getName())),
+ FSCallsAbbrev, FSCallsProfileAbbrev, Stream, F);
}
for (const GlobalAlias &A : M->aliases()) {
@@ -2830,10 +2987,25 @@ static void WritePerModuleFunctionSummary(
continue;
assert(FunctionIndex.count(F) == 1);
+ FunctionSummary *FS =
+ dyn_cast<FunctionSummary>(FunctionIndex[F]->summary());
+ // Add the alias to the reference list of aliasee function.
+ FS->addRefEdge(
+ VE.getValueID(M->getValueSymbolTable().lookup(A.getName())));
WritePerModuleFunctionSummaryRecord(
- NameVals, FunctionIndex[F]->functionSummary(),
- VE.getValueID(M->getValueSymbolTable().lookup(A.getName())), FSAbbrev,
- Stream);
+ NameVals, FS,
+ VE.getValueID(M->getValueSymbolTable().lookup(A.getName())),
+ FSCallsAbbrev, FSCallsProfileAbbrev, Stream, *F);
+ }
+
+ // Capture references from GlobalVariable initializers, which are outside
+ // of a function scope.
+ for (const GlobalVariable &G : M->globals())
+ WriteModuleLevelReferences(G, VE, NameVals, FSModRefsAbbrev, Stream);
+ for (const GlobalAlias &A : M->aliases()) {
+ const auto *GV = dyn_cast<GlobalVariable>(A.getBaseObject());
+ if (GV)
+ WriteModuleLevelReferences(*GV, VE, NameVals, FSModRefsAbbrev, Stream);
}
Stream.ExitBlock();
@@ -2841,35 +3013,132 @@ static void WritePerModuleFunctionSummary(
/// Emit the combined function summary section into the combined index
/// file.
-static void WriteCombinedFunctionSummary(const FunctionInfoIndex &I,
- BitstreamWriter &Stream) {
- Stream.EnterSubblock(bitc::FUNCTION_SUMMARY_BLOCK_ID, 3);
+static void WriteCombinedGlobalValueSummary(
+ const FunctionInfoIndex &I, BitstreamWriter &Stream,
+ std::map<uint64_t, unsigned> &GUIDToValueIdMap, unsigned GlobalValueId) {
+ Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3);
- // Abbrev for FS_CODE_COMBINED_ENTRY.
+ // Abbrev for FS_COMBINED.
BitCodeAbbrev *Abbv = new BitCodeAbbrev();
- Abbv->Add(BitCodeAbbrevOp(bitc::FS_CODE_COMBINED_ENTRY));
- Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage
- Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
- unsigned FSAbbrev = Stream.EmitAbbrev(Abbv);
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
+ // numrefs x valueid, n x (valueid, callsitecount)
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+ unsigned FSCallsAbbrev = Stream.EmitAbbrev(Abbv);
- SmallVector<unsigned, 64> NameVals;
+ // Abbrev for FS_COMBINED_PROFILE.
+ Abbv = new BitCodeAbbrev();
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
+ // numrefs x valueid, n x (valueid, callsitecount, profilecount)
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+ unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv);
+
+ // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS.
+ Abbv = new BitCodeAbbrev();
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+ unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv);
+
+ SmallVector<uint64_t, 64> NameVals;
for (const auto &FII : I) {
for (auto &FI : FII.second) {
- FunctionSummary *FS = FI->functionSummary();
- assert(FS);
+ GlobalValueSummary *S = FI->summary();
+ assert(S);
+
+ if (auto *VS = dyn_cast<GlobalVarSummary>(S)) {
+ assert(!VS->refs().empty() && "Expected at least one ref edge");
+ NameVals.push_back(I.getModuleId(VS->modulePath()));
+ NameVals.push_back(getEncodedLinkage(VS->linkage()));
+ for (auto &RI : VS->refs()) {
+ const auto &VMI = GUIDToValueIdMap.find(RI);
+ unsigned RefId;
+ // If this GUID doesn't have an entry, assign one.
+ if (VMI == GUIDToValueIdMap.end()) {
+ GUIDToValueIdMap[RI] = ++GlobalValueId;
+ RefId = GlobalValueId;
+ } else {
+ RefId = VMI->second;
+ }
+ NameVals.push_back(RefId);
+ }
+
+ // Record the starting offset of this summary entry for use
+ // in the VST entry. Add the current code size since the
+ // reader will invoke readRecord after the abbrev id read.
+ FI->setBitcodeIndex(Stream.GetCurrentBitNo() +
+ Stream.GetAbbrevIDWidth());
+
+ // Emit the finished record.
+ Stream.EmitRecord(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, NameVals,
+ FSModRefsAbbrev);
+ NameVals.clear();
+ continue;
+ }
+ auto *FS = dyn_cast<FunctionSummary>(S);
+ assert(FS);
NameVals.push_back(I.getModuleId(FS->modulePath()));
- NameVals.push_back(getEncodedLinkage(FS->getFunctionLinkage()));
+ NameVals.push_back(getEncodedLinkage(FS->linkage()));
NameVals.push_back(FS->instCount());
+ NameVals.push_back(FS->refs().size());
+
+ for (auto &RI : FS->refs()) {
+ const auto &VMI = GUIDToValueIdMap.find(RI);
+ unsigned RefId;
+ // If this GUID doesn't have an entry, assign one.
+ if (VMI == GUIDToValueIdMap.end()) {
+ GUIDToValueIdMap[RI] = ++GlobalValueId;
+ RefId = GlobalValueId;
+ } else {
+ RefId = VMI->second;
+ }
+ NameVals.push_back(RefId);
+ }
+
+ bool HasProfileData = false;
+ for (auto &EI : FS->edges()) {
+ HasProfileData |= EI.second.ProfileCount != 0;
+ if (HasProfileData)
+ break;
+ }
+
+ for (auto &EI : FS->edges()) {
+ const auto &VMI = GUIDToValueIdMap.find(EI.first);
+ // If this GUID doesn't have an entry, it doesn't have a function
+ // summary and we don't need to record any calls to it.
+ if (VMI == GUIDToValueIdMap.end())
+ continue;
+ NameVals.push_back(VMI->second);
+ assert(EI.second.CallsiteCount > 0 && "Expected at least one callsite");
+ NameVals.push_back(EI.second.CallsiteCount);
+ if (HasProfileData)
+ NameVals.push_back(EI.second.ProfileCount);
+ }
// Record the starting offset of this summary entry for use
// in the VST entry. Add the current code size since the
// reader will invoke readRecord after the abbrev id read.
FI->setBitcodeIndex(Stream.GetCurrentBitNo() + Stream.GetAbbrevIDWidth());
+ unsigned FSAbbrev =
+ (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev);
+ unsigned Code =
+ (HasProfileData ? bitc::FS_COMBINED_PROFILE : bitc::FS_COMBINED);
+
// Emit the finished record.
- Stream.EmitRecord(bitc::FS_CODE_COMBINED_ENTRY, NameVals, FSAbbrev);
+ Stream.EmitRecord(Code, NameVals, FSAbbrev);
NameVals.clear();
}
}
@@ -2904,7 +3173,7 @@ static void WriteIdentificationBlock(const Module *M, BitstreamWriter &Stream) {
/// WriteModule - Emit the specified module to the bitstream.
static void WriteModule(const Module *M, BitstreamWriter &Stream,
bool ShouldPreserveUseListOrder,
- uint64_t BitcodeStartBit, bool EmitFunctionSummary) {
+ uint64_t BitcodeStartBit, bool EmitSummaryIndex) {
Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3);
SmallVector<unsigned, 1> Vals;
@@ -2949,15 +3218,15 @@ static void WriteModule(const Module *M, BitstreamWriter &Stream,
WriteOperandBundleTags(M, Stream);
// Emit function bodies.
- DenseMap<const Function *, std::unique_ptr<FunctionInfo>> FunctionIndex;
+ DenseMap<const Function *, std::unique_ptr<GlobalValueInfo>> FunctionIndex;
for (Module::const_iterator F = M->begin(), E = M->end(); F != E; ++F)
if (!F->isDeclaration())
- WriteFunction(*F, VE, Stream, FunctionIndex, EmitFunctionSummary);
+ WriteFunction(*F, M, VE, Stream, FunctionIndex, EmitSummaryIndex);
// Need to write after the above call to WriteFunction which populates
// the summary information in the index.
- if (EmitFunctionSummary)
- WritePerModuleFunctionSummary(FunctionIndex, M, VE, Stream);
+ if (EmitSummaryIndex)
+ WritePerModuleGlobalValueSummary(FunctionIndex, M, VE, Stream);
WriteValueSymbolTable(M->getValueSymbolTable(), VE, Stream,
VSTOffsetPlaceholder, BitcodeStartBit, &FunctionIndex);
@@ -3046,7 +3315,7 @@ static void WriteBitcodeHeader(BitstreamWriter &Stream) {
/// stream.
void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out,
bool ShouldPreserveUseListOrder,
- bool EmitFunctionSummary) {
+ bool EmitSummaryIndex) {
SmallVector<char, 0> Buffer;
Buffer.reserve(256*1024);
@@ -3072,7 +3341,7 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out,
// Emit the module.
WriteModule(M, Stream, ShouldPreserveUseListOrder, BitcodeStartBit,
- EmitFunctionSummary);
+ EmitSummaryIndex);
}
if (TT.isOSDarwin() || TT.isOSBinFormatMachO())
@@ -3082,11 +3351,10 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out,
Out.write((char*)&Buffer.front(), Buffer.size());
}
-// Write the specified function summary index to the given raw output stream,
+// Write the specified module summary index to the given raw output stream,
// where it will be written in a new bitcode block. This is used when
// writing the combined index file for ThinLTO.
-void llvm::WriteFunctionSummaryToFile(const FunctionInfoIndex &Index,
- raw_ostream &Out) {
+void llvm::WriteIndexToFile(const FunctionInfoIndex &Index, raw_ostream &Out) {
SmallVector<char, 0> Buffer;
Buffer.reserve(256 * 1024);
@@ -3102,15 +3370,30 @@ void llvm::WriteFunctionSummaryToFile(const FunctionInfoIndex &Index,
Vals.push_back(CurVersion);
Stream.EmitRecord(bitc::MODULE_CODE_VERSION, Vals);
+ // If we have a VST, write the VSTOFFSET record placeholder and record
+ // its offset.
+ uint64_t VSTOffsetPlaceholder = WriteValueSymbolTableForwardDecl(Stream);
+
// Write the module paths in the combined index.
WriteModStrings(Index, Stream);
- // Write the function summary combined index records.
- WriteCombinedFunctionSummary(Index, Stream);
+ // Assign unique value ids to all functions in the index for use
+ // in writing out the call graph edges. Save the mapping from GUID
+ // to the new global value id to use when writing those edges, which
+ // are currently saved in the index in terms of GUID.
+ std::map<uint64_t, unsigned> GUIDToValueIdMap;
+ unsigned GlobalValueId = 0;
+ for (auto &II : Index)
+ GUIDToValueIdMap[II.first] = ++GlobalValueId;
+
+ // Write the summary combined index records.
+ WriteCombinedGlobalValueSummary(Index, Stream, GUIDToValueIdMap,
+ GlobalValueId);
// Need a special VST writer for the combined index (we don't have a
// real VST and real values when this is invoked).
- WriteCombinedValueSymbolTable(Index, Stream);
+ WriteCombinedValueSymbolTable(Index, Stream, GUIDToValueIdMap,
+ VSTOffsetPlaceholder);
Stream.ExitBlock();
diff --git a/llvm/lib/Bitcode/Writer/LLVMBuild.txt b/llvm/lib/Bitcode/Writer/LLVMBuild.txt
index 7d9e1de771b..a450b38fba2 100644
--- a/llvm/lib/Bitcode/Writer/LLVMBuild.txt
+++ b/llvm/lib/Bitcode/Writer/LLVMBuild.txt
@@ -19,4 +19,4 @@
type = Library
name = BitWriter
parent = Bitcode
-required_libraries = Core Support
+required_libraries = Analysis Core Support
OpenPOWER on IntegriCloud