summaryrefslogtreecommitdiffstats
path: root/llvm/lib/Bitcode/Writer
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Bitcode/Writer')
-rw-r--r--llvm/lib/Bitcode/Writer/BitcodeWriter.cpp465
-rw-r--r--llvm/lib/Bitcode/Writer/LLVMBuild.txt2
2 files changed, 375 insertions, 92 deletions
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 4b7b0d70044..84d09635d1c 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -11,24 +11,30 @@
//
//===----------------------------------------------------------------------===//
-#include "llvm/Bitcode/ReaderWriter.h"
#include "ValueEnumerator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Triple.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BlockFrequencyInfoImpl.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Bitcode/BitstreamWriter.h"
#include "llvm/Bitcode/LLVMBitCodes.h"
+#include "llvm/Bitcode/ReaderWriter.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Dominators.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Instructions.h"
-#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/UseListOrder.h"
#include "llvm/IR/ValueSymbolTable.h"
+#include "llvm/ProfileData/ProfileCommon.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
@@ -591,11 +597,7 @@ static void writeComdats(const ValueEnumerator &VE, BitstreamWriter &Stream) {
/// Write a record that will eventually hold the word offset of the
/// module-level VST. For now the offset is 0, which will be backpatched
/// after the real VST is written. Returns the bit offset to backpatch.
-static uint64_t WriteValueSymbolTableForwardDecl(const ValueSymbolTable &VST,
- BitstreamWriter &Stream) {
- if (VST.empty())
- return 0;
-
+static uint64_t WriteValueSymbolTableForwardDecl(BitstreamWriter &Stream) {
// Write a placeholder value in for the offset of the real VST,
// which is written after the function blocks so that it can include
// the offset of each function. The placeholder offset will be
@@ -844,9 +846,11 @@ static uint64_t WriteModuleInfo(const Module *M, const ValueEnumerator &VE,
Vals.clear();
}
- uint64_t VSTOffsetPlaceholder =
- WriteValueSymbolTableForwardDecl(M->getValueSymbolTable(), Stream);
- return VSTOffsetPlaceholder;
+ // If we have a VST, write the VSTOFFSET record placeholder and return
+ // its offset.
+ if (M->getValueSymbolTable().empty())
+ return 0;
+ return WriteValueSymbolTableForwardDecl(Stream);
}
static uint64_t GetOptimizationFlags(const Value *V) {
@@ -2248,8 +2252,8 @@ static void WriteValueSymbolTable(
const ValueSymbolTable &VST, const ValueEnumerator &VE,
BitstreamWriter &Stream, uint64_t VSTOffsetPlaceholder = 0,
uint64_t BitcodeStartBit = 0,
- DenseMap<const Function *, std::unique_ptr<FunctionInfo>> *FunctionIndex =
- nullptr) {
+ DenseMap<const Function *, std::unique_ptr<GlobalValueInfo>>
+ *FunctionIndex = nullptr) {
if (VST.empty()) {
// WriteValueSymbolTableForwardDecl should have returned early as
// well. Ensure this handling remains in sync by asserting that
@@ -2375,33 +2379,61 @@ static void WriteValueSymbolTable(
/// Emit function names and summary offsets for the combined index
/// used by ThinLTO.
-static void WriteCombinedValueSymbolTable(const FunctionInfoIndex &Index,
- BitstreamWriter &Stream) {
+static void
+WriteCombinedValueSymbolTable(const FunctionInfoIndex &Index,
+ BitstreamWriter &Stream,
+ std::map<uint64_t, unsigned> &GUIDToValueIdMap,
+ uint64_t VSTOffsetPlaceholder) {
+ assert(VSTOffsetPlaceholder > 0 && "Expected non-zero VSTOffsetPlaceholder");
+ // Get the offset of the VST we are writing, and backpatch it into
+ // the VST forward declaration record.
+ uint64_t VSTOffset = Stream.GetCurrentBitNo();
+ assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned");
+ Stream.BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32);
+
Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4);
BitCodeAbbrev *Abbv = new BitCodeAbbrev();
- Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_FNENTRY));
- Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcsumoffset
- Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcguid
- unsigned FnEntryAbbrev = Stream.EmitAbbrev(Abbv);
+ Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_GVDEFENTRY));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // sumoffset
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // guid
+ unsigned DefEntryAbbrev = Stream.EmitAbbrev(Abbv);
+
+ Abbv = new BitCodeAbbrev();
+ Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid
+ unsigned EntryAbbrev = Stream.EmitAbbrev(Abbv);
SmallVector<uint64_t, 64> NameVals;
for (const auto &FII : Index) {
+ uint64_t FuncGUID = FII.first;
+ const auto &VMI = GUIDToValueIdMap.find(FuncGUID);
+ assert(VMI != GUIDToValueIdMap.end());
+
for (const auto &FI : FII.second) {
+ // VST_CODE_COMBINED_GVDEFENTRY: [valueid, sumoffset, guid]
+ NameVals.push_back(VMI->second);
NameVals.push_back(FI->bitcodeIndex());
-
- uint64_t FuncGUID = FII.first;
-
- // VST_CODE_COMBINED_FNENTRY: [funcsumoffset, funcguid]
- unsigned AbbrevToUse = FnEntryAbbrev;
-
NameVals.push_back(FuncGUID);
// Emit the finished record.
- Stream.EmitRecord(bitc::VST_CODE_COMBINED_FNENTRY, NameVals, AbbrevToUse);
+ Stream.EmitRecord(bitc::VST_CODE_COMBINED_GVDEFENTRY, NameVals,
+ DefEntryAbbrev);
NameVals.clear();
}
+ GUIDToValueIdMap.erase(VMI);
+ }
+ for (const auto &GVI : GUIDToValueIdMap) {
+ // VST_CODE_COMBINED_ENTRY: [valueid, refguid]
+ NameVals.push_back(GVI.second);
+ NameVals.push_back(GVI.first);
+
+ // Emit the finished record.
+ Stream.EmitRecord(bitc::VST_CODE_COMBINED_ENTRY, NameVals, EntryAbbrev);
+ NameVals.clear();
}
Stream.ExitBlock();
}
@@ -2440,34 +2472,61 @@ static void WriteUseListBlock(const Function *F, ValueEnumerator &VE,
Stream.ExitBlock();
}
-/// \brief Save information for the given function into the function index.
-///
-/// At a minimum this saves the bitcode index of the function record that
-/// was just written. However, if we are emitting function summary information,
-/// for example for ThinLTO, then a \a FunctionSummary object is created
-/// to hold the provided summary information.
-static void SaveFunctionInfo(
- const Function &F,
- DenseMap<const Function *, std::unique_ptr<FunctionInfo>> &FunctionIndex,
- unsigned NumInsts, uint64_t BitcodeIndex, bool EmitFunctionSummary) {
- std::unique_ptr<FunctionSummary> FuncSummary;
- if (EmitFunctionSummary) {
- FuncSummary = llvm::make_unique<FunctionSummary>(NumInsts);
- FuncSummary->setFunctionLinkage(F.getLinkage());
+// Walk through the operands of a given User via worklist iteration and populate
+// the set of GlobalValue references encountered. Invoked either on an
+// Instruction or a GlobalVariable (which walks its initializer).
+static void findRefEdges(const User *CurUser, const ValueEnumerator &VE,
+ DenseSet<unsigned> &RefEdges,
+ SmallPtrSet<const User *, 8> &Visited) {
+ SmallVector<const User *, 32> Worklist;
+ Worklist.push_back(CurUser);
+
+ while (!Worklist.empty()) {
+ const User *U = Worklist.pop_back_val();
+
+ if (!Visited.insert(U).second)
+ continue;
+
+ ImmutableCallSite CS(U);
+
+ for (const auto &OI : U->operands()) {
+ const User *Operand = dyn_cast<User>(OI);
+ if (!Operand)
+ continue;
+ if (isa<BlockAddress>(Operand))
+ continue;
+ if (isa<GlobalValue>(Operand)) {
+ // We have a reference to a global value. This should be added to
+ // the reference set unless it is a callee. Callees are handled
+ // specially by WriteFunction and are added to a separate list.
+ if (!(CS && CS.isCallee(&OI)))
+ RefEdges.insert(VE.getValueID(Operand));
+ continue;
+ }
+ Worklist.push_back(Operand);
+ }
}
- FunctionIndex[&F] =
- llvm::make_unique<FunctionInfo>(BitcodeIndex, std::move(FuncSummary));
}
/// Emit a function body to the module stream.
static void WriteFunction(
- const Function &F, ValueEnumerator &VE, BitstreamWriter &Stream,
- DenseMap<const Function *, std::unique_ptr<FunctionInfo>> &FunctionIndex,
- bool EmitFunctionSummary) {
+ const Function &F, const Module *M, ValueEnumerator &VE,
+ BitstreamWriter &Stream,
+ DenseMap<const Function *, std::unique_ptr<GlobalValueInfo>> &FunctionIndex,
+ bool EmitSummaryIndex) {
// Save the bitcode index of the start of this function block for recording
// in the VST.
uint64_t BitcodeIndex = Stream.GetCurrentBitNo();
+ bool HasProfileData = F.getEntryCount().hasValue();
+ std::unique_ptr<BlockFrequencyInfo> BFI;
+ if (EmitSummaryIndex && HasProfileData) {
+ Function &Func = const_cast<Function &>(F);
+ LoopInfo LI{DominatorTree(Func)};
+ BranchProbabilityInfo BPI{Func, LI};
+ BFI = llvm::make_unique<BlockFrequencyInfo>(Func, BPI, LI);
+ }
+
Stream.EnterSubblock(bitc::FUNCTION_BLOCK_ID, 4);
VE.incorporateFunction(F);
@@ -2494,7 +2553,12 @@ static void WriteFunction(
DILocation *LastDL = nullptr;
unsigned NumInsts = 0;
+ // Map from callee ValueId to profile count. Used to accumulate profile
+ // counts for all static calls to a given callee.
+ DenseMap<unsigned, CalleeInfo> CallGraphEdges;
+ DenseSet<unsigned> RefEdges;
+ SmallPtrSet<const User *, 8> Visited;
// Finally, emit all the instructions, in order.
for (Function::const_iterator BB = F.begin(), E = F.end(); BB != E; ++BB)
for (BasicBlock::const_iterator I = BB->begin(), E = BB->end();
@@ -2507,6 +2571,24 @@ static void WriteFunction(
if (!I->getType()->isVoidTy())
++InstID;
+ if (EmitSummaryIndex) {
+ if (auto CS = ImmutableCallSite(&*I)) {
+ auto *CalledFunction = CS.getCalledFunction();
+ if (CalledFunction && CalledFunction->hasName() &&
+ !CalledFunction->isIntrinsic()) {
+ uint64_t ScaledCount = 0;
+ if (HasProfileData)
+ ScaledCount = getBlockProfileCount(
+ BFI->getBlockFreq(&(*BB)).getFrequency(), BFI->getEntryFreq(),
+ F.getEntryCount().getValue());
+ unsigned CalleeId = VE.getValueID(
+ M->getValueSymbolTable().lookup(CalledFunction->getName()));
+ CallGraphEdges[CalleeId] += ScaledCount;
+ }
+ }
+ findRefEdges(&*I, VE, RefEdges, Visited);
+ }
+
// If the instruction has metadata, write a metadata attachment later.
NeedsMetadataAttachment |= I->hasMetadataOtherThanDebugLoc();
@@ -2531,6 +2613,15 @@ static void WriteFunction(
LastDL = DL;
}
+ std::unique_ptr<FunctionSummary> FuncSummary;
+ if (EmitSummaryIndex) {
+ FuncSummary = llvm::make_unique<FunctionSummary>(F.getLinkage(), NumInsts);
+ FuncSummary->addCallGraphEdges(CallGraphEdges);
+ FuncSummary->addRefEdges(RefEdges);
+ }
+ FunctionIndex[&F] =
+ llvm::make_unique<GlobalValueInfo>(BitcodeIndex, std::move(FuncSummary));
+
// Emit names for all the instructions etc.
WriteValueSymbolTable(F.getValueSymbolTable(), VE, Stream);
@@ -2540,9 +2631,6 @@ static void WriteFunction(
WriteUseListBlock(&F, VE, Stream);
VE.purgeFunction();
Stream.ExitBlock();
-
- SaveFunctionInfo(F, FunctionIndex, NumInsts, BitcodeIndex,
- EmitFunctionSummary);
}
// Emit blockinfo, which defines the standard abbreviations etc.
@@ -2776,34 +2864,103 @@ static void WriteModStrings(const FunctionInfoIndex &I,
// Helper to emit a single function summary record.
static void WritePerModuleFunctionSummaryRecord(
- SmallVector<unsigned, 64> &NameVals, FunctionSummary *FS, unsigned ValueID,
- unsigned FSAbbrev, BitstreamWriter &Stream) {
+ SmallVector<uint64_t, 64> &NameVals, FunctionSummary *FS, unsigned ValueID,
+ unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev,
+ BitstreamWriter &Stream, const Function &F) {
assert(FS);
NameVals.push_back(ValueID);
- NameVals.push_back(getEncodedLinkage(FS->getFunctionLinkage()));
+ NameVals.push_back(getEncodedLinkage(FS->linkage()));
NameVals.push_back(FS->instCount());
+ NameVals.push_back(FS->refs().size());
+
+ for (auto &RI : FS->refs())
+ NameVals.push_back(RI);
+
+ bool HasProfileData = F.getEntryCount().hasValue();
+ for (auto &ECI : FS->edges()) {
+ NameVals.push_back(ECI.first);
+ assert(ECI.second.CallsiteCount > 0 && "Expected at least one callsite");
+ NameVals.push_back(ECI.second.CallsiteCount);
+ if (HasProfileData)
+ NameVals.push_back(ECI.second.ProfileCount);
+ }
+
+ unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev);
+ unsigned Code =
+ (HasProfileData ? bitc::FS_PERMODULE_PROFILE : bitc::FS_PERMODULE);
// Emit the finished record.
- Stream.EmitRecord(bitc::FS_CODE_PERMODULE_ENTRY, NameVals, FSAbbrev);
+ Stream.EmitRecord(Code, NameVals, FSAbbrev);
NameVals.clear();
}
-/// Emit the per-module function summary section alongside the rest of
+// Collect the global value references in the given variable's initializer,
+// and emit them in a summary record.
+static void WriteModuleLevelReferences(const GlobalVariable &V,
+ const ValueEnumerator &VE,
+ SmallVector<uint64_t, 64> &NameVals,
+ unsigned FSModRefsAbbrev,
+ BitstreamWriter &Stream) {
+ DenseSet<unsigned> RefEdges;
+ SmallPtrSet<const User *, 8> Visited;
+ findRefEdges(&V, VE, RefEdges, Visited);
+ unsigned RefCount = RefEdges.size();
+ if (RefCount) {
+ NameVals.push_back(VE.getValueID(&V));
+ NameVals.push_back(getEncodedLinkage(V.getLinkage()));
+ for (auto RefId : RefEdges) {
+ NameVals.push_back(RefId);
+ }
+ Stream.EmitRecord(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, NameVals,
+ FSModRefsAbbrev);
+ NameVals.clear();
+ }
+}
+
+/// Emit the per-module summary section alongside the rest of
/// the module's bitcode.
-static void WritePerModuleFunctionSummary(
- DenseMap<const Function *, std::unique_ptr<FunctionInfo>> &FunctionIndex,
+static void WritePerModuleGlobalValueSummary(
+ DenseMap<const Function *, std::unique_ptr<GlobalValueInfo>> &FunctionIndex,
const Module *M, const ValueEnumerator &VE, BitstreamWriter &Stream) {
- Stream.EnterSubblock(bitc::FUNCTION_SUMMARY_BLOCK_ID, 3);
+ if (M->empty())
+ return;
+
+ Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3);
- // Abbrev for FS_CODE_PERMODULE_ENTRY.
+ // Abbrev for FS_PERMODULE.
BitCodeAbbrev *Abbv = new BitCodeAbbrev();
- Abbv->Add(BitCodeAbbrevOp(bitc::FS_CODE_PERMODULE_ENTRY));
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE));
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
- unsigned FSAbbrev = Stream.EmitAbbrev(Abbv);
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
+ // numrefs x valueid, n x (valueid, callsitecount)
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+ unsigned FSCallsAbbrev = Stream.EmitAbbrev(Abbv);
- SmallVector<unsigned, 64> NameVals;
+ // Abbrev for FS_PERMODULE_PROFILE.
+ Abbv = new BitCodeAbbrev();
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
+ // numrefs x valueid, n x (valueid, callsitecount, profilecount)
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+ unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv);
+
+ // Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
+ Abbv = new BitCodeAbbrev();
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+ unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv);
+
+ SmallVector<uint64_t, 64> NameVals;
// Iterate over the list of functions instead of the FunctionIndex map to
// ensure the ordering is stable.
for (const Function &F : *M) {
@@ -2817,9 +2974,9 @@ static void WritePerModuleFunctionSummary(
assert(FunctionIndex.count(&F) == 1);
WritePerModuleFunctionSummaryRecord(
- NameVals, FunctionIndex[&F]->functionSummary(),
- VE.getValueID(M->getValueSymbolTable().lookup(F.getName())), FSAbbrev,
- Stream);
+ NameVals, dyn_cast<FunctionSummary>(FunctionIndex[&F]->summary()),
+ VE.getValueID(M->getValueSymbolTable().lookup(F.getName())),
+ FSCallsAbbrev, FSCallsProfileAbbrev, Stream, F);
}
for (const GlobalAlias &A : M->aliases()) {
@@ -2830,10 +2987,25 @@ static void WritePerModuleFunctionSummary(
continue;
assert(FunctionIndex.count(F) == 1);
+ FunctionSummary *FS =
+ dyn_cast<FunctionSummary>(FunctionIndex[F]->summary());
+ // Add the alias to the reference list of aliasee function.
+ FS->addRefEdge(
+ VE.getValueID(M->getValueSymbolTable().lookup(A.getName())));
WritePerModuleFunctionSummaryRecord(
- NameVals, FunctionIndex[F]->functionSummary(),
- VE.getValueID(M->getValueSymbolTable().lookup(A.getName())), FSAbbrev,
- Stream);
+ NameVals, FS,
+ VE.getValueID(M->getValueSymbolTable().lookup(A.getName())),
+ FSCallsAbbrev, FSCallsProfileAbbrev, Stream, *F);
+ }
+
+ // Capture references from GlobalVariable initializers, which are outside
+ // of a function scope.
+ for (const GlobalVariable &G : M->globals())
+ WriteModuleLevelReferences(G, VE, NameVals, FSModRefsAbbrev, Stream);
+ for (const GlobalAlias &A : M->aliases()) {
+ const auto *GV = dyn_cast<GlobalVariable>(A.getBaseObject());
+ if (GV)
+ WriteModuleLevelReferences(*GV, VE, NameVals, FSModRefsAbbrev, Stream);
}
Stream.ExitBlock();
@@ -2841,35 +3013,132 @@ static void WritePerModuleFunctionSummary(
/// Emit the combined function summary section into the combined index
/// file.
-static void WriteCombinedFunctionSummary(const FunctionInfoIndex &I,
- BitstreamWriter &Stream) {
- Stream.EnterSubblock(bitc::FUNCTION_SUMMARY_BLOCK_ID, 3);
+static void WriteCombinedGlobalValueSummary(
+ const FunctionInfoIndex &I, BitstreamWriter &Stream,
+ std::map<uint64_t, unsigned> &GUIDToValueIdMap, unsigned GlobalValueId) {
+ Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3);
- // Abbrev for FS_CODE_COMBINED_ENTRY.
+ // Abbrev for FS_COMBINED.
BitCodeAbbrev *Abbv = new BitCodeAbbrev();
- Abbv->Add(BitCodeAbbrevOp(bitc::FS_CODE_COMBINED_ENTRY));
- Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage
- Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
- unsigned FSAbbrev = Stream.EmitAbbrev(Abbv);
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
+ // numrefs x valueid, n x (valueid, callsitecount)
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+ unsigned FSCallsAbbrev = Stream.EmitAbbrev(Abbv);
- SmallVector<unsigned, 64> NameVals;
+ // Abbrev for FS_COMBINED_PROFILE.
+ Abbv = new BitCodeAbbrev();
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
+ // numrefs x valueid, n x (valueid, callsitecount, profilecount)
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+ unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv);
+
+ // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS.
+ Abbv = new BitCodeAbbrev();
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // linkage
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+ unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv);
+
+ SmallVector<uint64_t, 64> NameVals;
for (const auto &FII : I) {
for (auto &FI : FII.second) {
- FunctionSummary *FS = FI->functionSummary();
- assert(FS);
+ GlobalValueSummary *S = FI->summary();
+ assert(S);
+
+ if (auto *VS = dyn_cast<GlobalVarSummary>(S)) {
+ assert(!VS->refs().empty() && "Expected at least one ref edge");
+ NameVals.push_back(I.getModuleId(VS->modulePath()));
+ NameVals.push_back(getEncodedLinkage(VS->linkage()));
+ for (auto &RI : VS->refs()) {
+ const auto &VMI = GUIDToValueIdMap.find(RI);
+ unsigned RefId;
+ // If this GUID doesn't have an entry, assign one.
+ if (VMI == GUIDToValueIdMap.end()) {
+ GUIDToValueIdMap[RI] = ++GlobalValueId;
+ RefId = GlobalValueId;
+ } else {
+ RefId = VMI->second;
+ }
+ NameVals.push_back(RefId);
+ }
+
+ // Record the starting offset of this summary entry for use
+ // in the VST entry. Add the current code size since the
+ // reader will invoke readRecord after the abbrev id read.
+ FI->setBitcodeIndex(Stream.GetCurrentBitNo() +
+ Stream.GetAbbrevIDWidth());
+
+ // Emit the finished record.
+ Stream.EmitRecord(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, NameVals,
+ FSModRefsAbbrev);
+ NameVals.clear();
+ continue;
+ }
+ auto *FS = dyn_cast<FunctionSummary>(S);
+ assert(FS);
NameVals.push_back(I.getModuleId(FS->modulePath()));
- NameVals.push_back(getEncodedLinkage(FS->getFunctionLinkage()));
+ NameVals.push_back(getEncodedLinkage(FS->linkage()));
NameVals.push_back(FS->instCount());
+ NameVals.push_back(FS->refs().size());
+
+ for (auto &RI : FS->refs()) {
+ const auto &VMI = GUIDToValueIdMap.find(RI);
+ unsigned RefId;
+ // If this GUID doesn't have an entry, assign one.
+ if (VMI == GUIDToValueIdMap.end()) {
+ GUIDToValueIdMap[RI] = ++GlobalValueId;
+ RefId = GlobalValueId;
+ } else {
+ RefId = VMI->second;
+ }
+ NameVals.push_back(RefId);
+ }
+
+ bool HasProfileData = false;
+ for (auto &EI : FS->edges()) {
+ HasProfileData |= EI.second.ProfileCount != 0;
+ if (HasProfileData)
+ break;
+ }
+
+ for (auto &EI : FS->edges()) {
+ const auto &VMI = GUIDToValueIdMap.find(EI.first);
+ // If this GUID doesn't have an entry, it doesn't have a function
+ // summary and we don't need to record any calls to it.
+ if (VMI == GUIDToValueIdMap.end())
+ continue;
+ NameVals.push_back(VMI->second);
+ assert(EI.second.CallsiteCount > 0 && "Expected at least one callsite");
+ NameVals.push_back(EI.second.CallsiteCount);
+ if (HasProfileData)
+ NameVals.push_back(EI.second.ProfileCount);
+ }
// Record the starting offset of this summary entry for use
// in the VST entry. Add the current code size since the
// reader will invoke readRecord after the abbrev id read.
FI->setBitcodeIndex(Stream.GetCurrentBitNo() + Stream.GetAbbrevIDWidth());
+ unsigned FSAbbrev =
+ (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev);
+ unsigned Code =
+ (HasProfileData ? bitc::FS_COMBINED_PROFILE : bitc::FS_COMBINED);
+
// Emit the finished record.
- Stream.EmitRecord(bitc::FS_CODE_COMBINED_ENTRY, NameVals, FSAbbrev);
+ Stream.EmitRecord(Code, NameVals, FSAbbrev);
NameVals.clear();
}
}
@@ -2904,7 +3173,7 @@ static void WriteIdentificationBlock(const Module *M, BitstreamWriter &Stream) {
/// WriteModule - Emit the specified module to the bitstream.
static void WriteModule(const Module *M, BitstreamWriter &Stream,
bool ShouldPreserveUseListOrder,
- uint64_t BitcodeStartBit, bool EmitFunctionSummary) {
+ uint64_t BitcodeStartBit, bool EmitSummaryIndex) {
Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3);
SmallVector<unsigned, 1> Vals;
@@ -2949,15 +3218,15 @@ static void WriteModule(const Module *M, BitstreamWriter &Stream,
WriteOperandBundleTags(M, Stream);
// Emit function bodies.
- DenseMap<const Function *, std::unique_ptr<FunctionInfo>> FunctionIndex;
+ DenseMap<const Function *, std::unique_ptr<GlobalValueInfo>> FunctionIndex;
for (Module::const_iterator F = M->begin(), E = M->end(); F != E; ++F)
if (!F->isDeclaration())
- WriteFunction(*F, VE, Stream, FunctionIndex, EmitFunctionSummary);
+ WriteFunction(*F, M, VE, Stream, FunctionIndex, EmitSummaryIndex);
// Need to write after the above call to WriteFunction which populates
// the summary information in the index.
- if (EmitFunctionSummary)
- WritePerModuleFunctionSummary(FunctionIndex, M, VE, Stream);
+ if (EmitSummaryIndex)
+ WritePerModuleGlobalValueSummary(FunctionIndex, M, VE, Stream);
WriteValueSymbolTable(M->getValueSymbolTable(), VE, Stream,
VSTOffsetPlaceholder, BitcodeStartBit, &FunctionIndex);
@@ -3046,7 +3315,7 @@ static void WriteBitcodeHeader(BitstreamWriter &Stream) {
/// stream.
void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out,
bool ShouldPreserveUseListOrder,
- bool EmitFunctionSummary) {
+ bool EmitSummaryIndex) {
SmallVector<char, 0> Buffer;
Buffer.reserve(256*1024);
@@ -3072,7 +3341,7 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out,
// Emit the module.
WriteModule(M, Stream, ShouldPreserveUseListOrder, BitcodeStartBit,
- EmitFunctionSummary);
+ EmitSummaryIndex);
}
if (TT.isOSDarwin() || TT.isOSBinFormatMachO())
@@ -3082,11 +3351,10 @@ void llvm::WriteBitcodeToFile(const Module *M, raw_ostream &Out,
Out.write((char*)&Buffer.front(), Buffer.size());
}
-// Write the specified function summary index to the given raw output stream,
+// Write the specified module summary index to the given raw output stream,
// where it will be written in a new bitcode block. This is used when
// writing the combined index file for ThinLTO.
-void llvm::WriteFunctionSummaryToFile(const FunctionInfoIndex &Index,
- raw_ostream &Out) {
+void llvm::WriteIndexToFile(const FunctionInfoIndex &Index, raw_ostream &Out) {
SmallVector<char, 0> Buffer;
Buffer.reserve(256 * 1024);
@@ -3102,15 +3370,30 @@ void llvm::WriteFunctionSummaryToFile(const FunctionInfoIndex &Index,
Vals.push_back(CurVersion);
Stream.EmitRecord(bitc::MODULE_CODE_VERSION, Vals);
+ // If we have a VST, write the VSTOFFSET record placeholder and record
+ // its offset.
+ uint64_t VSTOffsetPlaceholder = WriteValueSymbolTableForwardDecl(Stream);
+
// Write the module paths in the combined index.
WriteModStrings(Index, Stream);
- // Write the function summary combined index records.
- WriteCombinedFunctionSummary(Index, Stream);
+ // Assign unique value ids to all functions in the index for use
+ // in writing out the call graph edges. Save the mapping from GUID
+ // to the new global value id to use when writing those edges, which
+ // are currently saved in the index in terms of GUID.
+ std::map<uint64_t, unsigned> GUIDToValueIdMap;
+ unsigned GlobalValueId = 0;
+ for (auto &II : Index)
+ GUIDToValueIdMap[II.first] = ++GlobalValueId;
+
+ // Write the summary combined index records.
+ WriteCombinedGlobalValueSummary(Index, Stream, GUIDToValueIdMap,
+ GlobalValueId);
// Need a special VST writer for the combined index (we don't have a
// real VST and real values when this is invoked).
- WriteCombinedValueSymbolTable(Index, Stream);
+ WriteCombinedValueSymbolTable(Index, Stream, GUIDToValueIdMap,
+ VSTOffsetPlaceholder);
Stream.ExitBlock();
diff --git a/llvm/lib/Bitcode/Writer/LLVMBuild.txt b/llvm/lib/Bitcode/Writer/LLVMBuild.txt
index 7d9e1de771b..a450b38fba2 100644
--- a/llvm/lib/Bitcode/Writer/LLVMBuild.txt
+++ b/llvm/lib/Bitcode/Writer/LLVMBuild.txt
@@ -19,4 +19,4 @@
type = Library
name = BitWriter
parent = Bitcode
-required_libraries = Core Support
+required_libraries = Analysis Core Support
OpenPOWER on IntegriCloud