summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/Analysis/ModuleSummaryAnalysis.cpp75
-rw-r--r--llvm/lib/AsmParser/LLParser.cpp3
-rw-r--r--llvm/lib/Bitcode/Reader/BitcodeReader.cpp51
-rw-r--r--llvm/lib/Bitcode/Writer/BitcodeWriter.cpp22
-rw-r--r--llvm/lib/IR/ModuleSummaryIndex.cpp107
-rw-r--r--llvm/lib/LTO/LTO.cpp5
-rw-r--r--llvm/lib/LTO/ThinLTOCodeGenerator.cpp3
-rw-r--r--llvm/lib/Linker/IRMover.cpp5
-rw-r--r--llvm/lib/Transforms/IPO/FunctionImport.cpp45
-rw-r--r--llvm/lib/Transforms/Utils/FunctionImportUtils.cpp21
10 files changed, 289 insertions, 48 deletions
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index 29b96ac746b..af2faa0dc6f 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -220,10 +220,19 @@ static void addIntrinsicToSummary(
}
}
-static void computeFunctionSummary(
- ModuleSummaryIndex &Index, const Module &M, const Function &F,
- BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, DominatorTree &DT,
- bool HasLocalsInUsedOrAsm, DenseSet<GlobalValue::GUID> &CantBePromoted) {
+static bool isNonVolatileLoad(const Instruction *I) {
+ if (const auto *LI = dyn_cast<LoadInst>(I))
+ return !LI->isVolatile();
+
+ return false;
+}
+
+static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
+ const Function &F, BlockFrequencyInfo *BFI,
+ ProfileSummaryInfo *PSI, DominatorTree &DT,
+ bool HasLocalsInUsedOrAsm,
+ DenseSet<GlobalValue::GUID> &CantBePromoted,
+ bool IsThinLTO) {
// Summary not currently supported for anonymous functions, they should
// have been named.
assert(F.hasName());
@@ -244,6 +253,7 @@ static void computeFunctionSummary(
// Add personality function, prefix data and prologue data to function's ref
// list.
findRefEdges(Index, &F, RefEdges, Visited);
+ std::vector<const Instruction *> NonVolatileLoads;
bool HasInlineAsmMaybeReferencingInternal = false;
for (const BasicBlock &BB : F)
@@ -251,6 +261,13 @@ static void computeFunctionSummary(
if (isa<DbgInfoIntrinsic>(I))
continue;
++NumInsts;
+ if (isNonVolatileLoad(&I)) {
+ // Postpone processing of non-volatile load instructions
+ // See comments below
+ Visited.insert(&I);
+ NonVolatileLoads.push_back(&I);
+ continue;
+ }
findRefEdges(Index, &I, RefEdges, Visited);
auto CS = ImmutableCallSite(&I);
if (!CS)
@@ -340,6 +357,24 @@ static void computeFunctionSummary(
}
}
+ // By now we processed all instructions in a function, except
+ // non-volatile loads. All new refs we add in a loop below
+ // are obviously constant. All constant refs are grouped in the
+ // end of RefEdges vector, so we can use a single integer value
+ // to identify them.
+ unsigned RefCnt = RefEdges.size();
+ for (const Instruction *I : NonVolatileLoads) {
+ Visited.erase(I);
+ findRefEdges(Index, I, RefEdges, Visited);
+ }
+ std::vector<ValueInfo> Refs = RefEdges.takeVector();
+ // Regular LTO module doesn't participate in ThinLTO import,
+ // so no reference from it can be readonly, since this would
+ // require importing variable as local copy
+ if (IsThinLTO)
+ for (; RefCnt < Refs.size(); ++RefCnt)
+ Refs[RefCnt].setReadOnly();
+
// Explicit add hot edges to enforce importing for designated GUIDs for
// sample PGO, to enable the same inlines as the profiled optimized binary.
for (auto &I : F.getImportGUIDs())
@@ -363,9 +398,9 @@ static void computeFunctionSummary(
// Don't try to import functions with noinline attribute.
F.getAttributes().hasFnAttribute(Attribute::NoInline)};
auto FuncSummary = llvm::make_unique<FunctionSummary>(
- Flags, NumInsts, FunFlags, RefEdges.takeVector(),
- CallGraphEdges.takeVector(), TypeTests.takeVector(),
- TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(),
+ Flags, NumInsts, FunFlags, std::move(Refs), CallGraphEdges.takeVector(),
+ TypeTests.takeVector(), TypeTestAssumeVCalls.takeVector(),
+ TypeCheckedLoadVCalls.takeVector(),
TypeTestAssumeConstVCalls.takeVector(),
TypeCheckedLoadConstVCalls.takeVector());
if (NonRenamableLocal)
@@ -382,8 +417,13 @@ computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V,
bool NonRenamableLocal = isNonRenamableLocal(V);
GlobalValueSummary::GVFlags Flags(V.getLinkage(), NonRenamableLocal,
/* Live = */ false, V.isDSOLocal());
- auto GVarSummary =
- llvm::make_unique<GlobalVarSummary>(Flags, RefEdges.takeVector());
+
+ // Don't mark variables we won't be able to internalize as read-only.
+ GlobalVarSummary::GVarFlags VarFlags(
+ !V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() &&
+ !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass());
+ auto GVarSummary = llvm::make_unique<GlobalVarSummary>(Flags, VarFlags,
+ RefEdges.takeVector());
if (NonRenamableLocal)
CantBePromoted.insert(V.getGUID());
if (HasBlockAddress)
@@ -487,13 +527,19 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
Index.addGlobalValueSummary(*GV, std::move(Summary));
} else {
std::unique_ptr<GlobalVarSummary> Summary =
- llvm::make_unique<GlobalVarSummary>(GVFlags,
- ArrayRef<ValueInfo>{});
+ llvm::make_unique<GlobalVarSummary>(
+ GVFlags, GlobalVarSummary::GVarFlags(),
+ ArrayRef<ValueInfo>{});
Index.addGlobalValueSummary(*GV, std::move(Summary));
}
});
}
+ bool IsThinLTO = true;
+ if (auto *MD =
+ mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
+ IsThinLTO = MD->getZExtValue();
+
// Compute summaries for all functions defined in module, and save in the
// index.
for (auto &F : M) {
@@ -514,7 +560,7 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
computeFunctionSummary(Index, M, F, BFI, PSI, DT,
!LocalsUsed.empty() || HasLocalInlineAsmSymbol,
- CantBePromoted);
+ CantBePromoted, IsThinLTO);
}
// Compute summaries for all variables defined in module, and save in the
@@ -545,11 +591,6 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
setLiveRoot(Index, "llvm.global_dtors");
setLiveRoot(Index, "llvm.global.annotations");
- bool IsThinLTO = true;
- if (auto *MD =
- mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
- IsThinLTO = MD->getZExtValue();
-
for (auto &GlobalList : Index) {
// Ignore entries for references that are undefined in the current module.
if (GlobalList.second.SummaryList.empty())
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 5fe1e125d48..57f73244acb 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -7642,7 +7642,8 @@ bool LLParser::ParseVariableSummary(std::string Name, GlobalValue::GUID GUID,
if (ParseToken(lltok::rparen, "expected ')' here"))
return true;
- auto GS = llvm::make_unique<GlobalVarSummary>(GVFlags, std::move(Refs));
+ auto GS = llvm::make_unique<GlobalVarSummary>(
+ GVFlags, GlobalVarSummary::GVarFlags(), std::move(Refs));
GS->setModulePath(ModulePath);
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 56e05f8f085..41acb9f3234 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -898,6 +898,11 @@ static GlobalValueSummary::GVFlags getDecodedGVSummaryFlags(uint64_t RawFlags,
return GlobalValueSummary::GVFlags(Linkage, NotEligibleToImport, Live, Local);
}
+// Decode the flags for GlobalVariable in the summary
+static GlobalVarSummary::GVarFlags getDecodedGVarFlags(uint64_t RawFlags) {
+ return GlobalVarSummary::GVarFlags((RawFlags & 0x1) ? true : false);
+}
+
static GlobalValue::VisibilityTypes getDecodedVisibility(unsigned Val) {
switch (Val) {
default: // Map unknown visibilities to default.
@@ -5170,6 +5175,12 @@ static void parseTypeIdSummaryRecord(ArrayRef<uint64_t> Record,
parseWholeProgramDevirtResolution(Record, Strtab, Slot, TypeId);
}
+static void setImmutableRefs(std::vector<ValueInfo> &Refs, unsigned Count) {
+ // Read-only refs are in the end of the refs list.
+ for (unsigned RefNo = Refs.size() - Count; RefNo < Refs.size(); ++RefNo)
+ Refs[RefNo].setReadOnly();
+}
+
// Eagerly parse the entire summary block. This populates the GlobalValueSummary
// objects in the index.
Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
@@ -5187,9 +5198,9 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
}
const uint64_t Version = Record[0];
const bool IsOldProfileFormat = Version == 1;
- if (Version < 1 || Version > 4)
+ if (Version < 1 || Version > 5)
return error("Invalid summary version " + Twine(Version) +
- ", 1, 2, 3 or 4 expected");
+ ", 1, 2, 3, 4 or 5 expected");
Record.clear();
// Keep around the last seen summary to be used when we see an optional
@@ -5268,11 +5279,16 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
unsigned InstCount = Record[2];
uint64_t RawFunFlags = 0;
unsigned NumRefs = Record[3];
+ unsigned NumImmutableRefs = 0;
int RefListStartIndex = 4;
if (Version >= 4) {
RawFunFlags = Record[3];
NumRefs = Record[4];
RefListStartIndex = 5;
+ if (Version >= 5) {
+ NumImmutableRefs = Record[5];
+ RefListStartIndex = 6;
+ }
}
auto Flags = getDecodedGVSummaryFlags(RawFlags, Version);
@@ -5291,6 +5307,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
std::vector<FunctionSummary::EdgeTy> Calls = makeCallList(
ArrayRef<uint64_t>(Record).slice(CallGraphEdgeStartIndex),
IsOldProfileFormat, HasProfile, HasRelBF);
+ setImmutableRefs(Refs, NumImmutableRefs);
auto FS = llvm::make_unique<FunctionSummary>(
Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs),
std::move(Calls), std::move(PendingTypeTests),
@@ -5339,14 +5356,21 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
TheIndex.addGlobalValueSummary(GUID.first, std::move(AS));
break;
}
- // FS_PERMODULE_GLOBALVAR_INIT_REFS: [valueid, flags, n x valueid]
+ // FS_PERMODULE_GLOBALVAR_INIT_REFS: [valueid, flags, varflags, n x valueid]
case bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS: {
unsigned ValueID = Record[0];
uint64_t RawFlags = Record[1];
+ unsigned RefArrayStart = 2;
+ GlobalVarSummary::GVarFlags GVF;
auto Flags = getDecodedGVSummaryFlags(RawFlags, Version);
+ if (Version >= 5) {
+ GVF = getDecodedGVarFlags(Record[2]);
+ RefArrayStart = 3;
+ }
std::vector<ValueInfo> Refs =
- makeRefList(ArrayRef<uint64_t>(Record).slice(2));
- auto FS = llvm::make_unique<GlobalVarSummary>(Flags, std::move(Refs));
+ makeRefList(ArrayRef<uint64_t>(Record).slice(RefArrayStart));
+ auto FS =
+ llvm::make_unique<GlobalVarSummary>(Flags, GVF, std::move(Refs));
FS->setModulePath(getThisModule()->first());
auto GUID = getValueInfoFromValueId(ValueID);
FS->setOriginalName(GUID.second);
@@ -5365,12 +5389,17 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
unsigned InstCount = Record[3];
uint64_t RawFunFlags = 0;
unsigned NumRefs = Record[4];
+ unsigned NumImmutableRefs = 0;
int RefListStartIndex = 5;
if (Version >= 4) {
RawFunFlags = Record[4];
NumRefs = Record[5];
RefListStartIndex = 6;
+ if (Version >= 5) {
+ NumImmutableRefs = Record[6];
+ RefListStartIndex = 7;
+ }
}
auto Flags = getDecodedGVSummaryFlags(RawFlags, Version);
@@ -5384,6 +5413,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
ArrayRef<uint64_t>(Record).slice(CallGraphEdgeStartIndex),
IsOldProfileFormat, HasProfile, false);
ValueInfo VI = getValueInfoFromValueId(ValueID).first;
+ setImmutableRefs(Refs, NumImmutableRefs);
auto FS = llvm::make_unique<FunctionSummary>(
Flags, InstCount, getDecodedFFlags(RawFunFlags), std::move(Refs),
std::move(Edges), std::move(PendingTypeTests),
@@ -5432,10 +5462,17 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
unsigned ValueID = Record[0];
uint64_t ModuleId = Record[1];
uint64_t RawFlags = Record[2];
+ unsigned RefArrayStart = 3;
+ GlobalVarSummary::GVarFlags GVF;
auto Flags = getDecodedGVSummaryFlags(RawFlags, Version);
+ if (Version >= 5) {
+ GVF = getDecodedGVarFlags(Record[3]);
+ RefArrayStart = 4;
+ }
std::vector<ValueInfo> Refs =
- makeRefList(ArrayRef<uint64_t>(Record).slice(3));
- auto FS = llvm::make_unique<GlobalVarSummary>(Flags, std::move(Refs));
+ makeRefList(ArrayRef<uint64_t>(Record).slice(RefArrayStart));
+ auto FS =
+ llvm::make_unique<GlobalVarSummary>(Flags, GVF, std::move(Refs));
LastSeenSummary = FS.get();
FS->setModulePath(ModuleIdMap[ModuleId]);
ValueInfo VI = getValueInfoFromValueId(ValueID).first;
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index f4634c9d3f4..89a20dd1c67 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -991,6 +991,11 @@ static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) {
return RawFlags;
}
+static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) {
+ uint64_t RawFlags = Flags.ReadOnly;
+ return RawFlags;
+}
+
static unsigned getEncodedVisibility(const GlobalValue &GV) {
switch (GV.getVisibility()) {
case GlobalValue::DefaultVisibility: return 0;
@@ -3489,6 +3494,7 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
NameVals.push_back(FS->instCount());
NameVals.push_back(getEncodedFFlags(FS->fflags()));
NameVals.push_back(FS->refs().size());
+ NameVals.push_back(FS->immutableRefCount());
for (auto &RI : FS->refs())
NameVals.push_back(VE.getValueID(RI.getValue()));
@@ -3530,6 +3536,7 @@ void ModuleBitcodeWriterBase::writeModuleLevelReferences(
NameVals.push_back(VE.getValueID(&V));
GlobalVarSummary *VS = cast<GlobalVarSummary>(Summary);
NameVals.push_back(getEncodedGVSummaryFlags(VS->flags()));
+ NameVals.push_back(getEncodedGVarFlags(VS->varflags()));
unsigned SizeBeforeRefs = NameVals.size();
for (auto &RI : VS->refs())
@@ -3546,7 +3553,7 @@ void ModuleBitcodeWriterBase::writeModuleLevelReferences(
// Current version for the summary.
// This is bumped whenever we introduce changes in the way some record are
// interpreted, like flags for instance.
-static const uint64_t INDEX_VERSION = 4;
+static const uint64_t INDEX_VERSION = 5;
/// Emit the per-module summary section alongside the rest of
/// the module's bitcode.
@@ -3581,6 +3588,7 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt
// numrefs x valueid, n x (valueid, hotness)
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
@@ -3597,6 +3605,7 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt
// numrefs x valueid, n x (valueid [, rel_block_freq])
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
@@ -3691,6 +3700,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt
// numrefs x valueid, n x (valueid)
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
@@ -3705,6 +3715,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt
// numrefs x valueid, n x (valueid, hotness)
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
@@ -3777,6 +3788,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
NameVals.push_back(*ValueId);
NameVals.push_back(Index.getModuleId(VS->modulePath()));
NameVals.push_back(getEncodedGVSummaryFlags(VS->flags()));
+ NameVals.push_back(getEncodedGVarFlags(VS->varflags()));
for (auto &RI : VS->refs()) {
auto RefValueId = getValueId(RI.getGUID());
if (!RefValueId)
@@ -3802,17 +3814,21 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
NameVals.push_back(FS->instCount());
NameVals.push_back(getEncodedFFlags(FS->fflags()));
// Fill in below
- NameVals.push_back(0);
+ NameVals.push_back(0); // numrefs
+ NameVals.push_back(0); // immutablerefcnt
- unsigned Count = 0;
+ unsigned Count = 0, ImmutableRefCnt = 0;
for (auto &RI : FS->refs()) {
auto RefValueId = getValueId(RI.getGUID());
if (!RefValueId)
continue;
NameVals.push_back(*RefValueId);
+ if (RI.isReadOnly())
+ ImmutableRefCnt++;
Count++;
}
NameVals[5] = Count;
+ NameVals[6] = ImmutableRefCnt;
bool HasProfileData = false;
for (auto &EI : FS->calls()) {
diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp
index 8d85f7901b0..6c0d3973715 100644
--- a/llvm/lib/IR/ModuleSummaryIndex.cpp
+++ b/llvm/lib/IR/ModuleSummaryIndex.cpp
@@ -30,6 +30,17 @@ bool ValueInfo::isDSOLocal() const {
});
}
+// Gets the number of immutable refs in RefEdgeList
+unsigned FunctionSummary::immutableRefCount() const {
+ // Here we take advantage of having all readonly references
+ // located in the end of the RefEdgeList.
+ auto Refs = refs();
+ unsigned ImmutableRefCnt = 0;
+ for (int I = Refs.size() - 1; I >= 0 && Refs[I].isReadOnly(); --I)
+ ImmutableRefCnt++;
+ return ImmutableRefCnt;
+}
+
// Collect for the given module the list of function it defines
// (GUID -> Summary).
void ModuleSummaryIndex::collectDefinedFunctionsForModule(
@@ -84,6 +95,73 @@ bool ModuleSummaryIndex::isGUIDLive(GlobalValue::GUID GUID) const {
return false;
}
+static void propagateConstantsToRefs(GlobalValueSummary *S) {
+ // If reference is not readonly then referenced summary is not
+ // readonly either. Note that:
+ // - All references from GlobalVarSummary are conservatively considered as
+ // not readonly. Tracking them properly requires more complex analysis
+ // then we have now.
+ //
+ // - AliasSummary objects have no refs at all so this function is a no-op
+ // for them.
+ for (auto &VI : S->refs()) {
+ if (VI.isReadOnly()) {
+ // We only mark refs as readonly when computing function summaries on
+ // analysis phase.
+ assert(isa<FunctionSummary>(S));
+ continue;
+ }
+ for (auto &Ref : VI.getSummaryList())
+ // If references to alias is not readonly then aliasee is not readonly
+ if (auto *GVS = dyn_cast<GlobalVarSummary>(Ref->getBaseObject()))
+ GVS->setReadOnly(false);
+ }
+}
+
+// Do the constant propagation in combined index.
+// The goal of constant propagation is internalization of readonly
+// variables. To determine which variables are readonly and which
+// are not we take following steps:
+// - During analysis we speculatively assign readonly attribute to
+// all variables which can be internalized. When computing function
+// summary we also assign readonly attribute to a reference if
+// function doesn't modify referenced variable.
+//
+// - After computing dead symbols in combined index we do the constant
+// propagation. During this step we clear readonly attribute from
+// all variables which:
+// a. are dead, preserved or can't be imported
+// b. referenced by any global variable initializer
+// c. referenced by a function and reference is not readonly
+//
+// Internalization itself happens in the backend after import is finished
+// See internalizeImmutableGVs.
+void ModuleSummaryIndex::propagateConstants(
+ const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
+ for (auto &P : *this)
+ for (auto &S : P.second.SummaryList) {
+ if (!isGlobalValueLive(S.get()))
+ // We don't examine references from dead objects
+ continue;
+
+ // Global variable can't be marked read only if it is not eligible
+ // to import since we need to ensure that all external references
+ // get a local (imported) copy. It also can't be marked read only
+ // if it or any alias (since alias points to the same memory) are
+ // preserved or notEligibleToImport, since either of those means
+ // there could be writes that are not visible (because preserved
+ // means it could have external to DSO writes, and notEligibleToImport
+ // means it could have writes via inline assembly leading it to be
+ // in the @llvm.*used).
+ if (auto *GVS = dyn_cast<GlobalVarSummary>(S->getBaseObject()))
+ // Here we intentionally pass S.get() not GVS, because S could be
+ // an alias.
+ if (!canImportGlobalVar(S.get()) || GUIDPreservedSymbols.count(P.first))
+ GVS->setReadOnly(false);
+ propagateConstantsToRefs(S.get());
+ }
+}
+
// TODO: write a graphviz dumper for SCCs (see ModuleSummaryIndex::exportToDot)
// then delete this function and update its tests
LLVM_DUMP_METHOD
@@ -108,6 +186,7 @@ namespace {
struct Attributes {
void add(const Twine &Name, const Twine &Value,
const Twine &Comment = Twine());
+ void addComment(const Twine &Comment);
std::string getAsString() const;
std::vector<std::string> Attrs;
@@ -129,6 +208,10 @@ void Attributes::add(const Twine &Name, const Twine &Value,
A += Value.str();
A += "\"";
Attrs.push_back(A);
+ addComment(Comment);
+}
+
+void Attributes::addComment(const Twine &Comment) {
if (!Comment.isTriviallyEmpty()) {
if (Comments.empty())
Comments = " // ";
@@ -237,6 +320,12 @@ static void defineExternalNode(raw_ostream &OS, const char *Pfx,
OS << "\"]; // defined externally\n";
}
+static bool hasReadOnlyFlag(const GlobalValueSummary *S) {
+ if (auto *GVS = dyn_cast<GlobalVarSummary>(S))
+ return GVS->isReadOnly();
+ return false;
+}
+
void ModuleSummaryIndex::exportToDot(raw_ostream &OS) const {
std::vector<Edge> CrossModuleEdges;
DenseMap<GlobalValue::GUID, std::vector<uint64_t>> NodeMap;
@@ -252,13 +341,17 @@ void ModuleSummaryIndex::exportToDot(raw_ostream &OS) const {
};
auto DrawEdge = [&](const char *Pfx, uint64_t SrcMod, GlobalValue::GUID SrcId,
- uint64_t DstMod, GlobalValue::GUID DstId, int TypeOrHotness) {
- // 0 corresponds to alias edge, 1 to ref edge, 2 to call with unknown
- // hotness, ...
- TypeOrHotness += 2;
+ uint64_t DstMod, GlobalValue::GUID DstId,
+ int TypeOrHotness) {
+ // 0 - alias
+ // 1 - reference
+ // 2 - constant reference
+ // Other value: (hotness - 3).
+ TypeOrHotness += 3;
static const char *EdgeAttrs[] = {
" [style=dotted]; // alias",
" [style=dashed]; // ref",
+ " [style=dashed,color=forestgreen]; // const-ref",
" // call (hotness : Unknown)",
" [color=blue]; // call (hotness : Cold)",
" // call (hotness : None)",
@@ -301,6 +394,8 @@ void ModuleSummaryIndex::exportToDot(raw_ostream &OS) const {
A.add("shape", "box");
} else {
A.add("shape", "Mrecord", "variable");
+ if (Flags.Live && hasReadOnlyFlag(SummaryIt.second))
+ A.addComment("immutable");
}
auto VI = getValueInfo(SummaryIt.first);
@@ -318,7 +413,7 @@ void ModuleSummaryIndex::exportToDot(raw_ostream &OS) const {
for (auto &SummaryIt : GVSMap) {
auto *GVS = SummaryIt.second;
for (auto &R : GVS->refs())
- Draw(SummaryIt.first, R.getGUID(), -1);
+ Draw(SummaryIt.first, R.getGUID(), R.isReadOnly() ? -1 : -2);
if (auto *AS = dyn_cast_or_null<AliasSummary>(SummaryIt.second)) {
GlobalValue::GUID AliaseeId;
@@ -331,7 +426,7 @@ void ModuleSummaryIndex::exportToDot(raw_ostream &OS) const {
AliaseeId = AliaseeOrigId;
}
- Draw(SummaryIt.first, AliaseeId, -2);
+ Draw(SummaryIt.first, AliaseeId, -3);
continue;
}
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 6b11f690ef3..68fd93a5d9c 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -187,6 +187,8 @@ static void computeCacheKey(
AddUnsigned(VI.isDSOLocal());
AddUsedCfiGlobal(VI.getGUID());
}
+ if (auto *GVS = dyn_cast<GlobalVarSummary>(GS))
+ AddUnsigned(GVS->isReadOnly());
if (auto *FS = dyn_cast<FunctionSummary>(GS)) {
for (auto &TT : FS->type_tests())
UsedTypeIds.insert(TT);
@@ -809,7 +811,8 @@ Error LTO::run(AddStreamFn AddStream, NativeObjectCache Cache) {
return PrevailingType::Unknown;
return It->second;
};
- computeDeadSymbols(ThinLTO.CombinedIndex, GUIDPreservedSymbols, isPrevailing);
+ computeDeadSymbolsWithConstProp(ThinLTO.CombinedIndex, GUIDPreservedSymbols,
+ isPrevailing, Conf.OptLevel > 0);
// Setup output file to emit statistics.
std::unique_ptr<ToolOutputFile> StatsFile = nullptr;
diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 8017527bf22..b817c462d4e 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -646,7 +646,8 @@ static void computeDeadSymbolsInIndex(
auto isPrevailing = [&](GlobalValue::GUID G) {
return PrevailingType::Unknown;
};
- computeDeadSymbols(Index, GUIDPreservedSymbols, isPrevailing);
+ computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
+ /* ImportEnabled = */ true);
}
/**
diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp
index b304bfc401a..72e20ae0ba1 100644
--- a/llvm/lib/Linker/IRMover.cpp
+++ b/llvm/lib/Linker/IRMover.cpp
@@ -1062,11 +1062,6 @@ void IRLinker::prepareCompileUnitsForImport() {
ValueMap.MD()[CU->getRawEnumTypes()].reset(nullptr);
ValueMap.MD()[CU->getRawMacros()].reset(nullptr);
ValueMap.MD()[CU->getRawRetainedTypes()].reset(nullptr);
- // We import global variables only temporarily in order for instcombine
- // and globalopt to perform constant folding and static constructor
- // evaluation. After that elim-avail-extern will covert imported globals
- // back to declarations, so we don't need debug info for them.
- ValueMap.MD()[CU->getRawGlobalVariables()].reset(nullptr);
// Imported entities only need to be mapped in if they have local
// scope, as those might correspond to an imported entity inside a
diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp
index 1196dd0099b..60ca8e32897 100644
--- a/llvm/lib/Transforms/IPO/FunctionImport.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp
@@ -294,10 +294,8 @@ static void computeImportForReferencedGlobals(
LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n");
for (auto &RefSummary : VI.getSummaryList())
- if (RefSummary->getSummaryKind() == GlobalValueSummary::GlobalVarKind &&
- !RefSummary->notEligibleToImport() &&
- !GlobalValue::isInterposableLinkage(RefSummary->linkage()) &&
- RefSummary->refs().empty()) {
+ if (isa<GlobalVarSummary>(RefSummary.get()) &&
+ canImportGlobalVar(RefSummary.get())) {
auto ILI = ImportList[RefSummary->modulePath()].insert(VI.getGUID());
// Only update stat if we haven't already imported this variable.
if (ILI.second)
@@ -824,6 +822,25 @@ void llvm::computeDeadSymbols(
NumLiveSymbols += LiveSymbols;
}
+// Compute dead symbols and propagate constants in combined index.
+void llvm::computeDeadSymbolsWithConstProp(
+ ModuleSummaryIndex &Index,
+ const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
+ function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing,
+ bool ImportEnabled) {
+ computeDeadSymbols(Index, GUIDPreservedSymbols, isPrevailing);
+ if (ImportEnabled) {
+ Index.propagateConstants(GUIDPreservedSymbols);
+ } else {
+ // If import is disabled we should drop read-only attribute
+ // from all summaries to prevent internalization.
+ for (auto &P : Index)
+ for (auto &S : P.second.SummaryList)
+ if (auto *GVS = dyn_cast<GlobalVarSummary>(S.get()))
+ GVS->setReadOnly(false);
+ }
+}
+
/// Compute the set of summaries needed for a ThinLTO backend compilation of
/// \p ModulePath.
void llvm::gatherImportedSummariesForModule(
@@ -1020,6 +1037,22 @@ static Function *replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA) {
return NewFn;
}
+// Internalize values that we marked with specific attribute
+// in processGlobalForThinLTO.
+static void internalizeImmutableGVs(Module &M) {
+ for (auto &GV : M.globals()) {
+ // Skip GVs which have been converted to declarations
+ // by dropDeadSymbols.
+ if (GV.isDeclaration())
+ continue;
+ if (auto *GVar = dyn_cast<GlobalVariable>(&GV))
+ if (GVar->hasAttribute("thinlto-internalize")) {
+ GVar->setLinkage(GlobalValue::InternalLinkage);
+ GVar->setVisibility(GlobalValue::DefaultVisibility);
+ }
+ }
+}
+
// Automatically import functions in Module \p DestModule based on the summaries
// index.
Expected<bool> FunctionImporter::importFunctions(
@@ -1143,6 +1176,8 @@ Expected<bool> FunctionImporter::importFunctions(
NumImportedModules++;
}
+ internalizeImmutableGVs(DestModule);
+
NumImportedFunctions += (ImportedCount - ImportedGVCount);
NumImportedGlobalVars += ImportedGVCount;
@@ -1159,7 +1194,7 @@ static bool doImportingForModule(Module &M) {
if (SummaryFile.empty())
report_fatal_error("error: -function-import requires -summary-file\n");
Expected<std::unique_ptr<ModuleSummaryIndex>> IndexPtrOrErr =
- getModuleSummaryIndexForFile(SummaryFile);
+ getModuleSummaryIndexForFile(SummaryFile);
if (!IndexPtrOrErr) {
logAllUnhandledErrors(IndexPtrOrErr.takeError(), errs(),
"Error loading file '" + SummaryFile + "': ");
diff --git a/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp b/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp
index 479816a339d..89e74da4d99 100644
--- a/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp
+++ b/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp
@@ -204,8 +204,9 @@ void FunctionImportGlobalProcessing::processGlobalForThinLTO(GlobalValue &GV) {
// Check the summaries to see if the symbol gets resolved to a known local
// definition.
+ ValueInfo VI;
if (GV.hasName()) {
- ValueInfo VI = ImportIndex.getValueInfo(GV.getGUID());
+ VI = ImportIndex.getValueInfo(GV.getGUID());
if (VI && VI.isDSOLocal()) {
GV.setDSOLocal(true);
if (GV.hasDLLImportStorageClass())
@@ -213,6 +214,22 @@ void FunctionImportGlobalProcessing::processGlobalForThinLTO(GlobalValue &GV) {
}
}
+ // Mark read-only variables which can be imported with specific attribute.
+ // We can't internalize them now because IRMover will fail to link variable
+ // definitions to their external declarations during ThinLTO import. We'll
+ // internalize read-only variables later, after import is finished.
+ // See internalizeImmutableGVs.
+ //
+ // If global value dead stripping is not enabled in summary then
+ // propagateConstants hasn't been run (may be because we're using
+ // distriuted import. We can't internalize GV in such case.
+ if (!GV.isDeclaration() && VI && ImportIndex.withGlobalValueDeadStripping()) {
+ const auto &SL = VI.getSummaryList();
+ auto *GVS = SL.empty() ? nullptr : dyn_cast<GlobalVarSummary>(SL[0].get());
+ if (GVS && GVS->isReadOnly())
+ cast<GlobalVariable>(&GV)->addAttribute("thinlto-internalize");
+ }
+
bool DoPromote = false;
if (GV.hasLocalLinkage() &&
((DoPromote = shouldPromoteLocalToGlobal(&GV)) || isPerformingImport())) {
@@ -230,7 +247,7 @@ void FunctionImportGlobalProcessing::processGlobalForThinLTO(GlobalValue &GV) {
// Remove functions imported as available externally defs from comdats,
// as this is a declaration for the linker, and will be dropped eventually.
// It is illegal for comdats to contain declarations.
- auto *GO = dyn_cast_or_null<GlobalObject>(&GV);
+ auto *GO = dyn_cast<GlobalObject>(&GV);
if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) {
// The IRMover should not have placed any imported declarations in
// a comdat, so the only declaration that should be in a comdat
OpenPOWER on IntegriCloud