summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/Analysis/ModuleSummaryAnalysis.cpp122
-rw-r--r--llvm/lib/AsmParser/LLParser.cpp57
-rw-r--r--llvm/lib/Bitcode/Reader/BitcodeReader.cpp47
-rw-r--r--llvm/lib/Bitcode/Writer/BitcodeWriter.cpp32
-rw-r--r--llvm/lib/IR/AsmWriter.cpp5
-rw-r--r--llvm/lib/IR/ModuleSummaryIndex.cpp141
-rw-r--r--llvm/lib/LTO/LTO.cpp21
-rw-r--r--llvm/lib/Transforms/IPO/FunctionImport.cpp12
-rw-r--r--llvm/lib/Transforms/Utils/FunctionImportUtils.cpp13
9 files changed, 307 insertions, 143 deletions
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index 914561f0f3f..e25eb290a66 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -231,6 +231,13 @@ static bool isNonVolatileLoad(const Instruction *I) {
return false;
}
+static bool isNonVolatileStore(const Instruction *I) {
+ if (const auto *SI = dyn_cast<StoreInst>(I))
+ return !SI->isVolatile();
+
+ return false;
+}
+
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
const Function &F, BlockFrequencyInfo *BFI,
ProfileSummaryInfo *PSI, DominatorTree &DT,
@@ -245,7 +252,7 @@ static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
// Map from callee ValueId to profile count. Used to accumulate profile
// counts for all static calls to a given callee.
MapVector<ValueInfo, CalleeInfo> CallGraphEdges;
- SetVector<ValueInfo> RefEdges;
+ SetVector<ValueInfo> RefEdges, LoadRefEdges, StoreRefEdges;
SetVector<GlobalValue::GUID> TypeTests;
SetVector<FunctionSummary::VFuncId> TypeTestAssumeVCalls,
TypeCheckedLoadVCalls;
@@ -258,6 +265,7 @@ static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
// list.
findRefEdges(Index, &F, RefEdges, Visited);
std::vector<const Instruction *> NonVolatileLoads;
+ std::vector<const Instruction *> NonVolatileStores;
bool HasInlineAsmMaybeReferencingInternal = false;
for (const BasicBlock &BB : F)
@@ -265,12 +273,34 @@ static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
if (isa<DbgInfoIntrinsic>(I))
continue;
++NumInsts;
- if (isNonVolatileLoad(&I)) {
- // Postpone processing of non-volatile load instructions
- // See comments below
- Visited.insert(&I);
- NonVolatileLoads.push_back(&I);
- continue;
+ // Regular LTO module doesn't participate in ThinLTO import,
+ // so no reference from it can be read/writeonly, since this
+ // would require importing variable as local copy
+ if (IsThinLTO) {
+ if (isNonVolatileLoad(&I)) {
+ // Postpone processing of non-volatile load instructions
+ // See comments below
+ Visited.insert(&I);
+ NonVolatileLoads.push_back(&I);
+ continue;
+ } else if (isNonVolatileStore(&I)) {
+ Visited.insert(&I);
+ NonVolatileStores.push_back(&I);
+ // All references from second operand of store (destination address)
+ // can be considered write-only if they're not referenced by any
+ // non-store instruction. References from first operand of store
+ // (stored value) can't be treated either as read- or as write-only
+ // so we add them to RefEdges as we do with all other instructions
+ // except non-volatile load.
+ Value *Stored = I.getOperand(0);
+ if (auto *GV = dyn_cast<GlobalValue>(Stored))
+ // findRefEdges will try to examine GV operands, so instead
+ // of calling it we should add GV to RefEdges directly.
+ RefEdges.insert(Index.getOrInsertValueInfo(GV));
+ else if (auto *U = dyn_cast<User>(Stored))
+ findRefEdges(Index, U, RefEdges, Visited);
+ continue;
+ }
}
findRefEdges(Index, &I, RefEdges, Visited);
auto CS = ImmutableCallSite(&I);
@@ -361,24 +391,61 @@ static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
}
}
- // By now we processed all instructions in a function, except
- // non-volatile loads. All new refs we add in a loop below
- // are obviously constant. All constant refs are grouped in the
- // end of RefEdges vector, so we can use a single integer value
- // to identify them.
- unsigned RefCnt = RefEdges.size();
- for (const Instruction *I : NonVolatileLoads) {
- Visited.erase(I);
- findRefEdges(Index, I, RefEdges, Visited);
- }
- std::vector<ValueInfo> Refs = RefEdges.takeVector();
- // Regular LTO module doesn't participate in ThinLTO import,
- // so no reference from it can be readonly, since this would
- // require importing variable as local copy
- if (IsThinLTO)
- for (; RefCnt < Refs.size(); ++RefCnt)
+ std::vector<ValueInfo> Refs;
+ if (IsThinLTO) {
+ auto AddRefEdges = [&](const std::vector<const Instruction *> &Instrs,
+ SetVector<ValueInfo> &Edges,
+ SmallPtrSet<const User *, 8> &Cache) {
+ for (const auto *I : Instrs) {
+ Cache.erase(I);
+ findRefEdges(Index, I, Edges, Cache);
+ }
+ };
+
+ // By now we processed all instructions in a function, except
+ // non-volatile loads and non-volatile value stores. Let's find
+ // ref edges for both of instruction sets
+ AddRefEdges(NonVolatileLoads, LoadRefEdges, Visited);
+ // We can add some values to the Visited set when processing load
+ // instructions which are also used by stores in NonVolatileStores.
+ // For example this can happen if we have following code:
+ //
+ // store %Derived* @foo, %Derived** bitcast (%Base** @bar to %Derived**)
+ // %42 = load %Derived*, %Derived** bitcast (%Base** @bar to %Derived**)
+ //
+ // After processing loads we'll add bitcast to the Visited set, and if
+ // we use the same set while processing stores, we'll never see store
+ // to @bar and @bar will be mistakenly treated as readonly.
+ SmallPtrSet<const llvm::User *, 8> StoreCache;
+ AddRefEdges(NonVolatileStores, StoreRefEdges, StoreCache);
+
+ // If both load and store instruction reference the same variable
+ // we won't be able to optimize it. Add all such reference edges
+ // to RefEdges set.
+ for (auto &VI : StoreRefEdges)
+ if (LoadRefEdges.remove(VI))
+ RefEdges.insert(VI);
+
+ unsigned RefCnt = RefEdges.size();
+ // All new reference edges inserted in two loops below are either
+ // read or write only. They will be grouped in the end of RefEdges
+ // vector, so we can use a single integer value to identify them.
+ for (auto &VI : LoadRefEdges)
+ RefEdges.insert(VI);
+
+ unsigned FirstWORef = RefEdges.size();
+ for (auto &VI : StoreRefEdges)
+ RefEdges.insert(VI);
+
+ Refs = RefEdges.takeVector();
+ for (; RefCnt < FirstWORef; ++RefCnt)
Refs[RefCnt].setReadOnly();
+ for (; RefCnt < Refs.size(); ++RefCnt)
+ Refs[RefCnt].setWriteOnly();
+ } else {
+ Refs = RefEdges.takeVector();
+ }
// Explicit add hot edges to enforce importing for designated GUIDs for
// sample PGO, to enable the same inlines as the profiled optimized binary.
for (auto &I : F.getImportGUIDs())
@@ -526,10 +593,11 @@ static void computeVariableSummary(ModuleSummaryIndex &Index,
}
}
- // Don't mark variables we won't be able to internalize as read-only.
- GlobalVarSummary::GVarFlags VarFlags(
+ // Don't mark variables we won't be able to internalize as read/write-only.
+ bool CanBeInternalized =
!V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() &&
- !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass());
+ !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass();
+ GlobalVarSummary::GVarFlags VarFlags(CanBeInternalized, CanBeInternalized);
auto GVarSummary = llvm::make_unique<GlobalVarSummary>(Flags, VarFlags,
RefEdges.takeVector());
if (NonRenamableLocal)
@@ -647,7 +715,7 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
} else {
std::unique_ptr<GlobalVarSummary> Summary =
llvm::make_unique<GlobalVarSummary>(
- GVFlags, GlobalVarSummary::GVarFlags(),
+ GVFlags, GlobalVarSummary::GVarFlags(false, false),
ArrayRef<ValueInfo>{});
Index.addGlobalValueSummary(*GV, std::move(Summary));
}
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 51fc5b6e1d2..ab645348c53 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -7849,9 +7849,13 @@ static const auto FwdVIRef = (GlobalValueSummaryMapTy::value_type *)-8;
static void resolveFwdRef(ValueInfo *Fwd, ValueInfo &Resolved) {
bool ReadOnly = Fwd->isReadOnly();
+ bool WriteOnly = Fwd->isWriteOnly();
+ assert(!(ReadOnly && WriteOnly));
*Fwd = Resolved;
if (ReadOnly)
Fwd->setReadOnly();
+ if (WriteOnly)
+ Fwd->setWriteOnly();
}
/// Stores the given Name/GUID and associated summary into the Index.
@@ -8081,7 +8085,8 @@ bool LLParser::ParseVariableSummary(std::string Name, GlobalValue::GUID GUID,
GlobalValueSummary::GVFlags GVFlags = GlobalValueSummary::GVFlags(
/*Linkage=*/GlobalValue::ExternalLinkage, /*NotEligibleToImport=*/false,
/*Live=*/false, /*IsLocal=*/false, /*CanAutoHide=*/false);
- GlobalVarSummary::GVarFlags GVarFlags(/*ReadOnly*/ false);
+ GlobalVarSummary::GVarFlags GVarFlags(/*ReadOnly*/ false,
+ /* WriteOnly */ false);
std::vector<ValueInfo> Refs;
VTableFuncList VTableFuncs;
if (ParseToken(lltok::colon, "expected ':' here") ||
@@ -8422,10 +8427,11 @@ bool LLParser::ParseOptionalRefs(std::vector<ValueInfo> &Refs) {
VContexts.push_back(VC);
} while (EatIfPresent(lltok::comma));
- // Sort value contexts so that ones with readonly ValueInfo are at the end
- // of VContexts vector. This is needed to match immutableRefCount() behavior.
+ // Sort value contexts so that ones with writeonly
+ // and readonly ValueInfo are at the end of VContexts vector.
+ // See FunctionSummary::specialRefCounts()
llvm::sort(VContexts, [](const ValueContext &VC1, const ValueContext &VC2) {
- return VC1.VI.isReadOnly() < VC2.VI.isReadOnly();
+ return VC1.VI.getAccessSpecifier() < VC2.VI.getAccessSpecifier();
});
IdToIndexMapType IdToIndexMap;
@@ -8743,24 +8749,41 @@ bool LLParser::ParseGVFlags(GlobalValueSummary::GVFlags &GVFlags) {
}
/// GVarFlags
-/// ::= 'varFlags' ':' '(' 'readonly' ':' Flag ')'
+/// ::= 'varFlags' ':' '(' 'readonly' ':' Flag
+/// ',' 'writeonly' ':' Flag ')'
bool LLParser::ParseGVarFlags(GlobalVarSummary::GVarFlags &GVarFlags) {
assert(Lex.getKind() == lltok::kw_varFlags);
Lex.Lex();
- unsigned Flag = 0;
if (ParseToken(lltok::colon, "expected ':' here") ||
- ParseToken(lltok::lparen, "expected '(' here") ||
- ParseToken(lltok::kw_readonly, "expected 'readonly' here") ||
- ParseToken(lltok::colon, "expected ':' here"))
+ ParseToken(lltok::lparen, "expected '(' here"))
return true;
- ParseFlag(Flag);
- GVarFlags.ReadOnly = Flag;
+ auto ParseRest = [this](unsigned int &Val) {
+ Lex.Lex();
+ if (ParseToken(lltok::colon, "expected ':'"))
+ return true;
+ return ParseFlag(Val);
+ };
- if (ParseToken(lltok::rparen, "expected ')' here"))
- return true;
- return false;
+ do {
+ unsigned Flag = 0;
+ switch (Lex.getKind()) {
+ case lltok::kw_readonly:
+ if (ParseRest(Flag))
+ return true;
+ GVarFlags.MaybeReadOnly = Flag;
+ break;
+ case lltok::kw_writeonly:
+ if (ParseRest(Flag))
+ return true;
+ GVarFlags.MaybeWriteOnly = Flag;
+ break;
+ default:
+ return Error(Lex.getLoc(), "expected gvar flag type");
+ }
+ } while (EatIfPresent(lltok::comma));
+ return ParseToken(lltok::rparen, "expected ')' here");
}
/// ModuleReference
@@ -8783,7 +8806,9 @@ bool LLParser::ParseModuleReference(StringRef &ModulePath) {
/// GVReference
/// ::= SummaryID
bool LLParser::ParseGVReference(ValueInfo &VI, unsigned &GVId) {
- bool ReadOnly = EatIfPresent(lltok::kw_readonly);
+ bool WriteOnly = false, ReadOnly = EatIfPresent(lltok::kw_readonly);
+ if (!ReadOnly)
+ WriteOnly = EatIfPresent(lltok::kw_writeonly);
if (ParseToken(lltok::SummaryID, "expected GV ID"))
return true;
@@ -8798,5 +8823,7 @@ bool LLParser::ParseGVReference(ValueInfo &VI, unsigned &GVId) {
if (ReadOnly)
VI.setReadOnly();
+ if (WriteOnly)
+ VI.setWriteOnly();
return false;
}
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index d54e40295b8..22d1b2165fd 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -984,7 +984,8 @@ static GlobalValueSummary::GVFlags getDecodedGVSummaryFlags(uint64_t RawFlags,
// Decode the flags for GlobalVariable in the summary
static GlobalVarSummary::GVarFlags getDecodedGVarFlags(uint64_t RawFlags) {
- return GlobalVarSummary::GVarFlags((RawFlags & 0x1) ? true : false);
+ return GlobalVarSummary::GVarFlags((RawFlags & 0x1) ? true : false,
+ (RawFlags & 0x2) ? true : false);
}
static GlobalValue::VisibilityTypes getDecodedVisibility(unsigned Val) {
@@ -5681,10 +5682,16 @@ void ModuleSummaryIndexBitcodeReader::parseTypeIdCompatibleVtableSummaryRecord(
parseTypeIdCompatibleVtableInfo(Record, Slot, TypeId);
}
-static void setImmutableRefs(std::vector<ValueInfo> &Refs, unsigned Count) {
- // Read-only refs are in the end of the refs list.
- for (unsigned RefNo = Refs.size() - Count; RefNo < Refs.size(); ++RefNo)
+static void setSpecialRefs(std::vector<ValueInfo> &Refs, unsigned ROCnt,
+ unsigned WOCnt) {
+ // Readonly and writeonly refs are in the end of the refs list.
+ assert(ROCnt + WOCnt <= Refs.size());
+ unsigned FirstWORef = Refs.size() - WOCnt;
+ unsigned RefNo = FirstWORef - ROCnt;
+ for (; RefNo < FirstWORef; ++RefNo)
Refs[RefNo].setReadOnly();
+ for (; RefNo < Refs.size(); ++RefNo)
+ Refs[RefNo].setWriteOnly();
}
// Eagerly parse the entire summary block. This populates the GlobalValueSummary
@@ -5711,9 +5718,9 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
}
const uint64_t Version = Record[0];
const bool IsOldProfileFormat = Version == 1;
- if (Version < 1 || Version > 6)
+ if (Version < 1 || Version > 7)
return error("Invalid summary version " + Twine(Version) +
- ". Version should be in the range [1-6].");
+ ". Version should be in the range [1-7].");
Record.clear();
// Keep around the last seen summary to be used when we see an optional
@@ -5812,15 +5819,19 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
unsigned InstCount = Record[2];
uint64_t RawFunFlags = 0;
unsigned NumRefs = Record[3];
- unsigned NumImmutableRefs = 0;
+ unsigned NumRORefs = 0, NumWORefs = 0;
int RefListStartIndex = 4;
if (Version >= 4) {
RawFunFlags = Record[3];
NumRefs = Record[4];
RefListStartIndex = 5;
if (Version >= 5) {
- NumImmutableRefs = Record[5];
+ NumRORefs = Record[5];
RefListStartIndex = 6;
+ if (Version >= 7) {
+ NumWORefs = Record[6];
+ RefListStartIndex = 7;
+ }
}
}
@@ -5840,7 +5851,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
std::vector<FunctionSummary::EdgeTy> Calls = makeCallList(
ArrayRef<uint64_t>(Record).slice(CallGraphEdgeStartIndex),
IsOldProfileFormat, HasProfile, HasRelBF);
- setImmutableRefs(Refs, NumImmutableRefs);
+ setSpecialRefs(Refs, NumRORefs, NumWORefs);
auto FS = llvm::make_unique<FunctionSummary>(
Flags, InstCount, getDecodedFFlags(RawFunFlags), /*EntryCount=*/0,
std::move(Refs), std::move(Calls), std::move(PendingTypeTests),
@@ -5891,7 +5902,8 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
unsigned ValueID = Record[0];
uint64_t RawFlags = Record[1];
unsigned RefArrayStart = 2;
- GlobalVarSummary::GVarFlags GVF;
+ GlobalVarSummary::GVarFlags GVF(/* ReadOnly */ false,
+ /* WriteOnly */ false);
auto Flags = getDecodedGVSummaryFlags(RawFlags, Version);
if (Version >= 5) {
GVF = getDecodedGVarFlags(Record[2]);
@@ -5948,7 +5960,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
uint64_t RawFunFlags = 0;
uint64_t EntryCount = 0;
unsigned NumRefs = Record[4];
- unsigned NumImmutableRefs = 0;
+ unsigned NumRORefs = 0, NumWORefs = 0;
int RefListStartIndex = 5;
if (Version >= 4) {
@@ -5956,13 +5968,19 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
RefListStartIndex = 6;
size_t NumRefsIndex = 5;
if (Version >= 5) {
+ unsigned NumRORefsOffset = 1;
RefListStartIndex = 7;
if (Version >= 6) {
NumRefsIndex = 6;
EntryCount = Record[5];
RefListStartIndex = 8;
+ if (Version >= 7) {
+ RefListStartIndex = 9;
+ NumWORefs = Record[8];
+ NumRORefsOffset = 2;
+ }
}
- NumImmutableRefs = Record[RefListStartIndex - 1];
+ NumRORefs = Record[RefListStartIndex - NumRORefsOffset];
}
NumRefs = Record[NumRefsIndex];
}
@@ -5978,7 +5996,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
ArrayRef<uint64_t>(Record).slice(CallGraphEdgeStartIndex),
IsOldProfileFormat, HasProfile, false);
ValueInfo VI = getValueInfoFromValueId(ValueID).first;
- setImmutableRefs(Refs, NumImmutableRefs);
+ setSpecialRefs(Refs, NumRORefs, NumWORefs);
auto FS = llvm::make_unique<FunctionSummary>(
Flags, InstCount, getDecodedFFlags(RawFunFlags), EntryCount,
std::move(Refs), std::move(Edges), std::move(PendingTypeTests),
@@ -6025,7 +6043,8 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
uint64_t ModuleId = Record[1];
uint64_t RawFlags = Record[2];
unsigned RefArrayStart = 3;
- GlobalVarSummary::GVarFlags GVF;
+ GlobalVarSummary::GVarFlags GVF(/* ReadOnly */ false,
+ /* WriteOnly */ false);
auto Flags = getDecodedGVSummaryFlags(RawFlags, Version);
if (Version >= 5) {
GVF = getDecodedGVarFlags(Record[3]);
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 6d51d9b9d77..90de4688c8c 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -1017,7 +1017,7 @@ static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) {
}
static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) {
- uint64_t RawFlags = Flags.ReadOnly;
+ uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1);
return RawFlags;
}
@@ -3629,11 +3629,13 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
FunctionSummary *FS = cast<FunctionSummary>(Summary);
writeFunctionTypeMetadataRecords(Stream, FS);
+ auto SpecialRefCnts = FS->specialRefCounts();
NameVals.push_back(getEncodedGVSummaryFlags(FS->flags()));
NameVals.push_back(FS->instCount());
NameVals.push_back(getEncodedFFlags(FS->fflags()));
NameVals.push_back(FS->refs().size());
- NameVals.push_back(FS->immutableRefCount());
+ NameVals.push_back(SpecialRefCnts.first); // rorefcnt
+ NameVals.push_back(SpecialRefCnts.second); // worefcnt
for (auto &RI : FS->refs())
NameVals.push_back(VE.getValueID(RI.getValue()));
@@ -3707,7 +3709,7 @@ void ModuleBitcodeWriterBase::writeModuleLevelReferences(
// Current version for the summary.
// This is bumped whenever we introduce changes in the way some record are
// interpreted, like flags for instance.
-static const uint64_t INDEX_VERSION = 6;
+static const uint64_t INDEX_VERSION = 7;
/// Emit the per-module summary section alongside the rest of
/// the module's bitcode.
@@ -3749,7 +3751,8 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
- Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
// numrefs x valueid, n x (valueid, hotness)
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
@@ -3766,7 +3769,8 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
- Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
// numrefs x valueid, n x (valueid [, rel_block_freq])
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
@@ -3898,7 +3902,8 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
- Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
// numrefs x valueid, n x (valueid)
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
@@ -3914,7 +3919,8 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
- Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
// numrefs x valueid, n x (valueid, hotness)
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
@@ -4016,20 +4022,24 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
// Fill in below
NameVals.push_back(0); // numrefs
- NameVals.push_back(0); // immutablerefcnt
+ NameVals.push_back(0); // rorefcnt
+ NameVals.push_back(0); // worefcnt
- unsigned Count = 0, ImmutableRefCnt = 0;
+ unsigned Count = 0, RORefCnt = 0, WORefCnt = 0;
for (auto &RI : FS->refs()) {
auto RefValueId = getValueId(RI.getGUID());
if (!RefValueId)
continue;
NameVals.push_back(*RefValueId);
if (RI.isReadOnly())
- ImmutableRefCnt++;
+ RORefCnt++;
+ else if (RI.isWriteOnly())
+ WORefCnt++;
Count++;
}
NameVals[6] = Count;
- NameVals[7] = ImmutableRefCnt;
+ NameVals[7] = RORefCnt;
+ NameVals[8] = WORefCnt;
bool HasProfileData = false;
for (auto &EI : FS->calls()) {
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 67344f4a620..a5adfd9c8a6 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -2888,7 +2888,8 @@ void AssemblyWriter::printAliasSummary(const AliasSummary *AS) {
}
void AssemblyWriter::printGlobalVarSummary(const GlobalVarSummary *GS) {
- Out << ", varFlags: (readonly: " << GS->VarFlags.ReadOnly << ")";
+ Out << ", varFlags: (readonly: " << GS->VarFlags.MaybeReadOnly << ", "
+ << "writeonly: " << GS->VarFlags.MaybeWriteOnly << ")";
auto VTableFuncs = GS->vTableFuncs();
if (!VTableFuncs.empty()) {
@@ -3098,6 +3099,8 @@ void AssemblyWriter::printSummary(const GlobalValueSummary &Summary) {
Out << FS;
if (Ref.isReadOnly())
Out << "readonly ";
+ else if (Ref.isWriteOnly())
+ Out << "writeonly ";
Out << "^" << Machine.getGUIDSlot(Ref.getGUID());
}
Out << ")";
diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp
index 18b7ac09388..9f347d8da01 100644
--- a/llvm/lib/IR/ModuleSummaryIndex.cpp
+++ b/llvm/lib/IR/ModuleSummaryIndex.cpp
@@ -23,6 +23,8 @@ using namespace llvm;
STATISTIC(ReadOnlyLiveGVars,
"Number of live global variables marked read only");
+STATISTIC(WriteOnlyLiveGVars,
+ "Number of live global variables marked write only");
FunctionSummary FunctionSummary::ExternalNode =
FunctionSummary::makeDummyFunctionSummary({});
@@ -45,15 +47,18 @@ bool ValueInfo::canAutoHide() const {
});
}
-// Gets the number of immutable refs in RefEdgeList
-unsigned FunctionSummary::immutableRefCount() const {
- // Here we take advantage of having all readonly references
+// Gets the number of readonly and writeonly refs in RefEdgeList
+std::pair<unsigned, unsigned> FunctionSummary::specialRefCounts() const {
+ // Here we take advantage of having all readonly and writeonly references
// located in the end of the RefEdgeList.
auto Refs = refs();
- unsigned ImmutableRefCnt = 0;
- for (int I = Refs.size() - 1; I >= 0 && Refs[I].isReadOnly(); --I)
- ImmutableRefCnt++;
- return ImmutableRefCnt;
+ unsigned RORefCnt = 0, WORefCnt = 0;
+ int I;
+ for (I = Refs.size() - 1; I >= 0 && Refs[I].isWriteOnly(); --I)
+ WORefCnt++;
+ for (; I >= 0 && Refs[I].isReadOnly(); --I)
+ RORefCnt++;
+ return {RORefCnt, WORefCnt};
}
// Collect for the given module the list of function it defines
@@ -99,48 +104,56 @@ bool ModuleSummaryIndex::isGUIDLive(GlobalValue::GUID GUID) const {
return false;
}
-static void propagateConstantsToRefs(GlobalValueSummary *S) {
- // If reference is not readonly then referenced summary is not
- // readonly either. Note that:
+static void propagateAttributesToRefs(GlobalValueSummary *S) {
+ // If reference is not readonly or writeonly then referenced summary is not
+ // read/writeonly either. Note that:
// - All references from GlobalVarSummary are conservatively considered as
- // not readonly. Tracking them properly requires more complex analysis
- // then we have now.
+ // not readonly or writeonly. Tracking them properly requires more complex
+ // analysis then we have now.
//
// - AliasSummary objects have no refs at all so this function is a no-op
// for them.
for (auto &VI : S->refs()) {
- if (VI.isReadOnly()) {
- // We only mark refs as readonly when computing function summaries on
- // analysis phase.
- assert(isa<FunctionSummary>(S));
- continue;
- }
+ assert(VI.getAccessSpecifier() == 0 || isa<FunctionSummary>(S));
for (auto &Ref : VI.getSummaryList())
- // If references to alias is not readonly then aliasee is not readonly
- if (auto *GVS = dyn_cast<GlobalVarSummary>(Ref->getBaseObject()))
- GVS->setReadOnly(false);
+ // If references to alias is not read/writeonly then aliasee
+ // is not read/writeonly
+ if (auto *GVS = dyn_cast<GlobalVarSummary>(Ref->getBaseObject())) {
+ if (!VI.isReadOnly())
+ GVS->setReadOnly(false);
+ if (!VI.isWriteOnly())
+ GVS->setWriteOnly(false);
+ }
}
}
-// Do the constant propagation in combined index.
-// The goal of constant propagation is internalization of readonly
-// variables. To determine which variables are readonly and which
-// are not we take following steps:
-// - During analysis we speculatively assign readonly attribute to
-// all variables which can be internalized. When computing function
-// summary we also assign readonly attribute to a reference if
-// function doesn't modify referenced variable.
+// Do the access attribute propagation in combined index.
+// The goal of attribute propagation is internalization of readonly (RO)
+// or writeonly (WO) variables. To determine which variables are RO or WO
+// and which are not we take following steps:
+// - During analysis we speculatively assign readonly and writeonly
+// attribute to all variables which can be internalized. When computing
+// function summary we also assign readonly or writeonly attribute to a
+// reference if function doesn't modify referenced variable (readonly)
+// or doesn't read it (writeonly).
+//
+// - After computing dead symbols in combined index we do the attribute
+// propagation. During this step we:
+// a. clear RO and WO attributes from variables which are preserved or
+// can't be imported
+// b. clear RO and WO attributes from variables referenced by any global
+// variable initializer
+// c. clear RO attribute from variable referenced by a function when
+// reference is not readonly
+// d. clear WO attribute from variable referenced by a function when
+// reference is not writeonly
//
-// - After computing dead symbols in combined index we do the constant
-// propagation. During this step we clear readonly attribute from
-// all variables which:
-// a. are preserved or can't be imported
-// b. referenced by any global variable initializer
-// c. referenced by a function and reference is not readonly
+// Because of (c, d) we don't internalize variables read by function A
+// and modified by function B.
//
// Internalization itself happens in the backend after import is finished
-// See internalizeImmutableGVs.
-void ModuleSummaryIndex::propagateConstants(
+// See internalizeGVsAfterImport.
+void ModuleSummaryIndex::propagateAttributes(
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
for (auto &P : *this)
for (auto &S : P.second.SummaryList) {
@@ -148,29 +161,36 @@ void ModuleSummaryIndex::propagateConstants(
// We don't examine references from dead objects
continue;
- // Global variable can't be marked read only if it is not eligible
- // to import since we need to ensure that all external references
- // get a local (imported) copy. It also can't be marked read only
- // if it or any alias (since alias points to the same memory) are
- // preserved or notEligibleToImport, since either of those means
- // there could be writes that are not visible (because preserved
- // means it could have external to DSO writes, and notEligibleToImport
- // means it could have writes via inline assembly leading it to be
- // in the @llvm.*used).
+ // Global variable can't be marked read/writeonly if it is not eligible
+ // to import since we need to ensure that all external references get
+ // a local (imported) copy. It also can't be marked read/writeonly if
+ // it or any alias (since alias points to the same memory) are preserved
+ // or notEligibleToImport, since either of those means there could be
+ // writes (or reads in case of writeonly) that are not visible (because
+ // preserved means it could have external to DSO writes or reads, and
+ // notEligibleToImport means it could have writes or reads via inline
+ // assembly leading it to be in the @llvm.*used).
if (auto *GVS = dyn_cast<GlobalVarSummary>(S->getBaseObject()))
// Here we intentionally pass S.get() not GVS, because S could be
// an alias.
- if (!canImportGlobalVar(S.get()) || GUIDPreservedSymbols.count(P.first))
+ if (!canImportGlobalVar(S.get()) ||
+ GUIDPreservedSymbols.count(P.first)) {
GVS->setReadOnly(false);
- propagateConstantsToRefs(S.get());
+ GVS->setWriteOnly(false);
+ }
+ propagateAttributesToRefs(S.get());
}
if (llvm::AreStatisticsEnabled())
for (auto &P : *this)
if (P.second.SummaryList.size())
if (auto *GVS = dyn_cast<GlobalVarSummary>(
P.second.SummaryList[0]->getBaseObject()))
- if (isGlobalValueLive(GVS) && GVS->isReadOnly())
- ReadOnlyLiveGVars++;
+ if (isGlobalValueLive(GVS)) {
+ if (GVS->maybeReadOnly())
+ ReadOnlyLiveGVars++;
+ if (GVS->maybeWriteOnly())
+ WriteOnlyLiveGVars++;
+ }
}
// TODO: write a graphviz dumper for SCCs (see ModuleSummaryIndex::exportToDot)
@@ -333,7 +353,13 @@ static void defineExternalNode(raw_ostream &OS, const char *Pfx,
static bool hasReadOnlyFlag(const GlobalValueSummary *S) {
if (auto *GVS = dyn_cast<GlobalVarSummary>(S))
- return GVS->isReadOnly();
+ return GVS->maybeReadOnly();
+ return false;
+}
+
+static bool hasWriteOnlyFlag(const GlobalValueSummary *S) {
+ if (auto *GVS = dyn_cast<GlobalVarSummary>(S))
+ return GVS->maybeWriteOnly();
return false;
}
@@ -358,12 +384,14 @@ void ModuleSummaryIndex::exportToDot(raw_ostream &OS) const {
// 0 - alias
// 1 - reference
// 2 - constant reference
- // Other value: (hotness - 3).
- TypeOrHotness += 3;
+ // 3 - writeonly reference
+ // Other value: (hotness - 4).
+ TypeOrHotness += 4;
static const char *EdgeAttrs[] = {
" [style=dotted]; // alias",
" [style=dashed]; // ref",
" [style=dashed,color=forestgreen]; // const-ref",
+ " [style=dashed,color=violetred]; // writeOnly-ref",
" // call (hotness : Unknown)",
" [color=blue]; // call (hotness : Cold)",
" // call (hotness : None)",
@@ -408,6 +436,8 @@ void ModuleSummaryIndex::exportToDot(raw_ostream &OS) const {
A.add("shape", "Mrecord", "variable");
if (Flags.Live && hasReadOnlyFlag(SummaryIt.second))
A.addComment("immutable");
+ if (Flags.Live && hasWriteOnlyFlag(SummaryIt.second))
+ A.addComment("writeOnly");
}
if (Flags.DSOLocal)
A.addComment("dsoLocal");
@@ -429,10 +459,11 @@ void ModuleSummaryIndex::exportToDot(raw_ostream &OS) const {
for (auto &SummaryIt : GVSMap) {
auto *GVS = SummaryIt.second;
for (auto &R : GVS->refs())
- Draw(SummaryIt.first, R.getGUID(), R.isReadOnly() ? -1 : -2);
+ Draw(SummaryIt.first, R.getGUID(),
+ R.isWriteOnly() ? -1 : (R.isReadOnly() ? -2 : -3));
if (auto *AS = dyn_cast_or_null<AliasSummary>(SummaryIt.second)) {
- Draw(SummaryIt.first, AS->getAliaseeGUID(), -3);
+ Draw(SummaryIt.first, AS->getAliaseeGUID(), -4);
continue;
}
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 4ed13701aa9..64506890956 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -192,8 +192,10 @@ void llvm::computeLTOCacheKey(
AddUnsigned(VI.isDSOLocal());
AddUsedCfiGlobal(VI.getGUID());
}
- if (auto *GVS = dyn_cast<GlobalVarSummary>(GS))
- AddUnsigned(GVS->isReadOnly());
+ if (auto *GVS = dyn_cast<GlobalVarSummary>(GS)) {
+ AddUnsigned(GVS->maybeReadOnly());
+ AddUnsigned(GVS->maybeWriteOnly());
+ }
if (auto *FS = dyn_cast<FunctionSummary>(GS)) {
for (auto &TT : FS->type_tests())
UsedTypeIds.insert(TT);
@@ -371,9 +373,9 @@ void llvm::thinLTOResolvePrevailingInIndex(
GUIDPreservedSymbols);
}
-static bool isWeakWriteableObject(GlobalValueSummary *GVS) {
+static bool isWeakObjectWithRWAccess(GlobalValueSummary *GVS) {
if (auto *VarSummary = dyn_cast<GlobalVarSummary>(GVS->getBaseObject()))
- return !VarSummary->isReadOnly() &&
+ return !VarSummary->maybeReadOnly() && !VarSummary->maybeWriteOnly() &&
(VarSummary->linkage() == GlobalValue::WeakODRLinkage ||
VarSummary->linkage() == GlobalValue::LinkOnceODRLinkage);
return false;
@@ -394,11 +396,12 @@ static void thinLTOInternalizeAndPromoteGUID(
// We can't internalize available_externally globals because this
// can break function pointer equality.
S->linkage() != GlobalValue::AvailableExternallyLinkage &&
- // Functions and read-only variables with linkonce_odr and weak_odr
- // linkage can be internalized. We can't internalize linkonce_odr
- // and weak_odr variables which are modified somewhere in the
- // program because reads and writes will become inconsistent.
- !isWeakWriteableObject(S.get()))
+ // Functions and read-only variables with linkonce_odr and
+ // weak_odr linkage can be internalized. We can't internalize
+ // linkonce_odr and weak_odr variables which are both modified
+ // and read somewhere in the program because reads and writes
+ // will become inconsistent.
+ !isWeakObjectWithRWAccess(S.get()))
S->setLinkage(GlobalValue::InternalLinkage);
}
}
diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp
index 9207f5fe0ef..62c7fbd0722 100644
--- a/llvm/lib/Transforms/IPO/FunctionImport.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp
@@ -850,14 +850,16 @@ void llvm::computeDeadSymbolsWithConstProp(
bool ImportEnabled) {
computeDeadSymbols(Index, GUIDPreservedSymbols, isPrevailing);
if (ImportEnabled) {
- Index.propagateConstants(GUIDPreservedSymbols);
+ Index.propagateAttributes(GUIDPreservedSymbols);
} else {
- // If import is disabled we should drop read-only attribute
+ // If import is disabled we should drop read/write-only attribute
// from all summaries to prevent internalization.
for (auto &P : Index)
for (auto &S : P.second.SummaryList)
- if (auto *GVS = dyn_cast<GlobalVarSummary>(S.get()))
+ if (auto *GVS = dyn_cast<GlobalVarSummary>(S.get())) {
GVS->setReadOnly(false);
+ GVS->setWriteOnly(false);
+ }
}
}
@@ -1064,7 +1066,7 @@ static Function *replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA) {
// Internalize values that we marked with specific attribute
// in processGlobalForThinLTO.
-static void internalizeImmutableGVs(Module &M) {
+static void internalizeGVsAfterImport(Module &M) {
for (auto &GV : M.globals())
// Skip GVs which have been converted to declarations
// by dropDeadSymbols.
@@ -1197,7 +1199,7 @@ Expected<bool> FunctionImporter::importFunctions(
NumImportedModules++;
}
- internalizeImmutableGVs(DestModule);
+ internalizeGVsAfterImport(DestModule);
NumImportedFunctions += (ImportedCount - ImportedGVCount);
NumImportedGlobalVars += ImportedGVCount;
diff --git a/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp b/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp
index 8e80ee2c549..c9cc0990f23 100644
--- a/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp
+++ b/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp
@@ -229,11 +229,11 @@ void FunctionImportGlobalProcessing::processGlobalForThinLTO(GlobalValue &GV) {
}
}
- // Mark read-only variables which can be imported with specific attribute.
- // We can't internalize them now because IRMover will fail to link variable
- // definitions to their external declarations during ThinLTO import. We'll
- // internalize read-only variables later, after import is finished.
- // See internalizeImmutableGVs.
+ // Mark read/write-only variables which can be imported with specific
+ // attribute. We can't internalize them now because IRMover will fail
+ // to link variable definitions to their external declarations during
+ // ThinLTO import. We'll internalize read-only variables later, after
+ // import is finished. See internalizeGVsAfterImport.
//
// If global value dead stripping is not enabled in summary then
// propagateConstants hasn't been run. We can't internalize GV
@@ -241,7 +241,8 @@ void FunctionImportGlobalProcessing::processGlobalForThinLTO(GlobalValue &GV) {
if (!GV.isDeclaration() && VI && ImportIndex.withGlobalValueDeadStripping()) {
const auto &SL = VI.getSummaryList();
auto *GVS = SL.empty() ? nullptr : dyn_cast<GlobalVarSummary>(SL[0].get());
- if (GVS && GVS->isReadOnly())
+ // At this stage "maybe" is "definitely"
+ if (GVS && (GVS->maybeReadOnly() || GVS->maybeWriteOnly()))
cast<GlobalVariable>(&GV)->addAttribute("thinlto-internalize");
}
OpenPOWER on IntegriCloud