summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/Analysis/ModuleSummaryAnalysis.cpp6
-rw-r--r--llvm/lib/Bitcode/Reader/BitcodeReader.cpp79
-rw-r--r--llvm/lib/Bitcode/Writer/BitcodeWriter.cpp11
-rw-r--r--llvm/lib/LTO/LTO.cpp9
-rw-r--r--llvm/lib/Transforms/IPO/LowerTypeTests.cpp7
-rw-r--r--llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp14
-rw-r--r--llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp11
7 files changed, 127 insertions, 10 deletions
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index 6bda1d1b1a3..87f76d43bb1 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -457,7 +457,11 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
ProfileSummaryInfo *PSI) {
assert(PSI);
- ModuleSummaryIndex Index(/*HaveGVs=*/true);
+ bool EnableSplitLTOUnit = false;
+ if (auto *MD = mdconst::extract_or_null<ConstantInt>(
+ M.getModuleFlag("EnableSplitLTOUnit")))
+ EnableSplitLTOUnit = MD->getZExtValue();
+ ModuleSummaryIndex Index(/*HaveGVs=*/true, EnableSplitLTOUnit);
// Identify the local values in the llvm.used and llvm.compiler.used sets,
// which should not be exported as they would then require renaming and
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 2f3d2f3f032..fe051e7a912 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -5294,18 +5294,30 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
break;
case bitc::FS_FLAGS: { // [flags]
uint64_t Flags = Record[0];
- // Scan flags (set only on the combined index).
- assert(Flags <= 0x3 && "Unexpected bits in flag");
+ // Scan flags.
+ assert(Flags <= 0x1f && "Unexpected bits in flag");
// 1 bit: WithGlobalValueDeadStripping flag.
+ // Set on combined index only.
if (Flags & 0x1)
TheIndex.setWithGlobalValueDeadStripping();
// 1 bit: SkipModuleByDistributedBackend flag.
+ // Set on combined index only.
if (Flags & 0x2)
TheIndex.setSkipModuleByDistributedBackend();
// 1 bit: HasSyntheticEntryCounts flag.
+ // Set on combined index only.
if (Flags & 0x4)
TheIndex.setHasSyntheticEntryCounts();
+ // 1 bit: DisableSplitLTOUnit flag.
+ // Set on per module indexes. It is up to the client to validate
+ // the consistency of this flag across modules being linked.
+ if (Flags & 0x8)
+ TheIndex.setEnableSplitLTOUnit();
+ // 1 bit: PartiallySplitLTOUnits flag.
+ // Set on combined index only.
+ if (Flags & 0x10)
+ TheIndex.setPartiallySplitLTOUnits();
break;
}
case bitc::FS_VALUE_GUID: { // [valueid, refguid]
@@ -5917,6 +5929,46 @@ Expected<std::unique_ptr<ModuleSummaryIndex>> BitcodeModule::getSummary() {
return std::move(Index);
}
+static Expected<bool> getEnableSplitLTOUnitFlag(BitstreamCursor &Stream,
+ unsigned ID) {
+ if (Stream.EnterSubBlock(ID))
+ return error("Invalid record");
+ SmallVector<uint64_t, 64> Record;
+
+ while (true) {
+ BitstreamEntry Entry = Stream.advanceSkippingSubblocks();
+
+ switch (Entry.Kind) {
+ case BitstreamEntry::SubBlock: // Handled for us already.
+ case BitstreamEntry::Error:
+ return error("Malformed block");
+ case BitstreamEntry::EndBlock:
+ // If no flags record found, conservatively return true to mimic
+ // behavior before this flag was added.
+ return true;
+ case BitstreamEntry::Record:
+ // The interesting case.
+ break;
+ }
+
+ // Look for the FS_FLAGS record.
+ Record.clear();
+ auto BitCode = Stream.readRecord(Entry.ID, Record);
+ switch (BitCode) {
+ default: // Default behavior: ignore.
+ break;
+ case bitc::FS_FLAGS: { // [flags]
+ uint64_t Flags = Record[0];
+ // Scan flags.
+ assert(Flags <= 0x1f && "Unexpected bits in flag");
+
+ return Flags & 0x8;
+ }
+ }
+ }
+ llvm_unreachable("Exit infinite loop");
+}
+
// Check if the given bitcode buffer contains a global value summary block.
Expected<BitcodeLTOInfo> BitcodeModule::getLTOInfo() {
BitstreamCursor Stream(Buffer);
@@ -5932,14 +5984,27 @@ Expected<BitcodeLTOInfo> BitcodeModule::getLTOInfo() {
case BitstreamEntry::Error:
return error("Malformed block");
case BitstreamEntry::EndBlock:
- return BitcodeLTOInfo{/*IsThinLTO=*/false, /*HasSummary=*/false};
+ return BitcodeLTOInfo{/*IsThinLTO=*/false, /*HasSummary=*/false,
+ /*EnableSplitLTOUnit=*/false};
case BitstreamEntry::SubBlock:
- if (Entry.ID == bitc::GLOBALVAL_SUMMARY_BLOCK_ID)
- return BitcodeLTOInfo{/*IsThinLTO=*/true, /*HasSummary=*/true};
+ if (Entry.ID == bitc::GLOBALVAL_SUMMARY_BLOCK_ID) {
+ Expected<bool> EnableSplitLTOUnit =
+ getEnableSplitLTOUnitFlag(Stream, Entry.ID);
+ if (!EnableSplitLTOUnit)
+ return EnableSplitLTOUnit.takeError();
+ return BitcodeLTOInfo{/*IsThinLTO=*/true, /*HasSummary=*/true,
+ *EnableSplitLTOUnit};
+ }
- if (Entry.ID == bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID)
- return BitcodeLTOInfo{/*IsThinLTO=*/false, /*HasSummary=*/true};
+ if (Entry.ID == bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID) {
+ Expected<bool> EnableSplitLTOUnit =
+ getEnableSplitLTOUnitFlag(Stream, Entry.ID);
+ if (!EnableSplitLTOUnit)
+ return EnableSplitLTOUnit.takeError();
+ return BitcodeLTOInfo{/*IsThinLTO=*/false, /*HasSummary=*/true,
+ *EnableSplitLTOUnit};
+ }
// Ignore other sub-blocks.
if (Stream.SkipBlock())
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 68d79edceaf..ba4f932e2e6 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -3618,6 +3618,13 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION});
+ // Write the index flags.
+ uint64_t Flags = 0;
+ // Bits 1-3 are set only in the combined index, skip them.
+ if (Index->enableSplitLTOUnit())
+ Flags |= 0x8;
+ Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Flags});
+
if (Index->begin() == Index->end()) {
Stream.ExitBlock();
return;
@@ -3734,6 +3741,10 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
Flags |= 0x2;
if (Index.hasSyntheticEntryCounts())
Flags |= 0x4;
+ if (Index.enableSplitLTOUnit())
+ Flags |= 0x8;
+ if (Index.partiallySplitLTOUnits())
+ Flags |= 0x10;
Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Flags});
for (const auto &GVI : valueIds()) {
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index f736ef8b7f9..3a955060dea 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -546,6 +546,15 @@ Error LTO::addModule(InputFile &Input, unsigned ModI,
if (!LTOInfo)
return LTOInfo.takeError();
+ if (EnableSplitLTOUnit.hasValue()) {
+ // If only some modules were split, flag this in the index so that
+ // we can skip or error on optimizations that need consistently split
+ // modules (whole program devirt and lower type tests).
+ if (EnableSplitLTOUnit.getValue() != LTOInfo->EnableSplitLTOUnit)
+ ThinLTO.CombinedIndex.setPartiallySplitLTOUnits();
+ } else
+ EnableSplitLTOUnit = LTOInfo->EnableSplitLTOUnit;
+
BitcodeModule BM = Input.Mods[ModI];
auto ModSyms = Input.module_symbols(ModI);
addModuleToGlobalRes(ModSyms, {ResI, ResE},
diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index e4dcd4d4dd7..87c65db0951 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -1702,6 +1702,13 @@ bool LowerTypeTestsModule::lower() {
!ExportSummary && !ImportSummary)
return false;
+ // If only some of the modules were split, we cannot correctly handle
+ // code that contains type tests.
+ if (TypeTestFunc && !TypeTestFunc->use_empty() &&
+ ((ExportSummary && ExportSummary->partiallySplitLTOUnits()) ||
+ (ImportSummary && ImportSummary->partiallySplitLTOUnits())))
+ report_fatal_error("inconsistent LTO Unit splitting with llvm.type.test");
+
if (ImportSummary) {
if (TypeTestFunc) {
for (auto UI = TypeTestFunc->use_begin(), UE = TypeTestFunc->use_end();
diff --git a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
index a5382c4638d..510ecb516dc 100644
--- a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
+++ b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
@@ -418,8 +418,18 @@ void splitAndWriteThinLTOBitcode(
}
}
-// Returns whether this module needs to be split because it uses type metadata.
+// Returns whether this module needs to be split because splitting is
+// enabled and it uses type metadata.
bool requiresSplit(Module &M) {
+ // First check if the LTO Unit splitting has been enabled.
+ bool EnableSplitLTOUnit = false;
+ if (auto *MD = mdconst::extract_or_null<ConstantInt>(
+ M.getModuleFlag("EnableSplitLTOUnit")))
+ EnableSplitLTOUnit = MD->getZExtValue();
+ if (!EnableSplitLTOUnit)
+ return false;
+
+ // Module only needs to be split if it contains type metadata.
for (auto &GO : M.global_objects()) {
if (GO.hasMetadata(LLVMContext::MD_type))
return true;
@@ -431,7 +441,7 @@ bool requiresSplit(Module &M) {
void writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS,
function_ref<AAResults &(Function &)> AARGetter,
Module &M, const ModuleSummaryIndex *Index) {
- // See if this module has any type metadata. If so, we need to split it.
+ // Split module if splitting is enabled and it contains any type metadata.
if (requiresSplit(M))
return splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M);
diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index 37905daee4c..48bd0cda759 100644
--- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -1563,6 +1563,17 @@ bool DevirtModule::run() {
M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load));
Function *AssumeFunc = M.getFunction(Intrinsic::getName(Intrinsic::assume));
+ // If only some of the modules were split, we cannot correctly handle
+ // code that contains type tests or type checked loads.
+ if ((ExportSummary && ExportSummary->partiallySplitLTOUnits()) ||
+ (ImportSummary && ImportSummary->partiallySplitLTOUnits())) {
+ if ((TypeTestFunc && !TypeTestFunc->use_empty()) ||
+ (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()))
+ report_fatal_error("inconsistent LTO Unit splitting with llvm.type.test "
+ "or llvm.type.checked.load");
+ return false;
+ }
+
// Normally if there are no users of the devirtualization intrinsics in the
// module, this pass has nothing to do. But if we are exporting, we also need
// to handle any users that appear only in the function summaries.
OpenPOWER on IntegriCloud