diff options
Diffstat (limited to 'llvm/lib')
-rw-r--r-- | llvm/lib/Analysis/ModuleSummaryAnalysis.cpp | 6 | ||||
-rw-r--r-- | llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 79 | ||||
-rw-r--r-- | llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 11 | ||||
-rw-r--r-- | llvm/lib/LTO/LTO.cpp | 9 | ||||
-rw-r--r-- | llvm/lib/Transforms/IPO/LowerTypeTests.cpp | 7 | ||||
-rw-r--r-- | llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp | 14 | ||||
-rw-r--r-- | llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp | 11 |
7 files changed, 127 insertions, 10 deletions
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp index 6bda1d1b1a3..87f76d43bb1 100644 --- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -457,7 +457,11 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex( std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback, ProfileSummaryInfo *PSI) { assert(PSI); - ModuleSummaryIndex Index(/*HaveGVs=*/true); + bool EnableSplitLTOUnit = false; + if (auto *MD = mdconst::extract_or_null<ConstantInt>( + M.getModuleFlag("EnableSplitLTOUnit"))) + EnableSplitLTOUnit = MD->getZExtValue(); + ModuleSummaryIndex Index(/*HaveGVs=*/true, EnableSplitLTOUnit); // Identify the local values in the llvm.used and llvm.compiler.used sets, // which should not be exported as they would then require renaming and diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 2f3d2f3f032..fe051e7a912 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -5294,18 +5294,30 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { break; case bitc::FS_FLAGS: { // [flags] uint64_t Flags = Record[0]; - // Scan flags (set only on the combined index). - assert(Flags <= 0x3 && "Unexpected bits in flag"); + // Scan flags. + assert(Flags <= 0x1f && "Unexpected bits in flag"); // 1 bit: WithGlobalValueDeadStripping flag. + // Set on combined index only. if (Flags & 0x1) TheIndex.setWithGlobalValueDeadStripping(); // 1 bit: SkipModuleByDistributedBackend flag. + // Set on combined index only. if (Flags & 0x2) TheIndex.setSkipModuleByDistributedBackend(); // 1 bit: HasSyntheticEntryCounts flag. + // Set on combined index only. if (Flags & 0x4) TheIndex.setHasSyntheticEntryCounts(); + // 1 bit: DisableSplitLTOUnit flag. + // Set on per module indexes. It is up to the client to validate + // the consistency of this flag across modules being linked. + if (Flags & 0x8) + TheIndex.setEnableSplitLTOUnit(); + // 1 bit: PartiallySplitLTOUnits flag. + // Set on combined index only. + if (Flags & 0x10) + TheIndex.setPartiallySplitLTOUnits(); break; } case bitc::FS_VALUE_GUID: { // [valueid, refguid] @@ -5917,6 +5929,46 @@ Expected<std::unique_ptr<ModuleSummaryIndex>> BitcodeModule::getSummary() { return std::move(Index); } +static Expected<bool> getEnableSplitLTOUnitFlag(BitstreamCursor &Stream, + unsigned ID) { + if (Stream.EnterSubBlock(ID)) + return error("Invalid record"); + SmallVector<uint64_t, 64> Record; + + while (true) { + BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + + switch (Entry.Kind) { + case BitstreamEntry::SubBlock: // Handled for us already. + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: + // If no flags record found, conservatively return true to mimic + // behavior before this flag was added. + return true; + case BitstreamEntry::Record: + // The interesting case. + break; + } + + // Look for the FS_FLAGS record. + Record.clear(); + auto BitCode = Stream.readRecord(Entry.ID, Record); + switch (BitCode) { + default: // Default behavior: ignore. + break; + case bitc::FS_FLAGS: { // [flags] + uint64_t Flags = Record[0]; + // Scan flags. + assert(Flags <= 0x1f && "Unexpected bits in flag"); + + return Flags & 0x8; + } + } + } + llvm_unreachable("Exit infinite loop"); +} + // Check if the given bitcode buffer contains a global value summary block. Expected<BitcodeLTOInfo> BitcodeModule::getLTOInfo() { BitstreamCursor Stream(Buffer); @@ -5932,14 +5984,27 @@ Expected<BitcodeLTOInfo> BitcodeModule::getLTOInfo() { case BitstreamEntry::Error: return error("Malformed block"); case BitstreamEntry::EndBlock: - return BitcodeLTOInfo{/*IsThinLTO=*/false, /*HasSummary=*/false}; + return BitcodeLTOInfo{/*IsThinLTO=*/false, /*HasSummary=*/false, + /*EnableSplitLTOUnit=*/false}; case BitstreamEntry::SubBlock: - if (Entry.ID == bitc::GLOBALVAL_SUMMARY_BLOCK_ID) - return BitcodeLTOInfo{/*IsThinLTO=*/true, /*HasSummary=*/true}; + if (Entry.ID == bitc::GLOBALVAL_SUMMARY_BLOCK_ID) { + Expected<bool> EnableSplitLTOUnit = + getEnableSplitLTOUnitFlag(Stream, Entry.ID); + if (!EnableSplitLTOUnit) + return EnableSplitLTOUnit.takeError(); + return BitcodeLTOInfo{/*IsThinLTO=*/true, /*HasSummary=*/true, + *EnableSplitLTOUnit}; + } - if (Entry.ID == bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID) - return BitcodeLTOInfo{/*IsThinLTO=*/false, /*HasSummary=*/true}; + if (Entry.ID == bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID) { + Expected<bool> EnableSplitLTOUnit = + getEnableSplitLTOUnitFlag(Stream, Entry.ID); + if (!EnableSplitLTOUnit) + return EnableSplitLTOUnit.takeError(); + return BitcodeLTOInfo{/*IsThinLTO=*/false, /*HasSummary=*/true, + *EnableSplitLTOUnit}; + } // Ignore other sub-blocks. if (Stream.SkipBlock()) diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index 68d79edceaf..ba4f932e2e6 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -3618,6 +3618,13 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION}); + // Write the index flags. + uint64_t Flags = 0; + // Bits 1-3 are set only in the combined index, skip them. + if (Index->enableSplitLTOUnit()) + Flags |= 0x8; + Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Flags}); + if (Index->begin() == Index->end()) { Stream.ExitBlock(); return; @@ -3734,6 +3741,10 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Flags |= 0x2; if (Index.hasSyntheticEntryCounts()) Flags |= 0x4; + if (Index.enableSplitLTOUnit()) + Flags |= 0x8; + if (Index.partiallySplitLTOUnits()) + Flags |= 0x10; Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Flags}); for (const auto &GVI : valueIds()) { diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index f736ef8b7f9..3a955060dea 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -546,6 +546,15 @@ Error LTO::addModule(InputFile &Input, unsigned ModI, if (!LTOInfo) return LTOInfo.takeError(); + if (EnableSplitLTOUnit.hasValue()) { + // If only some modules were split, flag this in the index so that + // we can skip or error on optimizations that need consistently split + // modules (whole program devirt and lower type tests). + if (EnableSplitLTOUnit.getValue() != LTOInfo->EnableSplitLTOUnit) + ThinLTO.CombinedIndex.setPartiallySplitLTOUnits(); + } else + EnableSplitLTOUnit = LTOInfo->EnableSplitLTOUnit; + BitcodeModule BM = Input.Mods[ModI]; auto ModSyms = Input.module_symbols(ModI); addModuleToGlobalRes(ModSyms, {ResI, ResE}, diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp index e4dcd4d4dd7..87c65db0951 100644 --- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp +++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp @@ -1702,6 +1702,13 @@ bool LowerTypeTestsModule::lower() { !ExportSummary && !ImportSummary) return false; + // If only some of the modules were split, we cannot correctly handle + // code that contains type tests. + if (TypeTestFunc && !TypeTestFunc->use_empty() && + ((ExportSummary && ExportSummary->partiallySplitLTOUnits()) || + (ImportSummary && ImportSummary->partiallySplitLTOUnits()))) + report_fatal_error("inconsistent LTO Unit splitting with llvm.type.test"); + if (ImportSummary) { if (TypeTestFunc) { for (auto UI = TypeTestFunc->use_begin(), UE = TypeTestFunc->use_end(); diff --git a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp index a5382c4638d..510ecb516dc 100644 --- a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp +++ b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp @@ -418,8 +418,18 @@ void splitAndWriteThinLTOBitcode( } } -// Returns whether this module needs to be split because it uses type metadata. +// Returns whether this module needs to be split because splitting is +// enabled and it uses type metadata. bool requiresSplit(Module &M) { + // First check if the LTO Unit splitting has been enabled. + bool EnableSplitLTOUnit = false; + if (auto *MD = mdconst::extract_or_null<ConstantInt>( + M.getModuleFlag("EnableSplitLTOUnit"))) + EnableSplitLTOUnit = MD->getZExtValue(); + if (!EnableSplitLTOUnit) + return false; + + // Module only needs to be split if it contains type metadata. for (auto &GO : M.global_objects()) { if (GO.hasMetadata(LLVMContext::MD_type)) return true; @@ -431,7 +441,7 @@ bool requiresSplit(Module &M) { void writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS, function_ref<AAResults &(Function &)> AARGetter, Module &M, const ModuleSummaryIndex *Index) { - // See if this module has any type metadata. If so, we need to split it. + // Split module if splitting is enabled and it contains any type metadata. if (requiresSplit(M)) return splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M); diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp index 37905daee4c..48bd0cda759 100644 --- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp +++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp @@ -1563,6 +1563,17 @@ bool DevirtModule::run() { M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load)); Function *AssumeFunc = M.getFunction(Intrinsic::getName(Intrinsic::assume)); + // If only some of the modules were split, we cannot correctly handle + // code that contains type tests or type checked loads. + if ((ExportSummary && ExportSummary->partiallySplitLTOUnits()) || + (ImportSummary && ImportSummary->partiallySplitLTOUnits())) { + if ((TypeTestFunc && !TypeTestFunc->use_empty()) || + (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty())) + report_fatal_error("inconsistent LTO Unit splitting with llvm.type.test " + "or llvm.type.checked.load"); + return false; + } + // Normally if there are no users of the devirtualization intrinsics in the // module, this pass has nothing to do. But if we are exporting, we also need // to handle any users that appear only in the function summaries. |