summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
authorTeresa Johnson <tejohnson@google.com>2015-12-17 17:14:09 +0000
committerTeresa Johnson <tejohnson@google.com>2015-12-17 17:14:09 +0000
commite5a619173274bf2a9033ea1c8b2346f1bee2766b (patch)
treedf1a23654ddaef760e878af89977d13945649896 /llvm/lib
parentcaaa3aa07c8fa129c8032597f2d0317637a1569a (diff)
downloadbcm5719-llvm-e5a619173274bf2a9033ea1c8b2346f1bee2766b.tar.gz
bcm5719-llvm-e5a619173274bf2a9033ea1c8b2346f1bee2766b.zip
[ThinLTO] Metadata linking for imported functions
Summary: Second patch split out from http://reviews.llvm.org/D14752. Maps metadata as a post-pass from each module when importing complete, suturing up final metadata to the temporary metadata left on the imported instructions. This entails saving the mapping from bitcode value id to temporary metadata in the importing pass, and from bitcode value id to final metadata during the metadata linking postpass. Depends on D14825. Reviewers: dexonsmith, joker.eph Subscribers: davidxl, llvm-commits, joker.eph Differential Revision: http://reviews.llvm.org/D14838 llvm-svn: 255909
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/Bitcode/Reader/BitcodeReader.cpp45
-rw-r--r--llvm/lib/IR/Metadata.cpp4
-rw-r--r--llvm/lib/IRReader/IRReader.cpp12
-rw-r--r--llvm/lib/Linker/IRMover.cpp210
-rw-r--r--llvm/lib/Linker/LinkModules.cpp33
-rw-r--r--llvm/lib/Transforms/IPO/FunctionImport.cpp24
-rw-r--r--llvm/lib/Transforms/Utils/ValueMapper.cpp74
7 files changed, 343 insertions, 59 deletions
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 0905b5fb6d2..ff08f55d43f 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -97,6 +97,7 @@ public:
class BitcodeReaderMDValueList {
unsigned NumFwdRefs;
bool AnyFwdRefs;
+ bool SavedFwdRefs;
unsigned MinFwdRef;
unsigned MaxFwdRef;
std::vector<TrackingMDRef> MDValuePtrs;
@@ -104,7 +105,12 @@ class BitcodeReaderMDValueList {
LLVMContext &Context;
public:
BitcodeReaderMDValueList(LLVMContext &C)
- : NumFwdRefs(0), AnyFwdRefs(false), Context(C) {}
+ : NumFwdRefs(0), AnyFwdRefs(false), SavedFwdRefs(false), Context(C) {}
+ ~BitcodeReaderMDValueList() {
+ // Assert that we either replaced all forward references, or saved
+ // them for later replacement.
+ assert(!NumFwdRefs || SavedFwdRefs);
+ }
// vector compatibility methods
unsigned size() const { return MDValuePtrs.size(); }
@@ -115,6 +121,8 @@ public:
void pop_back() { MDValuePtrs.pop_back(); }
bool empty() const { return MDValuePtrs.empty(); }
+ void savedFwdRefs() { SavedFwdRefs = true; }
+
Metadata *operator[](unsigned i) const {
assert(i < MDValuePtrs.size());
return MDValuePtrs[i];
@@ -274,6 +282,14 @@ public:
void setStripDebugInfo() override;
+ /// Save the mapping between the metadata values and the corresponding
+ /// value id that were recorded in the MDValueList during parsing. If
+ /// OnlyTempMD is true, then only record those entries that are still
+ /// temporary metadata. This interface is used when metadata linking is
+ /// performed as a postpass, such as during function importing.
+ void saveMDValueList(DenseMap<const Metadata *, unsigned> &MDValueToValIDMap,
+ bool OnlyTempMD) override;
+
private:
/// Parse the "IDENTIFICATION_BLOCK_ID" block, populate the
// ProducerIdentification data member, and do some basic enforcement on the
@@ -1069,6 +1085,9 @@ Metadata *BitcodeReaderMDValueList::getValueFwdRef(unsigned Idx) {
MinFwdRef = MaxFwdRef = Idx;
}
++NumFwdRefs;
+ // Reset flag to ensure that we save this forward reference if we
+ // are delaying metadata mapping (e.g. for function importing).
+ SavedFwdRefs = false;
// Create and return a placeholder, which will later be RAUW'd.
Metadata *MD = MDNode::getTemporary(Context, None).release();
@@ -3062,6 +3081,30 @@ std::error_code BitcodeReader::materializeMetadata() {
void BitcodeReader::setStripDebugInfo() { StripDebugInfo = true; }
+void BitcodeReader::saveMDValueList(
+ DenseMap<const Metadata *, unsigned> &MDValueToValIDMap, bool OnlyTempMD) {
+ for (unsigned ValID = 0; ValID < MDValueList.size(); ++ValID) {
+ Metadata *MD = MDValueList[ValID];
+ auto *N = dyn_cast_or_null<MDNode>(MD);
+ // Save all values if !OnlyTempMD, otherwise just the temporary metadata.
+ if (!OnlyTempMD || (N && N->isTemporary())) {
+ // Will call this after materializing each function, in order to
+ // handle remapping of the function's instructions/metadata.
+ // See if we already have an entry in that case.
+ if (OnlyTempMD && MDValueToValIDMap.count(MD)) {
+ assert(MDValueToValIDMap[MD] == ValID &&
+ "Inconsistent metadata value id");
+ continue;
+ }
+ MDValueToValIDMap[MD] = ValID;
+ // Flag that we saved the forward refs (temporary metadata) for error
+ // checking during MDValueList destruction.
+ if (OnlyTempMD)
+ MDValueList.savedFwdRefs();
+ }
+ }
+}
+
/// When we see the block for a function body, remember where it is and then
/// skip it. This lets us lazily deserialize the functions.
std::error_code BitcodeReader::rememberAndSkipFunctionBody() {
diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp
index 80f18daa79b..b1da0301ecf 100644
--- a/llvm/lib/IR/Metadata.cpp
+++ b/llvm/lib/IR/Metadata.cpp
@@ -517,7 +517,7 @@ void MDNode::decrementUnresolvedOperandCount() {
resolve();
}
-void MDNode::resolveCycles() {
+void MDNode::resolveCycles(bool MDMaterialized) {
if (isResolved())
return;
@@ -530,6 +530,8 @@ void MDNode::resolveCycles() {
if (!N)
continue;
+ if (N->isTemporary() && !MDMaterialized)
+ continue;
assert(!N->isTemporary() &&
"Expected all forward declarations to be resolved");
if (!N->isResolved())
diff --git a/llvm/lib/IRReader/IRReader.cpp b/llvm/lib/IRReader/IRReader.cpp
index 43fee65db7f..9b243fc571d 100644
--- a/llvm/lib/IRReader/IRReader.cpp
+++ b/llvm/lib/IRReader/IRReader.cpp
@@ -31,11 +31,11 @@ static const char *const TimeIRParsingName = "Parse IR";
static std::unique_ptr<Module>
getLazyIRModule(std::unique_ptr<MemoryBuffer> Buffer, SMDiagnostic &Err,
- LLVMContext &Context) {
+ LLVMContext &Context, bool ShouldLazyLoadMetadata) {
if (isBitcode((const unsigned char *)Buffer->getBufferStart(),
(const unsigned char *)Buffer->getBufferEnd())) {
- ErrorOr<std::unique_ptr<Module>> ModuleOrErr =
- getLazyBitcodeModule(std::move(Buffer), Context);
+ ErrorOr<std::unique_ptr<Module>> ModuleOrErr = getLazyBitcodeModule(
+ std::move(Buffer), Context, ShouldLazyLoadMetadata);
if (std::error_code EC = ModuleOrErr.getError()) {
Err = SMDiagnostic(Buffer->getBufferIdentifier(), SourceMgr::DK_Error,
EC.message());
@@ -49,7 +49,8 @@ getLazyIRModule(std::unique_ptr<MemoryBuffer> Buffer, SMDiagnostic &Err,
std::unique_ptr<Module> llvm::getLazyIRFileModule(StringRef Filename,
SMDiagnostic &Err,
- LLVMContext &Context) {
+ LLVMContext &Context,
+ bool ShouldLazyLoadMetadata) {
ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
MemoryBuffer::getFileOrSTDIN(Filename);
if (std::error_code EC = FileOrErr.getError()) {
@@ -58,7 +59,8 @@ std::unique_ptr<Module> llvm::getLazyIRFileModule(StringRef Filename,
return nullptr;
}
- return getLazyIRModule(std::move(FileOrErr.get()), Err, Context);
+ return getLazyIRModule(std::move(FileOrErr.get()), Err, Context,
+ ShouldLazyLoadMetadata);
}
std::unique_ptr<Module> llvm::parseIR(MemoryBufferRef Buffer, SMDiagnostic &Err,
diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp
index 8a11a0099b5..01a0a425d8d 100644
--- a/llvm/lib/Linker/IRMover.cpp
+++ b/llvm/lib/Linker/IRMover.cpp
@@ -14,6 +14,7 @@
#include "llvm/ADT/Triple.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DiagnosticPrinter.h"
+#include "llvm/IR/GVMaterializer.h"
#include "llvm/IR/TypeFinder.h"
#include "llvm/Transforms/Utils/Cloning.h"
using namespace llvm;
@@ -349,6 +350,9 @@ public:
GlobalValueMaterializer(IRLinker *ModLinker) : ModLinker(ModLinker) {}
Value *materializeDeclFor(Value *V) override;
void materializeInitFor(GlobalValue *New, GlobalValue *Old) override;
+ Metadata *mapTemporaryMetadata(Metadata *MD) override;
+ void replaceTemporaryMetadata(const Metadata *OrigMD,
+ Metadata *NewMD) override;
};
class LocalValueMaterializer final : public ValueMaterializer {
@@ -358,6 +362,9 @@ public:
LocalValueMaterializer(IRLinker *ModLinker) : ModLinker(ModLinker) {}
Value *materializeDeclFor(Value *V) override;
void materializeInitFor(GlobalValue *New, GlobalValue *Old) override;
+ Metadata *mapTemporaryMetadata(Metadata *MD) override;
+ void replaceTemporaryMetadata(const Metadata *OrigMD,
+ Metadata *NewMD) override;
};
/// This is responsible for keeping track of the state used for moving data
@@ -394,6 +401,24 @@ class IRLinker {
bool HasError = false;
+ /// Flag indicating that we are just linking metadata (after function
+ /// importing).
+ bool IsMetadataLinkingPostpass;
+
+ /// Flags to pass to value mapper invocations.
+ RemapFlags ValueMapperFlags = RF_MoveDistinctMDs;
+
+ /// Association between metadata values created during bitcode parsing and
+ /// the value id. Used to correlate temporary metadata created during
+ /// function importing with the final metadata parsed during the subsequent
+ /// metadata linking postpass.
+ DenseMap<const Metadata *, unsigned> MDValueToValIDMap;
+
+ /// Association between metadata value id and temporary metadata that
+ /// remains unmapped after function importing. Saved during function
+ /// importing and consumed during the metadata linking postpass.
+ DenseMap<unsigned, MDNode *> *ValIDToTempMDMap;
+
/// Handles cloning of a global values from the source module into
/// the destination module, including setting the attributes and visibility.
GlobalValue *copyGlobalValueProto(const GlobalValue *SGV, bool ForDefinition);
@@ -409,6 +434,14 @@ class IRLinker {
SrcM.getContext().diagnose(LinkDiagnosticInfo(DS_Warning, Message));
}
+ /// Check whether we should be linking metadata from the source module.
+ bool shouldLinkMetadata() {
+ // ValIDToTempMDMap will be non-null when we are importing or otherwise want
+ // to link metadata lazily, and then when linking the metadata.
+ // We only want to return true for the former case.
+ return ValIDToTempMDMap == nullptr || IsMetadataLinkingPostpass;
+ }
+
/// Given a global in the source module, return the global in the
/// destination module that is being linked to, if any.
GlobalValue *getLinkedToGlobal(const GlobalValue *SrcGV) {
@@ -457,16 +490,35 @@ class IRLinker {
public:
IRLinker(Module &DstM, IRMover::IdentifiedStructTypeSet &Set, Module &SrcM,
ArrayRef<GlobalValue *> ValuesToLink,
- std::function<void(GlobalValue &, IRMover::ValueAdder)> AddLazyFor)
+ std::function<void(GlobalValue &, IRMover::ValueAdder)> AddLazyFor,
+ DenseMap<unsigned, MDNode *> *ValIDToTempMDMap = nullptr,
+ bool IsMetadataLinkingPostpass = false)
: DstM(DstM), SrcM(SrcM), AddLazyFor(AddLazyFor), TypeMap(Set),
- GValMaterializer(this), LValMaterializer(this) {
+ GValMaterializer(this), LValMaterializer(this),
+ IsMetadataLinkingPostpass(IsMetadataLinkingPostpass),
+ ValIDToTempMDMap(ValIDToTempMDMap) {
for (GlobalValue *GV : ValuesToLink)
maybeAdd(GV);
+
+ // If appropriate, tell the value mapper that it can expect to see
+ // temporary metadata.
+ if (!shouldLinkMetadata())
+ ValueMapperFlags = ValueMapperFlags | RF_HaveUnmaterializedMetadata;
}
bool run();
Value *materializeDeclFor(Value *V, bool ForAlias);
void materializeInitFor(GlobalValue *New, GlobalValue *Old, bool ForAlias);
+
+ /// Save the mapping between the given temporary metadata and its metadata
+ /// value id. Used to support metadata linking as a postpass for function
+ /// importing.
+ Metadata *mapTemporaryMetadata(Metadata *MD);
+
+ /// Replace any temporary metadata saved for the source metadata's id with
+ /// the new non-temporary metadata. Used when metadata linking as a postpass
+ /// for function importing.
+ void replaceTemporaryMetadata(const Metadata *OrigMD, Metadata *NewMD);
};
}
@@ -500,6 +552,15 @@ void GlobalValueMaterializer::materializeInitFor(GlobalValue *New,
ModLinker->materializeInitFor(New, Old, false);
}
+Metadata *GlobalValueMaterializer::mapTemporaryMetadata(Metadata *MD) {
+ return ModLinker->mapTemporaryMetadata(MD);
+}
+
+void GlobalValueMaterializer::replaceTemporaryMetadata(const Metadata *OrigMD,
+ Metadata *NewMD) {
+ ModLinker->replaceTemporaryMetadata(OrigMD, NewMD);
+}
+
Value *LocalValueMaterializer::materializeDeclFor(Value *V) {
return ModLinker->materializeDeclFor(V, true);
}
@@ -509,6 +570,15 @@ void LocalValueMaterializer::materializeInitFor(GlobalValue *New,
ModLinker->materializeInitFor(New, Old, true);
}
+Metadata *LocalValueMaterializer::mapTemporaryMetadata(Metadata *MD) {
+ return ModLinker->mapTemporaryMetadata(MD);
+}
+
+void LocalValueMaterializer::replaceTemporaryMetadata(const Metadata *OrigMD,
+ Metadata *NewMD) {
+ ModLinker->replaceTemporaryMetadata(OrigMD, NewMD);
+}
+
Value *IRLinker::materializeDeclFor(Value *V, bool ForAlias) {
auto *SGV = dyn_cast<GlobalValue>(V);
if (!SGV)
@@ -536,6 +606,51 @@ void IRLinker::materializeInitFor(GlobalValue *New, GlobalValue *Old,
linkGlobalValueBody(*New, *Old);
}
+Metadata *IRLinker::mapTemporaryMetadata(Metadata *MD) {
+ if (!ValIDToTempMDMap)
+ return nullptr;
+ // If this temporary metadata has a value id recorded during function
+ // parsing, record that in the ValIDToTempMDMap if one was provided.
+ if (MDValueToValIDMap.count(MD)) {
+ unsigned Idx = MDValueToValIDMap[MD];
+ // Check if we created a temp MD when importing a different function from
+ // this module. If so, reuse it the same temporary metadata, otherwise
+ // add this temporary metadata to the map.
+ if (!ValIDToTempMDMap->count(Idx)) {
+ MDNode *Node = cast<MDNode>(MD);
+ assert(Node->isTemporary());
+ (*ValIDToTempMDMap)[Idx] = Node;
+ }
+ return (*ValIDToTempMDMap)[Idx];
+ }
+ return nullptr;
+}
+
+void IRLinker::replaceTemporaryMetadata(const Metadata *OrigMD,
+ Metadata *NewMD) {
+ if (!ValIDToTempMDMap)
+ return;
+#ifndef NDEBUG
+ auto *N = dyn_cast_or_null<MDNode>(NewMD);
+ assert(!N || !N->isTemporary());
+#endif
+ // If a mapping between metadata value ids and temporary metadata
+ // created during function importing was provided, and the source
+ // metadata has a value id recorded during metadata parsing, replace
+ // the temporary metadata with the final mapped metadata now.
+ if (MDValueToValIDMap.count(OrigMD)) {
+ unsigned Idx = MDValueToValIDMap[OrigMD];
+ // Nothing to do if we didn't need to create a temporary metadata during
+ // function importing.
+ if (!ValIDToTempMDMap->count(Idx))
+ return;
+ MDNode *TempMD = (*ValIDToTempMDMap)[Idx];
+ TempMD->replaceAllUsesWith(NewMD);
+ MDNode::deleteTemporary(TempMD);
+ ValIDToTempMDMap->erase(Idx);
+ }
+}
+
/// Loop through the global variables in the src module and merge them into the
/// dest module.
GlobalVariable *IRLinker::copyGlobalVariableProto(const GlobalVariable *SGVar) {
@@ -793,16 +908,16 @@ Constant *IRLinker::linkAppendingVarProto(GlobalVariable *DstGV,
Constant *NewV;
if (IsOldStructor) {
auto *S = cast<ConstantStruct>(V);
- auto *E1 = MapValue(S->getOperand(0), ValueMap, RF_MoveDistinctMDs,
+ auto *E1 = MapValue(S->getOperand(0), ValueMap, ValueMapperFlags,
&TypeMap, &GValMaterializer);
- auto *E2 = MapValue(S->getOperand(1), ValueMap, RF_MoveDistinctMDs,
+ auto *E2 = MapValue(S->getOperand(1), ValueMap, ValueMapperFlags,
&TypeMap, &GValMaterializer);
Value *Null = Constant::getNullValue(VoidPtrTy);
NewV =
ConstantStruct::get(cast<StructType>(EltTy), E1, E2, Null, nullptr);
} else {
- NewV = MapValue(V, ValueMap, RF_MoveDistinctMDs, &TypeMap,
- &GValMaterializer);
+ NewV =
+ MapValue(V, ValueMap, ValueMapperFlags, &TypeMap, &GValMaterializer);
}
DstElements.push_back(NewV);
}
@@ -837,6 +952,14 @@ static bool useExistingDest(GlobalValue &SGV, GlobalValue *DGV,
}
bool IRLinker::shouldLink(GlobalValue *DGV, GlobalValue &SGV) {
+ // Already imported all the values. Just map to the Dest value
+ // in case it is referenced in the metadata.
+ if (IsMetadataLinkingPostpass) {
+ assert(!ValuesToLink.count(&SGV) &&
+ "Source value unexpectedly requested for link during metadata link");
+ return false;
+ }
+
if (ValuesToLink.count(&SGV))
return true;
@@ -925,8 +1048,8 @@ Constant *IRLinker::linkGlobalValueProto(GlobalValue *SGV, bool ForAlias) {
/// referenced are in Dest.
void IRLinker::linkGlobalInit(GlobalVariable &Dst, GlobalVariable &Src) {
// Figure out what the initializer looks like in the dest module.
- Dst.setInitializer(MapValue(Src.getInitializer(), ValueMap,
- RF_MoveDistinctMDs, &TypeMap, &GValMaterializer));
+ Dst.setInitializer(MapValue(Src.getInitializer(), ValueMap, ValueMapperFlags,
+ &TypeMap, &GValMaterializer));
}
/// Copy the source function over into the dest function and fix up references
@@ -939,22 +1062,28 @@ bool IRLinker::linkFunctionBody(Function &Dst, Function &Src) {
if (std::error_code EC = Src.materialize())
return emitError(EC.message());
+ if (!shouldLinkMetadata())
+ // This is only supported for lazy links. Do after materialization of
+ // a function and before remapping metadata on instructions below
+ // in RemapInstruction, as the saved mapping is used to handle
+ // the temporary metadata hanging off instructions.
+ SrcM.getMaterializer()->saveMDValueList(MDValueToValIDMap, true);
+
// Link in the prefix data.
if (Src.hasPrefixData())
- Dst.setPrefixData(MapValue(Src.getPrefixData(), ValueMap,
- RF_MoveDistinctMDs, &TypeMap,
- &GValMaterializer));
+ Dst.setPrefixData(MapValue(Src.getPrefixData(), ValueMap, ValueMapperFlags,
+ &TypeMap, &GValMaterializer));
// Link in the prologue data.
if (Src.hasPrologueData())
Dst.setPrologueData(MapValue(Src.getPrologueData(), ValueMap,
- RF_MoveDistinctMDs, &TypeMap,
+ ValueMapperFlags, &TypeMap,
&GValMaterializer));
// Link in the personality function.
if (Src.hasPersonalityFn())
Dst.setPersonalityFn(MapValue(Src.getPersonalityFn(), ValueMap,
- RF_MoveDistinctMDs, &TypeMap,
+ ValueMapperFlags, &TypeMap,
&GValMaterializer));
// Go through and convert function arguments over, remembering the mapping.
@@ -971,7 +1100,7 @@ bool IRLinker::linkFunctionBody(Function &Dst, Function &Src) {
SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
Src.getAllMetadata(MDs);
for (const auto &I : MDs)
- Dst.setMetadata(I.first, MapMetadata(I.second, ValueMap, RF_MoveDistinctMDs,
+ Dst.setMetadata(I.first, MapMetadata(I.second, ValueMap, ValueMapperFlags,
&TypeMap, &GValMaterializer));
// Splice the body of the source function into the dest function.
@@ -983,9 +1112,8 @@ bool IRLinker::linkFunctionBody(Function &Dst, Function &Src) {
// functions and patch them up to point to the local versions.
for (BasicBlock &BB : Dst)
for (Instruction &I : BB)
- RemapInstruction(&I, ValueMap,
- RF_IgnoreMissingEntries | RF_MoveDistinctMDs, &TypeMap,
- &GValMaterializer);
+ RemapInstruction(&I, ValueMap, RF_IgnoreMissingEntries | ValueMapperFlags,
+ &TypeMap, &GValMaterializer);
// There is no need to map the arguments anymore.
for (Argument &Arg : Src.args())
@@ -997,7 +1125,7 @@ bool IRLinker::linkFunctionBody(Function &Dst, Function &Src) {
void IRLinker::linkAliasBody(GlobalAlias &Dst, GlobalAlias &Src) {
Constant *Aliasee = Src.getAliasee();
- Constant *Val = MapValue(Aliasee, AliasValueMap, RF_MoveDistinctMDs, &TypeMap,
+ Constant *Val = MapValue(Aliasee, AliasValueMap, ValueMapperFlags, &TypeMap,
&LValMaterializer);
Dst.setAliasee(Val);
}
@@ -1024,7 +1152,7 @@ void IRLinker::linkNamedMDNodes() {
// Add Src elements into Dest node.
for (const MDNode *op : NMD.operands())
DestNMD->addOperand(MapMetadata(
- op, ValueMap, RF_MoveDistinctMDs | RF_NullMapMissingGlobalValues,
+ op, ValueMap, ValueMapperFlags | RF_NullMapMissingGlobalValues,
&TypeMap, &GValMaterializer));
}
}
@@ -1260,7 +1388,7 @@ bool IRLinker::run() {
continue;
assert(!GV->isDeclaration());
- MapValue(GV, ValueMap, RF_MoveDistinctMDs, &TypeMap, &GValMaterializer);
+ MapValue(GV, ValueMap, ValueMapperFlags, &TypeMap, &GValMaterializer);
if (HasError)
return true;
}
@@ -1272,11 +1400,39 @@ bool IRLinker::run() {
// Remap all of the named MDNodes in Src into the DstM module. We do this
// after linking GlobalValues so that MDNodes that reference GlobalValues
// are properly remapped.
- linkNamedMDNodes();
+ if (shouldLinkMetadata()) {
+ // Even if just linking metadata we should link decls above in case
+ // any are referenced by metadata. IRLinker::shouldLink ensures that
+ // we don't actually link anything from source.
+ if (IsMetadataLinkingPostpass) {
+ // Ensure metadata materialized
+ if (SrcM.getMaterializer()->materializeMetadata())
+ return true;
+ SrcM.getMaterializer()->saveMDValueList(MDValueToValIDMap, false);
+ }
- // Merge the module flags into the DstM module.
- if (linkModuleFlagsMetadata())
- return true;
+ linkNamedMDNodes();
+
+ if (IsMetadataLinkingPostpass) {
+ // Handle anything left in the ValIDToTempMDMap, such as metadata nodes
+ // not reached by the dbg.cu NamedMD (i.e. only reached from
+ // instructions).
+ // Walk the MDValueToValIDMap once to find the set of new (imported) MD
+ // that still has corresponding temporary metadata, and invoke metadata
+ // mapping on each one.
+ for (auto MDI : MDValueToValIDMap) {
+ if (!ValIDToTempMDMap->count(MDI.second))
+ continue;
+ MapMetadata(MDI.first, ValueMap, ValueMapperFlags, &TypeMap,
+ &GValMaterializer);
+ }
+ assert(ValIDToTempMDMap->empty());
+ }
+
+ // Merge the module flags into the DstM module.
+ if (linkModuleFlagsMetadata())
+ return true;
+ }
return false;
}
@@ -1384,9 +1540,11 @@ IRMover::IRMover(Module &M) : Composite(M) {
bool IRMover::move(
Module &Src, ArrayRef<GlobalValue *> ValuesToLink,
- std::function<void(GlobalValue &, ValueAdder Add)> AddLazyFor) {
+ std::function<void(GlobalValue &, ValueAdder Add)> AddLazyFor,
+ DenseMap<unsigned, MDNode *> *ValIDToTempMDMap,
+ bool IsMetadataLinkingPostpass) {
IRLinker TheLinker(Composite, IdentifiedStructTypes, Src, ValuesToLink,
- AddLazyFor);
+ AddLazyFor, ValIDToTempMDMap, IsMetadataLinkingPostpass);
bool RetCode = TheLinker.run();
Composite.dropTriviallyDeadConstantArrays();
return RetCode;
diff --git a/llvm/lib/Linker/LinkModules.cpp b/llvm/lib/Linker/LinkModules.cpp
index 9c9c0ef321f..556251092a2 100644
--- a/llvm/lib/Linker/LinkModules.cpp
+++ b/llvm/lib/Linker/LinkModules.cpp
@@ -48,6 +48,11 @@ class ModuleLinker {
/// as part of a different backend compilation process.
bool HasExportedFunctions = false;
+ /// Association between metadata value id and temporary metadata that
+ /// remains unmapped after function importing. Saved during function
+ /// importing and consumed during the metadata linking postpass.
+ DenseMap<unsigned, MDNode *> *ValIDToTempMDMap;
+
/// Used as the callback for lazy linking.
/// The mover has just hit GV and we have to decide if it, and other members
/// of the same comdat, should be linked. Every member to be linked is passed
@@ -150,9 +155,10 @@ class ModuleLinker {
public:
ModuleLinker(IRMover &Mover, Module &SrcM, unsigned Flags,
const FunctionInfoIndex *Index = nullptr,
- DenseSet<const GlobalValue *> *FunctionsToImport = nullptr)
+ DenseSet<const GlobalValue *> *FunctionsToImport = nullptr,
+ DenseMap<unsigned, MDNode *> *ValIDToTempMDMap = nullptr)
: Mover(Mover), SrcM(SrcM), Flags(Flags), ImportIndex(Index),
- ImportFunction(FunctionsToImport) {
+ ImportFunction(FunctionsToImport), ValIDToTempMDMap(ValIDToTempMDMap) {
assert((ImportIndex || !ImportFunction) &&
"Expect a FunctionInfoIndex when importing");
// If we have a FunctionInfoIndex but no function to import,
@@ -161,6 +167,8 @@ public:
// may be exported to another backend compilation.
if (ImportIndex && !ImportFunction)
HasExportedFunctions = ImportIndex->hasExportedFunctions(SrcM);
+ assert((ValIDToTempMDMap || !ImportFunction) &&
+ "Function importing must provide a ValIDToTempMDMap");
}
bool run();
@@ -502,6 +510,7 @@ bool ModuleLinker::getComdatResult(const Comdat *SrcC,
bool ModuleLinker::shouldLinkFromSource(bool &LinkFromSrc,
const GlobalValue &Dest,
const GlobalValue &Src) {
+
// Should we unconditionally use the Src?
if (shouldOverrideFromSrc()) {
LinkFromSrc = true;
@@ -776,7 +785,8 @@ bool ModuleLinker::run() {
if (Mover.move(SrcM, ValuesToLink.getArrayRef(),
[this](GlobalValue &GV, IRMover::ValueAdder Add) {
addLazyFor(GV, Add);
- }))
+ },
+ ValIDToTempMDMap, false))
return true;
Module &DstM = Mover.getModule();
for (auto &P : Internalize) {
@@ -791,8 +801,10 @@ Linker::Linker(Module &M) : Mover(M) {}
bool Linker::linkInModule(std::unique_ptr<Module> Src, unsigned Flags,
const FunctionInfoIndex *Index,
- DenseSet<const GlobalValue *> *FunctionsToImport) {
- ModuleLinker TheLinker(Mover, *Src, Flags, Index, FunctionsToImport);
+ DenseSet<const GlobalValue *> *FunctionsToImport,
+ DenseMap<unsigned, MDNode *> *ValIDToTempMDMap) {
+ ModuleLinker TheLinker(Mover, *Src, Flags, Index, FunctionsToImport,
+ ValIDToTempMDMap);
return TheLinker.run();
}
@@ -801,6 +813,17 @@ bool Linker::linkInModuleForCAPI(Module &Src) {
return TheLinker.run();
}
+bool Linker::linkInMetadata(Module &Src,
+ DenseMap<unsigned, MDNode *> *ValIDToTempMDMap) {
+ SetVector<GlobalValue *> ValuesToLink;
+ if (Mover.move(
+ Src, ValuesToLink.getArrayRef(),
+ [this](GlobalValue &GV, IRMover::ValueAdder Add) { assert(false); },
+ ValIDToTempMDMap, true))
+ return true;
+ return false;
+}
+
//===----------------------------------------------------------------------===//
// LinkModules entrypoint.
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp
index b8c2ea7dce2..639f15a03f5 100644
--- a/llvm/lib/Transforms/IPO/FunctionImport.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp
@@ -292,6 +292,9 @@ bool FunctionImporter::importFunctions(Module &DestModule) {
ModuleToFunctionsToImportMap, Index, ModuleLoaderCache);
assert(Worklist.empty() && "Worklist hasn't been flushed in GetImportList");
+ StringMap<std::unique_ptr<DenseMap<unsigned, MDNode *>>>
+ ModuleToTempMDValsMap;
+
// Do the actual import of functions now, one Module at a time
for (auto &FunctionsToImportPerModule : ModuleToFunctionsToImportMap) {
// Get the module for the import
@@ -301,13 +304,32 @@ bool FunctionImporter::importFunctions(Module &DestModule) {
assert(&DestModule.getContext() == &SrcModule->getContext() &&
"Context mismatch");
+ // Save the mapping of value ids to temporary metadata created when
+ // importing this function. If we have already imported from this module,
+ // add new temporary metadata to the existing mapping.
+ auto &TempMDVals = ModuleToTempMDValsMap[SrcModule->getModuleIdentifier()];
+ if (!TempMDVals)
+ TempMDVals = llvm::make_unique<DenseMap<unsigned, MDNode *>>();
+
// Link in the specified functions.
if (TheLinker.linkInModule(std::move(SrcModule), Linker::Flags::None,
- &Index, &FunctionsToImport))
+ &Index, &FunctionsToImport, TempMDVals.get()))
report_fatal_error("Function Import: link error");
ImportedCount += FunctionsToImport.size();
}
+
+ // Now link in metadata for all modules from which we imported functions.
+ for (StringMapEntry<std::unique_ptr<DenseMap<unsigned, MDNode *>>> &SME :
+ ModuleToTempMDValsMap) {
+ // Load the specified source module.
+ auto &SrcModule = ModuleLoaderCache(SME.getKey());
+
+ // Link in all necessary metadata from this module.
+ if (TheLinker.linkInMetadata(SrcModule, SME.getValue().get()))
+ return false;
+ }
+
DEBUG(dbgs() << "Imported " << ImportedCount << " functions for Module "
<< DestModule.getModuleIdentifier() << "\n");
return ImportedCount;
diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp
index 00a8984845d..00ee3385981 100644
--- a/llvm/lib/Transforms/Utils/ValueMapper.cpp
+++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp
@@ -167,13 +167,21 @@ Value *llvm::MapValue(const Value *V, ValueToValueMapTy &VM, RemapFlags Flags,
}
static Metadata *mapToMetadata(ValueToValueMapTy &VM, const Metadata *Key,
- Metadata *Val) {
+ Metadata *Val, ValueMaterializer *Materializer,
+ RemapFlags Flags) {
VM.MD()[Key].reset(Val);
+ if (Materializer && !(Flags & RF_HaveUnmaterializedMetadata)) {
+ auto *N = dyn_cast_or_null<MDNode>(Val);
+ // Need to invoke this once we have non-temporary MD.
+ if (!N || !N->isTemporary())
+ Materializer->replaceTemporaryMetadata(Key, Val);
+ }
return Val;
}
-static Metadata *mapToSelf(ValueToValueMapTy &VM, const Metadata *MD) {
- return mapToMetadata(VM, MD, const_cast<Metadata *>(MD));
+static Metadata *mapToSelf(ValueToValueMapTy &VM, const Metadata *MD,
+ ValueMaterializer *Materializer, RemapFlags Flags) {
+ return mapToMetadata(VM, MD, const_cast<Metadata *>(MD), Materializer, Flags);
}
static Metadata *MapMetadataImpl(const Metadata *MD,
@@ -206,10 +214,13 @@ static Metadata *mapMetadataOp(Metadata *Op,
}
/// Resolve uniquing cycles involving the given metadata.
-static void resolveCycles(Metadata *MD) {
- if (auto *N = dyn_cast_or_null<MDNode>(MD))
+static void resolveCycles(Metadata *MD, bool MDMaterialized) {
+ if (auto *N = dyn_cast_or_null<MDNode>(MD)) {
+ if (!MDMaterialized && N->isTemporary())
+ return;
if (!N->isResolved())
- N->resolveCycles();
+ N->resolveCycles(MDMaterialized);
+ }
}
/// Remap the operands of an MDNode.
@@ -238,7 +249,7 @@ static bool remapOperands(MDNode &Node,
// Resolve uniquing cycles underneath distinct nodes on the fly so they
// don't infect later operands.
if (IsDistinct)
- resolveCycles(New);
+ resolveCycles(New, !(Flags & RF_HaveUnmaterializedMetadata));
}
}
@@ -266,7 +277,7 @@ static Metadata *mapDistinctNode(const MDNode *Node,
// Remap operands later.
DistinctWorklist.push_back(NewMD);
- return mapToMetadata(VM, Node, NewMD);
+ return mapToMetadata(VM, Node, NewMD, Materializer, Flags);
}
/// \brief Map a uniqued MDNode.
@@ -277,22 +288,29 @@ static Metadata *mapUniquedNode(const MDNode *Node,
ValueToValueMapTy &VM, RemapFlags Flags,
ValueMapTypeRemapper *TypeMapper,
ValueMaterializer *Materializer) {
- assert(Node->isUniqued() && "Expected uniqued node");
+ assert(((Flags & RF_HaveUnmaterializedMetadata) || Node->isUniqued()) &&
+ "Expected uniqued node");
// Create a temporary node and map it upfront in case we have a uniquing
// cycle. If necessary, this mapping will get updated by RAUW logic before
// returning.
auto ClonedMD = Node->clone();
- mapToMetadata(VM, Node, ClonedMD.get());
+ mapToMetadata(VM, Node, ClonedMD.get(), Materializer, Flags);
if (!remapOperands(*ClonedMD, DistinctWorklist, VM, Flags, TypeMapper,
Materializer)) {
// No operands changed, so use the original.
ClonedMD->replaceAllUsesWith(const_cast<MDNode *>(Node));
- return const_cast<MDNode *>(Node);
+ // Even though replaceAllUsesWith would have replaced the value map
+ // entry, we need to explictly map with the final non-temporary node
+ // to replace any temporary metadata via the callback.
+ return mapToSelf(VM, Node, Materializer, Flags);
}
- // Uniquify the cloned node.
- return MDNode::replaceWithUniqued(std::move(ClonedMD));
+ // Uniquify the cloned node. Explicitly map it with the final non-temporary
+ // node so that replacement of temporary metadata via the callback occurs.
+ return mapToMetadata(VM, Node,
+ MDNode::replaceWithUniqued(std::move(ClonedMD)),
+ Materializer, Flags);
}
static Metadata *MapMetadataImpl(const Metadata *MD,
@@ -305,18 +323,18 @@ static Metadata *MapMetadataImpl(const Metadata *MD,
return NewMD;
if (isa<MDString>(MD))
- return mapToSelf(VM, MD);
+ return mapToSelf(VM, MD, Materializer, Flags);
if (isa<ConstantAsMetadata>(MD))
if ((Flags & RF_NoModuleLevelChanges))
- return mapToSelf(VM, MD);
+ return mapToSelf(VM, MD, Materializer, Flags);
if (const auto *VMD = dyn_cast<ValueAsMetadata>(MD)) {
Value *MappedV =
MapValue(VMD->getValue(), VM, Flags, TypeMapper, Materializer);
if (VMD->getValue() == MappedV ||
(!MappedV && (Flags & RF_IgnoreMissingEntries)))
- return mapToSelf(VM, MD);
+ return mapToSelf(VM, MD, Materializer, Flags);
// FIXME: This assert crashes during bootstrap, but I think it should be
// correct. For now, just match behaviour from before the metadata/value
@@ -325,7 +343,8 @@ static Metadata *MapMetadataImpl(const Metadata *MD,
// assert((MappedV || (Flags & RF_NullMapMissingGlobalValues)) &&
// "Referenced metadata not in value map!");
if (MappedV)
- return mapToMetadata(VM, MD, ValueAsMetadata::get(MappedV));
+ return mapToMetadata(VM, MD, ValueAsMetadata::get(MappedV), Materializer,
+ Flags);
return nullptr;
}
@@ -336,10 +355,25 @@ static Metadata *MapMetadataImpl(const Metadata *MD,
// If this is a module-level metadata and we know that nothing at the
// module level is changing, then use an identity mapping.
if (Flags & RF_NoModuleLevelChanges)
- return mapToSelf(VM, MD);
+ return mapToSelf(VM, MD, Materializer, Flags);
// Require resolved nodes whenever metadata might be remapped.
- assert(Node->isResolved() && "Unexpected unresolved node");
+ assert(((Flags & RF_HaveUnmaterializedMetadata) || Node->isResolved()) &&
+ "Unexpected unresolved node");
+
+ if (Materializer && Node->isTemporary()) {
+ assert(Flags & RF_HaveUnmaterializedMetadata);
+ Metadata *TempMD =
+ Materializer->mapTemporaryMetadata(const_cast<Metadata *>(MD));
+ // If the above callback returned an existing temporary node, use it
+ // instead of the current temporary node. This happens when earlier
+ // function importing passes already created and saved a temporary
+ // metadata node for the same value id.
+ if (TempMD) {
+ mapToMetadata(VM, MD, TempMD, Materializer, Flags);
+ return TempMD;
+ }
+ }
if (Node->isDistinct())
return mapDistinctNode(Node, DistinctWorklist, VM, Flags, TypeMapper,
@@ -363,7 +397,7 @@ Metadata *llvm::MapMetadata(const Metadata *MD, ValueToValueMapTy &VM,
return NewMD;
// Resolve cycles involving the entry metadata.
- resolveCycles(NewMD);
+ resolveCycles(NewMD, !(Flags & RF_HaveUnmaterializedMetadata));
// Remap the operands of distinct MDNodes.
while (!DistinctWorklist.empty())
OpenPOWER on IntegriCloud