summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--clang/include/clang/Basic/Attr.td33
-rw-r--r--clang/lib/CodeGen/CodeGenFunction.cpp80
-rw-r--r--clang/lib/CodeGen/CodeGenFunction.h41
-rw-r--r--clang/lib/CodeGen/CodeGenModule.cpp47
4 files changed, 108 insertions, 93 deletions
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 1f17819dba7..dc9edace50d 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -1953,6 +1953,39 @@ def Target : InheritableAttr {
return parse(getFeaturesStr());
}
+ StringRef getArchitecture() const {
+ StringRef Features = getFeaturesStr();
+ if (Features == "default") return {};
+
+ SmallVector<StringRef, 1> AttrFeatures;
+ Features.split(AttrFeatures, ",");
+
+ for (auto &Feature : AttrFeatures) {
+ Feature = Feature.trim();
+ if (Feature.startswith("arch="))
+ return Feature.drop_front(sizeof("arch=") - 1);
+ }
+ return "";
+ }
+
+ // Gets the list of features as simple string-refs with no +/- or 'no-'.
+ // Only adds the items to 'Out' that are additions.
+ void getAddedFeatures(llvm::SmallVectorImpl<StringRef> &Out) const {
+ StringRef Features = getFeaturesStr();
+ if (Features == "default") return;
+
+ SmallVector<StringRef, 1> AttrFeatures;
+ Features.split(AttrFeatures, ",");
+
+ for (auto &Feature : AttrFeatures) {
+ Feature = Feature.trim();
+
+ if (!Feature.startswith("no-") && !Feature.startswith("arch=") &&
+ !Feature.startswith("fpmath=") && !Feature.startswith("tune="))
+ Out.push_back(Feature);
+ }
+ }
+
template<class Compare>
ParsedTargetAttr parse(Compare cmp) const {
ParsedTargetAttr Attrs = parse();
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 3c582688e91..166d588dd55 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -2359,91 +2359,53 @@ void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
CGM.getSanStats().create(IRB, SSK);
}
-llvm::Value *CodeGenFunction::FormResolverCondition(
- const TargetMultiVersionResolverOption &RO) {
- llvm::Value *TrueCondition = nullptr;
- if (!RO.ParsedAttribute.Architecture.empty())
- TrueCondition = EmitX86CpuIs(RO.ParsedAttribute.Architecture);
-
- if (!RO.ParsedAttribute.Features.empty()) {
- SmallVector<StringRef, 8> FeatureList;
- llvm::for_each(RO.ParsedAttribute.Features,
- [&FeatureList](const std::string &Feature) {
- FeatureList.push_back(StringRef{Feature}.substr(1));
- });
- llvm::Value *FeatureCmp = EmitX86CpuSupports(FeatureList);
- TrueCondition = TrueCondition ? Builder.CreateAnd(TrueCondition, FeatureCmp)
- : FeatureCmp;
- }
- return TrueCondition;
-}
-
-void CodeGenFunction::EmitTargetMultiVersionResolver(
- llvm::Function *Resolver,
- ArrayRef<TargetMultiVersionResolverOption> Options) {
- assert((getContext().getTargetInfo().getTriple().getArch() ==
- llvm::Triple::x86 ||
- getContext().getTargetInfo().getTriple().getArch() ==
- llvm::Triple::x86_64) &&
- "Only implemented for x86 targets");
-
- // Main function's basic block.
- llvm::BasicBlock *CurBlock = createBasicBlock("entry", Resolver);
- Builder.SetInsertPoint(CurBlock);
- EmitX86CpuInit();
+llvm::Value *
+CodeGenFunction::FormResolverCondition(const MultiVersionResolverOption &RO) {
+ llvm::Value *Condition = nullptr;
- llvm::Function *DefaultFunc = nullptr;
- for (const TargetMultiVersionResolverOption &RO : Options) {
- Builder.SetInsertPoint(CurBlock);
- llvm::Value *TrueCondition = FormResolverCondition(RO);
+ if (!RO.Conditions.Architecture.empty())
+ Condition = EmitX86CpuIs(RO.Conditions.Architecture);
- if (!TrueCondition) {
- DefaultFunc = RO.Function;
- } else {
- llvm::BasicBlock *RetBlock = createBasicBlock("ro_ret", Resolver);
- llvm::IRBuilder<> RetBuilder(RetBlock);
- RetBuilder.CreateRet(RO.Function);
- CurBlock = createBasicBlock("ro_else", Resolver);
- Builder.CreateCondBr(TrueCondition, RetBlock, CurBlock);
- }
+ if (!RO.Conditions.Features.empty()) {
+ llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Conditions.Features);
+ Condition =
+ Condition ? Builder.CreateAnd(Condition, FeatureCond) : FeatureCond;
}
-
- assert(DefaultFunc && "No default version?");
- // Emit return from the 'else-ist' block.
- Builder.SetInsertPoint(CurBlock);
- Builder.CreateRet(DefaultFunc);
+ return Condition;
}
-void CodeGenFunction::EmitCPUDispatchMultiVersionResolver(
- llvm::Function *Resolver,
- ArrayRef<CPUDispatchMultiVersionResolverOption> Options) {
+void CodeGenFunction::EmitMultiVersionResolver(
+ llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
assert((getContext().getTargetInfo().getTriple().getArch() ==
llvm::Triple::x86 ||
getContext().getTargetInfo().getTriple().getArch() ==
llvm::Triple::x86_64) &&
"Only implemented for x86 targets");
-
// Main function's basic block.
llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
Builder.SetInsertPoint(CurBlock);
EmitX86CpuInit();
- for (const CPUDispatchMultiVersionResolverOption &RO : Options) {
+ for (const MultiVersionResolverOption &RO : Options) {
Builder.SetInsertPoint(CurBlock);
+ llvm::Value *Condition = FormResolverCondition(RO);
- // "generic" case should catch-all.
- if (RO.FeatureMask == 0) {
+ // The 'default' or 'generic' case.
+ if (!Condition) {
+ assert(&RO == Options.end() - 1 &&
+ "Default or Generic case must be last");
Builder.CreateRet(RO.Function);
return;
}
+
llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
llvm::IRBuilder<> RetBuilder(RetBlock);
RetBuilder.CreateRet(RO.Function);
CurBlock = createBasicBlock("resolver_else", Resolver);
- llvm::Value *TrueCondition = EmitX86CpuSupports(RO.FeatureMask);
- Builder.CreateCondBr(TrueCondition, RetBlock, CurBlock);
+ Builder.CreateCondBr(Condition, RetBlock, CurBlock);
}
+ // If no generic/default, emit an unreachable.
Builder.SetInsertPoint(CurBlock);
llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
TrapCall->setDoesNotReturn();
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 878923a85bd..d374f394366 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -4247,30 +4247,26 @@ public:
void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);
- struct TargetMultiVersionResolverOption {
+ struct MultiVersionResolverOption {
llvm::Function *Function;
- TargetAttr::ParsedTargetAttr ParsedAttribute;
- unsigned Priority;
- TargetMultiVersionResolverOption(
- const TargetInfo &TargInfo, llvm::Function *F,
- const clang::TargetAttr::ParsedTargetAttr &PT)
- : Function(F), ParsedAttribute(PT), Priority(0u) {
- for (StringRef Feat : PT.Features)
- Priority = std::max(Priority,
- TargInfo.multiVersionSortPriority(Feat.substr(1)));
-
- if (!PT.Architecture.empty())
- Priority = std::max(Priority,
- TargInfo.multiVersionSortPriority(PT.Architecture));
- }
+ struct Conds {
+ StringRef Architecture;
+ llvm::SmallVector<StringRef, 8> Features;
- bool operator>(const TargetMultiVersionResolverOption &Other) const {
- return Priority > Other.Priority;
- }
+ Conds(StringRef Arch, ArrayRef<StringRef> Feats)
+ : Architecture(Arch), Features(Feats.begin(), Feats.end()) {}
+ } Conditions;
+
+ MultiVersionResolverOption(llvm::Function *F, StringRef Arch,
+ ArrayRef<StringRef> Feats)
+ : Function(F), Conditions(Arch, Feats) {}
};
- void EmitTargetMultiVersionResolver(
- llvm::Function *Resolver,
- ArrayRef<TargetMultiVersionResolverOption> Options);
+
+ // Emits the body of a multiversion function's resolver. Assumes that the
+ // options are already sorted in the proper order, with the 'default' option
+ // last (if it exists).
+ void EmitMultiVersionResolver(llvm::Function *Resolver,
+ ArrayRef<MultiVersionResolverOption> Options);
struct CPUDispatchMultiVersionResolverOption {
llvm::Function *Function;
@@ -4306,8 +4302,7 @@ private:
llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
llvm::Value *EmitX86CpuSupports(uint32_t Mask);
llvm::Value *EmitX86CpuInit();
- llvm::Value *
- FormResolverCondition(const TargetMultiVersionResolverOption &RO);
+ llvm::Value *FormResolverCondition(const MultiVersionResolverOption &RO);
};
inline DominatingLLVMValue::saved_type
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 76112e191c7..2654247b959 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -2399,9 +2399,22 @@ void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) {
static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
llvm::Function *NewFn);
+static unsigned
+TargetMVPriority(const TargetInfo &TI,
+ const CodeGenFunction::MultiVersionResolverOption &RO) {
+ unsigned Priority = 0;
+ for (StringRef Feat : RO.Conditions.Features)
+ Priority = std::max(Priority, TI.multiVersionSortPriority(Feat));
+
+ if (!RO.Conditions.Architecture.empty())
+ Priority = std::max(
+ Priority, TI.multiVersionSortPriority(RO.Conditions.Architecture));
+ return Priority;
+}
+
void CodeGenModule::emitMultiVersionFunctions() {
for (GlobalDecl GD : MultiVersionFuncs) {
- SmallVector<CodeGenFunction::TargetMultiVersionResolverOption, 10> Options;
+ SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
getContext().forEachMultiversionedFunctionVersion(
FD, [this, &GD, &Options](const FunctionDecl *CurFD) {
@@ -2422,8 +2435,13 @@ void CodeGenModule::emitMultiVersionFunctions() {
}
assert(Func && "This should have just been created");
}
- Options.emplace_back(getTarget(), cast<llvm::Function>(Func),
- CurFD->getAttr<TargetAttr>()->parse());
+
+ const auto *TA = CurFD->getAttr<TargetAttr>();
+ llvm::SmallVector<StringRef, 8> Feats;
+ TA->getAddedFeatures(Feats);
+
+ Options.emplace_back(cast<llvm::Function>(Func),
+ TA->getArchitecture(), Feats);
});
llvm::Function *ResolverFunc = cast<llvm::Function>(
@@ -2431,11 +2449,16 @@ void CodeGenModule::emitMultiVersionFunctions() {
if (supportsCOMDAT())
ResolverFunc->setComdat(
getModule().getOrInsertComdat(ResolverFunc->getName()));
+
+ const TargetInfo &TI = getTarget();
std::stable_sort(
Options.begin(), Options.end(),
- std::greater<CodeGenFunction::TargetMultiVersionResolverOption>());
+ [&TI](const CodeGenFunction::MultiVersionResolverOption &LHS,
+ const CodeGenFunction::MultiVersionResolverOption &RHS) {
+ return TargetMVPriority(TI, LHS) > TargetMVPriority(TI, RHS);
+ });
CodeGenFunction CGF(*this);
- CGF.EmitTargetMultiVersionResolver(ResolverFunc, Options);
+ CGF.EmitMultiVersionResolver(ResolverFunc, Options);
}
}
@@ -2455,8 +2478,7 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
GetOrCreateLLVMFunction(ResolverName, ResolverType, GlobalDecl{},
/*ForVTable=*/false));
- SmallVector<CodeGenFunction::CPUDispatchMultiVersionResolverOption, 10>
- Options;
+ SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
const TargetInfo &Target = getTarget();
for (const IdentifierInfo *II : DD->cpus()) {
// Get the name of the target function so we can look it up/create it.
@@ -2473,15 +2495,18 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
Features.begin(), Features.end(), [&Target](StringRef Feat) {
return !Target.validateCpuSupports(Feat);
}), Features.end());
- Options.emplace_back(cast<llvm::Function>(Func),
- CodeGenFunction::GetX86CpuSupportsMask(Features));
+ Options.emplace_back(cast<llvm::Function>(Func), StringRef{}, Features);
}
llvm::sort(
Options.begin(), Options.end(),
- std::greater<CodeGenFunction::CPUDispatchMultiVersionResolverOption>());
+ [](const CodeGenFunction::MultiVersionResolverOption &LHS,
+ const CodeGenFunction::MultiVersionResolverOption &RHS) {
+ return CodeGenFunction::GetX86CpuSupportsMask(LHS.Conditions.Features) >
+ CodeGenFunction::GetX86CpuSupportsMask(RHS.Conditions.Features);
+ });
CodeGenFunction CGF(*this);
- CGF.EmitCPUDispatchMultiVersionResolver(ResolverFunc, Options);
+ CGF.EmitMultiVersionResolver(ResolverFunc, Options);
}
/// If an ifunc for the specified mangled name is not in the module, create and
OpenPOWER on IntegriCloud