summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--llvm/include/llvm/IR/Function.h6
-rw-r--r--llvm/include/llvm/IR/LLVMContext.h1
-rw-r--r--llvm/include/llvm/IR/MDBuilder.h3
-rw-r--r--llvm/lib/CodeGen/CodeGenPrepare.cpp22
-rw-r--r--llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp8
-rw-r--r--llvm/lib/IR/Function.cpp17
-rw-r--r--llvm/lib/IR/LLVMContext.cpp5
-rw-r--r--llvm/lib/IR/MDBuilder.cpp6
-rw-r--r--llvm/test/Transforms/CodeGenPrepare/section.ll38
9 files changed, 102 insertions, 4 deletions
diff --git a/llvm/include/llvm/IR/Function.h b/llvm/include/llvm/IR/Function.h
index d390e29ab6c..57b328c39c7 100644
--- a/llvm/include/llvm/IR/Function.h
+++ b/llvm/include/llvm/IR/Function.h
@@ -203,6 +203,12 @@ public:
/// pgo data.
Optional<uint64_t> getEntryCount() const;
+ /// Set the section prefix for this function.
+ void setSectionPrefix(StringRef Prefix);
+
+ /// Get the section prefix for this function.
+ Optional<StringRef> getSectionPrefix() const;
+
/// @brief Return true if the function has the attribute.
bool hasFnAttribute(Attribute::AttrKind Kind) const {
return AttributeSets.hasFnAttribute(Kind);
diff --git a/llvm/include/llvm/IR/LLVMContext.h b/llvm/include/llvm/IR/LLVMContext.h
index df9368904a6..16286d73289 100644
--- a/llvm/include/llvm/IR/LLVMContext.h
+++ b/llvm/include/llvm/IR/LLVMContext.h
@@ -72,6 +72,7 @@ public:
MD_align = 17, // "align"
MD_loop = 18, // "llvm.loop"
MD_type = 19, // "type"
+ MD_section_prefix = 20, // "section_prefix"
};
/// Known operand bundle tag IDs, which always have the same value. All
diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h
index 35341e3271f..bab8728ed49 100644
--- a/llvm/include/llvm/IR/MDBuilder.h
+++ b/llvm/include/llvm/IR/MDBuilder.h
@@ -66,6 +66,9 @@ public:
/// Return metadata containing the entry count for a function.
MDNode *createFunctionEntryCount(uint64_t Count);
+ /// Return metadata containing the section prefix for a function.
+ MDNode *createFunctionSectionPrefix(StringRef Prefix);
+
//===------------------------------------------------------------------===//
// Range metadata.
//===------------------------------------------------------------------===//
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 62bc755ebab..d102ccbd1b8 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -19,6 +19,7 @@
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
@@ -119,6 +120,10 @@ static cl::opt<bool> DisablePreheaderProtect(
"disable-preheader-prot", cl::Hidden, cl::init(false),
cl::desc("Disable protection against removing loop preheaders"));
+static cl::opt<bool> ProfileGuidedSectionPrefix(
+ "profile-guided-section-prefix", cl::Hidden, cl::init(true),
+ cl::desc("Use profile info to add section prefix for hot/cold functions"));
+
namespace {
typedef SmallPtrSet<Instruction *, 16> SetOfInstrs;
typedef PointerIntPair<Type *, 1, bool> TypeIsSExt;
@@ -168,6 +173,7 @@ class TypePromotionTransaction;
void getAnalysisUsage(AnalysisUsage &AU) const override {
// FIXME: When we can selectively preserve passes, preserve the domtree.
+ AU.addRequired<ProfileSummaryInfoWrapperPass>();
AU.addRequired<TargetLibraryInfoWrapperPass>();
AU.addRequired<TargetTransformInfoWrapperPass>();
AU.addRequired<LoopInfoWrapperPass>();
@@ -205,8 +211,11 @@ class TypePromotionTransaction;
}
char CodeGenPrepare::ID = 0;
-INITIALIZE_TM_PASS(CodeGenPrepare, "codegenprepare",
- "Optimize for code generation", false, false)
+INITIALIZE_TM_PASS_BEGIN(CodeGenPrepare, "codegenprepare",
+ "Optimize for code generation", false, false)
+INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
+INITIALIZE_TM_PASS_END(CodeGenPrepare, "codegenprepare",
+ "Optimize for code generation", false, false)
FunctionPass *llvm::createCodeGenPreparePass(const TargetMachine *TM) {
return new CodeGenPrepare(TM);
@@ -231,6 +240,15 @@ bool CodeGenPrepare::runOnFunction(Function &F) {
LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
OptSize = F.optForSize();
+ if (ProfileGuidedSectionPrefix) {
+ ProfileSummaryInfo *PSI =
+ getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
+ if (PSI->isFunctionEntryHot(&F))
+ F.setSectionPrefix(".hot");
+ else if (PSI->isFunctionEntryCold(&F))
+ F.setSectionPrefix(".cold");
+ }
+
/// This optimization identifies DIV instructions that can be
/// profitably bypassed and carried out with a shorter, faster divide.
if (!OptSize && TLI && TLI->isSlowDivBypassed()) {
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index edaa778a605..91cee337c98 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -296,8 +296,12 @@ selectELFSectionForGlobal(MCContext &Ctx, const GlobalValue *GV,
} else {
Name = getSectionPrefixForGlobal(Kind);
}
- // FIXME: Extend the section prefix to include hotness catagories such as .hot
- // or .unlikely for functions.
+
+ if (const Function *F = dyn_cast<Function>(GV)) {
+ const auto &OptionalPrefix = F->getSectionPrefix();
+ if (OptionalPrefix)
+ Name += *OptionalPrefix;
+ }
if (EmitUniqueSection && UniqueSectionNames) {
Name.push_back('.');
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index f9f33fef452..e1b4d4af457 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -1274,3 +1274,20 @@ Optional<uint64_t> Function::getEntryCount() const {
}
return None;
}
+
+void Function::setSectionPrefix(StringRef Prefix) {
+ MDBuilder MDB(getContext());
+ setMetadata(LLVMContext::MD_section_prefix,
+ MDB.createFunctionSectionPrefix(Prefix));
+}
+
+Optional<StringRef> Function::getSectionPrefix() const {
+ if (MDNode *MD = getMetadata(LLVMContext::MD_section_prefix)) {
+ assert(dyn_cast<MDString>(MD->getOperand(0))
+ ->getString()
+ .equals("function_section_prefix") &&
+ "Metadata not match");
+ return dyn_cast<MDString>(MD->getOperand(1))->getString();
+ }
+ return None;
+}
diff --git a/llvm/lib/IR/LLVMContext.cpp b/llvm/lib/IR/LLVMContext.cpp
index c97228fd51a..9dac6151499 100644
--- a/llvm/lib/IR/LLVMContext.cpp
+++ b/llvm/lib/IR/LLVMContext.cpp
@@ -138,6 +138,11 @@ LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) {
assert(TypeID == MD_type && "type kind id drifted");
(void)TypeID;
+ unsigned SectionPrefixID = getMDKindID("section_prefix");
+ assert(SectionPrefixID == MD_section_prefix &&
+ "section_prefix kind id drifted");
+ (void)SectionPrefixID;
+
auto *DeoptEntry = pImpl->getOrInsertBundleTag("deopt");
assert(DeoptEntry->second == LLVMContext::OB_deopt &&
"deopt operand bundle id drifted!");
diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp
index a5a4cd06db0..f4bfd599215 100644
--- a/llvm/lib/IR/MDBuilder.cpp
+++ b/llvm/lib/IR/MDBuilder.cpp
@@ -63,6 +63,12 @@ MDNode *MDBuilder::createFunctionEntryCount(uint64_t Count) {
createConstant(ConstantInt::get(Int64Ty, Count))});
}
+MDNode *MDBuilder::createFunctionSectionPrefix(StringRef Prefix) {
+ return MDNode::get(Context,
+ {createString("function_section_prefix"),
+ createString(Prefix)});
+}
+
MDNode *MDBuilder::createRange(const APInt &Lo, const APInt &Hi) {
assert(Lo.getBitWidth() == Hi.getBitWidth() && "Mismatched bitwidths!");
diff --git a/llvm/test/Transforms/CodeGenPrepare/section.ll b/llvm/test/Transforms/CodeGenPrepare/section.ll
new file mode 100644
index 00000000000..a7652da8a94
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/section.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -codegenprepare -S | FileCheck --check-prefixes=CHECK-OPT %s
+; RUN: llc < %s -mtriple=x86_64-pc-linux-gnu -o - | FileCheck --check-prefixes=CHECK-LLC %s
+
+; This tests that hot/cold functions get correct section prefix assigned
+
+; CHECK-OPT: hot_func{{.*}}!section_prefix ![[HOT_ID:[0-9]+]]
+; CHECK-LLC: .section .text.hot
+; CHECK-LLC-NEXT: .globl hot_func
+define void @hot_func() !prof !15 {
+ ret void
+}
+
+; CHECK-OPT: cold_func{{.*}}!section_prefix ![[COLD_ID:[0-9]+]]
+; CHECK-LLC: .section .text.cold
+; CHECK-LLC-NEXT: .globl cold_func
+define void @cold_func() !prof !16 {
+ ret void
+}
+
+; CHECK-OPT: ![[HOT_ID]] = !{!"function_section_prefix", !".hot"}
+; CHECK-OPT: ![[COLD_ID]] = !{!"function_section_prefix", !".cold"}
+!llvm.module.flags = !{!1}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 1000}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
+!15 = !{!"function_entry_count", i64 1000}
+!16 = !{!"function_entry_count", i64 1}
OpenPOWER on IntegriCloud