PGO: instrumentation based profiling sets function attributes.

We collect a maximal function count among all functions in the pgo data file.
For functions that are hot, we set its InlineHint attribute. For functions that
are cold, we set its Cold attribute.

We currently treat functions with >= 30% of the maximal function count as hot
and functions with <= 1% of the maximal function count are treated as cold.
These two numbers are from preliminary tuning on SPEC.

This commit should not affect non-PGO builds and should boost performance on
instrumentation based PGO.

llvm-svn: 200874

5 files changed, 146 insertions, 1 deletions

diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 7d451e8f6d0..c0b4d8926a9 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -591,6 +591,16 @@ void CodeGenFunction::StartFunction(GlobalDecl GD,
     EmitMCountInstrumentation();
 
   PGO.assignRegionCounters(GD);
+  if (CGM.getPGOData()) {
+    if (const Decl *D = GD.getDecl()) {
+      // Turn on InlineHint attribute for hot functions.
+      if (CGM.getPGOData()->isHotFunction(CGM.getMangledName(GD)))
+        Fn->addFnAttr(llvm::Attribute::InlineHint);
+      // Turn on Cold attribute for cold functions.
+      else if (CGM.getPGOData()->isColdFunction(CGM.getMangledName(GD)))
+        Fn->addFnAttr(llvm::Attribute::Cold);
+    }
+  }
 
   if (RetTy->isVoidType()) {
     // Void type; nothing to return.
diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp
index a1782d32b44..d7c4709612a 100644
--- a/clang/lib/CodeGen/CodeGenPGO.cpp
+++ b/clang/lib/CodeGen/CodeGenPGO.cpp
@@ -45,6 +45,7 @@ PGOProfileData::PGOProfileData(CodeGenModule &CGM, std::string Path)
   const char *BufferStart = DataBuffer->getBufferStart();
   const char *BufferEnd = DataBuffer->getBufferEnd();
   const char *CurPtr = BufferStart;
+  uint64_t MaxCount = 0;
   while (CurPtr < BufferEnd) {
     // Read the mangled function name.
     const char *FuncName = CurPtr;
@@ -65,8 +66,19 @@ PGOProfileData::PGOProfileData(CodeGenModule &CGM, std::string Path)
     }
     CurPtr = EndPtr;
 
+    // Read function count.
+    uint64_t Count = strtoll(CurPtr, &EndPtr, 10);
+    if (EndPtr == CurPtr || *EndPtr != '\n') {
+      ReportBadPGOData(CGM, "pgo-data file has bad count value");
+      return;
+    }
+    CurPtr = EndPtr + 1;
+    FunctionCounts[MangledName] = Count;
+    MaxCount = Count > MaxCount ? Count : MaxCount;
+
     // There is one line for each counter; skip over those lines.
-    for (unsigned N = 0; N < NumCounters; ++N) {
+    // Since function count is already read, we start the loop from 1.
+    for (unsigned N = 1; N < NumCounters; ++N) {
       CurPtr = strchr(++CurPtr, '\n');
       if (!CurPtr) {
         ReportBadPGOData(CGM, "pgo data file is missing some counter info");
@@ -79,6 +91,33 @@ PGOProfileData::PGOProfileData(CodeGenModule &CGM, std::string Path)
 
     DataOffsets[MangledName] = FuncName - BufferStart;
   }
+  MaxFunctionCount = MaxCount;
+}
+
+/// Return true if a function is hot. If we know nothing about the function,
+/// return false.
+bool PGOProfileData::isHotFunction(StringRef MangledName) {
+  llvm::StringMap<uint64_t>::const_iterator CountIter =
+    FunctionCounts.find(MangledName);
+  // If we know nothing about the function, return false.
+  if (CountIter == FunctionCounts.end())
+    return false;
+  // FIXME: functions with >= 30% of the maximal function count are
+  // treated as hot. This number is from preliminary tuning on SPEC.
+  return CountIter->getValue() >= (uint64_t)(0.3 * (double)MaxFunctionCount);
+}
+
+/// Return true if a function is cold. If we know nothing about the function,
+/// return false.
+bool PGOProfileData::isColdFunction(StringRef MangledName) {
+  llvm::StringMap<uint64_t>::const_iterator CountIter =
+    FunctionCounts.find(MangledName);
+  // If we know nothing about the function, return false.
+  if (CountIter == FunctionCounts.end())
+    return false;
+  // FIXME: functions with <= 1% of the maximal function count are treated as
+  // cold. This number is from preliminary tuning on SPEC.
+  return CountIter->getValue() <= (uint64_t)(0.01 * (double)MaxFunctionCount);
 }
 
 bool PGOProfileData::getFunctionCounts(StringRef MangledName,
diff --git a/clang/lib/CodeGen/CodeGenPGO.h b/clang/lib/CodeGen/CodeGenPGO.h
index 2d1193bab0b..6409a10525b 100644
--- a/clang/lib/CodeGen/CodeGenPGO.h
+++ b/clang/lib/CodeGen/CodeGenPGO.h
@@ -33,12 +33,22 @@ private:
   llvm::OwningPtr<llvm::MemoryBuffer> DataBuffer;
   /// Offsets into DataBuffer for each function's counters
   llvm::StringMap<unsigned> DataOffsets;
+  /// Execution counts for each function.
+  llvm::StringMap<uint64_t> FunctionCounts;
+  /// The maximal execution count among all functions.
+  uint64_t MaxFunctionCount;
   CodeGenModule &CGM;
 public:
   PGOProfileData(CodeGenModule &CGM, std::string Path);
   /// Fill Counts with the profile data for the given function name. Returns
   /// false on success.
   bool getFunctionCounts(StringRef MangledName, std::vector<uint64_t> &Counts);
+  /// Return true if a function is hot. If we know nothing about the function,
+  /// return false.
+  bool isHotFunction(StringRef MangledName);
+  /// Return true if a function is cold. If we know nothing about the function,
+  /// return false.
+  bool isColdFunction(StringRef MangledName);
 };
 
 /// Per-function PGO state. This class should generally not be used directly,
diff --git a/clang/test/CodeGen/Inputs/instr-attribute.pgodata b/clang/test/CodeGen/Inputs/instr-attribute.pgodata
new file mode 100644
index 00000000000..0419b3b3bef
--- /dev/null
+++ b/clang/test/CodeGen/Inputs/instr-attribute.pgodata
@@ -0,0 +1,39 @@
+hot_100_percent 4
+100000
+4999950000
+0
+0
+
+hot_40_percent 4
+40000
+799980000
+0
+0
+
+normal_func 4
+20000
+199990000
+0
+0
+
+cold_func 4
+500
+124750
+0
+0
+
+main 13
+1
+100000
+0
+0
+40000
+0
+0
+20000
+0
+0
+500
+0
+0
+
diff --git a/clang/test/CodeGen/instr-attribute.c b/clang/test/CodeGen/instr-attribute.c
new file mode 100644
index 00000000000..917a0276434
--- /dev/null
+++ b/clang/test/CodeGen/instr-attribute.c
@@ -0,0 +1,47 @@
+// Test that instrumentation based profiling sets function attributes correctly.
+
+// RUN: %clang %s -o - -mllvm -disable-llvm-optzns -emit-llvm -S -fprofile-instr-use=%S/Inputs/instr-attribute.pgodata | FileCheck %s
+
+extern int atoi(const char *);
+
+// CHECK: hot_100_percent(i32 %i) [[HOT:#[0-9]+]]
+void hot_100_percent(int i) {
+  while (i > 0)
+    i--;
+}
+
+// CHECK: hot_40_percent(i32 %i) [[HOT]]
+void hot_40_percent(int i) {
+  while (i > 0)
+    i--;
+}
+
+// CHECK: normal_func(i32 %i) [[NORMAL:#[0-9]+]]
+void normal_func(int i) {
+  while (i > 0)
+    i--;
+}
+
+// CHECK: cold_func(i32 %i) [[COLD:#[0-9]+]]
+void cold_func(int i) {
+  while (i > 0)
+    i--;
+}
+
+// CHECK: attributes [[HOT]] = { inlinehint nounwind {{.*}} }
+// CHECK: attributes [[NORMAL]] = { nounwind {{.*}} }
+// CHECK: attributes [[COLD]] = { cold nounwind {{.*}} }
+
+int main(int argc, const char *argv[]) {
+  int max = atoi(argv[1]);
+  int i;
+  for (i = 0; i < max; i++)
+    hot_100_percent(i);
+  for (i = 0; i < max * 4 / 10; i++)
+    hot_40_percent(i);
+  for (i = 0; i < max * 2 / 10; i++)
+    normal_func(i);
+  for (i = 0; i < max / 200; i++)
+    cold_func(i);
+  return 0;
+}