Clang changes for value profiling

Differential Revision: http://reviews.llvm.org/D8940

llvm-svn: 258650

4 files changed, 109 insertions, 6 deletions

diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 715e8e7aadc..b589ae3f88f 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -3539,6 +3539,11 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   CS.setAttributes(Attrs);
   CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
 
+  // Insert instrumentation or attach profile metadata at indirect call sites
+  if (!CS.getCalledFunction())
+    PGO.valueProfile(Builder, llvm::IPVK_IndirectCallTarget,
+                     CS.getInstruction(), Callee);
+
   // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
   // optimizer it can aggressively ignore unwind edges.
   if (CGM.getLangOpts().ObjCAutoRefCount)
diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp
index 2c0d93b394a..df043537dd4 100644
--- a/clang/lib/CodeGen/CodeGenPGO.cpp
+++ b/clang/lib/CodeGen/CodeGenPGO.cpp
@@ -18,11 +18,14 @@
 #include "clang/AST/StmtVisitor.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/MDBuilder.h"
-#include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MD5.h"
 
+static llvm::cl::opt<bool> EnableValueProfiling(
+  "enable-value-profiling", llvm::cl::ZeroOrMore,
+  llvm::cl::desc("Enable value profiling"), llvm::cl::init(false));
+
 using namespace clang;
 using namespace CodeGen;
 
@@ -740,12 +743,83 @@ void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S) {
                       Builder.getInt32(Counter)});
 }
 
+// This method either inserts a call to the profile run-time during
+// instrumentation or puts profile data into metadata for PGO use.
+void CodeGenPGO::valueProfile(CGBuilderTy &Builder, uint32_t ValueKind,
+    llvm::Instruction *ValueSite, llvm::Value *ValuePtr) {
+
+  if (!EnableValueProfiling)
+    return;
+
+  if (!ValuePtr || !ValueSite || !Builder.GetInsertBlock())
+    return;
+
+  bool InstrumentValueSites = CGM.getCodeGenOpts().ProfileInstrGenerate;
+  if (InstrumentValueSites && RegionCounterMap) {
+    llvm::LLVMContext &Ctx = CGM.getLLVMContext();
+    auto *I8PtrTy = llvm::Type::getInt8PtrTy(Ctx);
+    llvm::Value *Args[5] = {
+        llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
+        Builder.getInt64(FunctionHash),
+        Builder.CreatePtrToInt(ValuePtr, Builder.getInt64Ty()),
+        Builder.getInt32(ValueKind),
+        Builder.getInt32(NumValueSites[ValueKind]++)
+    };
+    Builder.CreateCall(
+        CGM.getIntrinsic(llvm::Intrinsic::instrprof_value_profile), Args);
+    return;
+  }
+
+  llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader();
+  if (PGOReader && haveRegionCounts()) {
+    // We record the top most called three functions at each call site.
+    // Profile metadata contains "VP" string identifying this metadata
+    // as value profiling data, then a uint32_t value for the value profiling
+    // kind, a uint64_t value for the total number of times the call is
+    // executed, followed by the function hash and execution count (uint64_t)
+    // pairs for each function.
+    if (NumValueSites[ValueKind] >= ProfRecord->getNumValueSites(ValueKind))
+      return;
+    uint32_t NV = ProfRecord->getNumValueDataForSite(ValueKind,
+                                                     NumValueSites[ValueKind]);
+    std::unique_ptr<InstrProfValueData[]> VD =
+        ProfRecord->getValueForSite(ValueKind, NumValueSites[ValueKind]);
+
+    uint64_t Sum = 0;
+    for (uint32_t I = 0; I < NV; ++I)
+      Sum += VD[I].Count;
+
+    llvm::LLVMContext &Ctx = CGM.getLLVMContext();
+    llvm::MDBuilder MDHelper(Ctx);
+    SmallVector<llvm::Metadata*, 3> Vals;
+    Vals.push_back(MDHelper.createString("VP"));
+    Vals.push_back(MDHelper.createConstant(
+        llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), ValueKind)));
+    Vals.push_back(MDHelper.createConstant(
+        llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), Sum)));
+
+    uint32_t MDCount = 3;
+    for (uint32_t I = 0; I < NV; ++I) {
+      Vals.push_back(MDHelper.createConstant(
+          llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), VD[I].Value)));
+      Vals.push_back(MDHelper.createConstant(
+          llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), VD[I].Count)));
+      if (--MDCount == 0)
+        break;
+    }
+    ValueSite->setMetadata(
+        llvm::LLVMContext::MD_prof, llvm::MDNode::get(Ctx, Vals));
+    NumValueSites[ValueKind]++;
+  }
+}
+
 void CodeGenPGO::loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader,
                                   bool IsInMainFile) {
   CGM.getPGOStats().addVisited(IsInMainFile);
   RegionCounts.clear();
-  if (std::error_code EC =
-          PGOReader->getFunctionCounts(FuncName, FunctionHash, RegionCounts)) {
+  llvm::ErrorOr<llvm::InstrProfRecord> RecordErrorOr =
+      PGOReader->getInstrProfRecord(FuncName, FunctionHash);
+  if (std::error_code EC = RecordErrorOr.getError()) {
     if (EC == llvm::instrprof_error::unknown_function)
       CGM.getPGOStats().addMissing(IsInMainFile);
     else if (EC == llvm::instrprof_error::hash_mismatch)
@@ -753,8 +827,11 @@ void CodeGenPGO::loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader,
     else if (EC == llvm::instrprof_error::malformed)
       // TODO: Consider a more specific warning for this case.
       CGM.getPGOStats().addMismatched(IsInMainFile);
-    RegionCounts.clear();
+    return;
   }
+  ProfRecord =
+      llvm::make_unique<llvm::InstrProfRecord>(std::move(RecordErrorOr.get()));
+  RegionCounts = ProfRecord->Counts;
 }
 
 /// \brief Calculate what to divide by to scale weights.
diff --git a/clang/lib/CodeGen/CodeGenPGO.h b/clang/lib/CodeGen/CodeGenPGO.h
index 6bf29ecaa7c..a181cb958c7 100644
--- a/clang/lib/CodeGen/CodeGenPGO.h
+++ b/clang/lib/CodeGen/CodeGenPGO.h
@@ -19,6 +19,7 @@
 #include "CodeGenTypes.h"
 #include "clang/Frontend/CodeGenOptions.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <memory>
 
@@ -32,10 +33,12 @@ private:
   std::string FuncName;
   llvm::GlobalVariable *FuncNameVar;
 
+  unsigned NumValueSites[llvm::IPVK_Last + 1];
   unsigned NumRegionCounters;
   uint64_t FunctionHash;
   std::unique_ptr<llvm::DenseMap<const Stmt *, unsigned>> RegionCounterMap;
   std::unique_ptr<llvm::DenseMap<const Stmt *, uint64_t>> StmtCountMap;
+  std::unique_ptr<llvm::InstrProfRecord> ProfRecord;
   std::vector<uint64_t> RegionCounts;
   uint64_t CurrentRegionCount;
   /// \brief A flag that is set to true when this function doesn't need
@@ -44,8 +47,8 @@ private:
 
 public:
   CodeGenPGO(CodeGenModule &CGM)
-      : CGM(CGM), NumRegionCounters(0), FunctionHash(0), CurrentRegionCount(0),
-        SkipCoverageMapping(false) {}
+      : CGM(CGM), NumValueSites{0}, NumRegionCounters(0),
+        FunctionHash(0), CurrentRegionCount(0), SkipCoverageMapping(false) {}
 
   /// Whether or not we have PGO region data for the current function. This is
   /// false both when we have no data at all and when our data has been
@@ -87,6 +90,9 @@ public:
   /// for an unused declaration.
   void emitEmptyCounterMapping(const Decl *D, StringRef FuncName,
                                llvm::GlobalValue::LinkageTypes Linkage);
+  // Insert instrumentation or attach profile metadata at value sites
+  void valueProfile(CGBuilderTy &Builder, uint32_t ValueKind,
+                    llvm::Instruction *ValueSite, llvm::Value *ValuePtr);
 private:
   void setFuncName(llvm::Function *Fn);
   void setFuncName(StringRef Name, llvm::GlobalValue::LinkageTypes Linkage);
diff --git a/clang/test/Profile/c-indirect-call.c b/clang/test/Profile/c-indirect-call.c
new file mode 100644
index 00000000000..0d8f3babdbd
--- /dev/null
+++ b/clang/test/Profile/c-indirect-call.c
@@ -0,0 +1,15 @@
+// Check the data structures emitted by instrumentation.
+// RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name c-indirect-call.c %s -o - -emit-llvm -fprofile-instr-generate -mllvm -enable-value-profiling | FileCheck %s
+
+void (*foo)(void);
+
+int main(void) {
+// CHECK:  [[REG1:%[0-9]+]] = load void ()*, void ()** @foo, align 8
+// CHECK-NEXT:  call void [[REG1]]()
+// CHECK-NEXT:  [[REG2:%[0-9]+]] = ptrtoint void ()* [[REG1]] to i64
+// CHECK-NEXT:  call void @__llvm_profile_instrument_target(i64 [[REG2]], i8* bitcast ({ i32, i32, i64, i8*, i64*, i8*, i8*, [1 x i16] }* @__profd_main to i8*), i32 0)
+  foo();
+  return 0;
+}
+
+// CHECK: declare void @__llvm_profile_instrument_target(i64, i8*, i32)