[PGO] Value profiling support

This change introduces an instrumentation intrinsic instruction for
value profiling purposes, the lowering of the instrumentation intrinsic
and raw reader updates. The raw profile data files for llvm-profdata
testing are updated.

llvm-svn: 253484

3 files changed, 200 insertions, 29 deletions

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 50a4f247efb..de0c0fba5f7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5211,7 +5211,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
   }
   case Intrinsic::instrprof_increment:
     llvm_unreachable("instrprof failed to lower an increment");
-
+  case Intrinsic::instrprof_value_profile:
+    llvm_unreachable("instrprof failed to lower a value profiling call");
   case Intrinsic::localescape: {
     MachineFunction &MF = DAG.getMachineFunction();
     const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 479b6bc61d9..8a0354dc98c 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -206,15 +206,21 @@ std::error_code RawInstrProfReader<IntPtrT>::readHeader(
 
   CountersDelta = swap(Header.CountersDelta);
   NamesDelta = swap(Header.NamesDelta);
+  ValueDataDelta = swap(Header.ValueDataDelta);
   auto DataSize = swap(Header.DataSize);
   auto CountersSize = swap(Header.CountersSize);
   auto NamesSize = swap(Header.NamesSize);
+  auto ValueDataSize = swap(Header.ValueDataSize);
+  ValueKindLast = swap(Header.ValueKindLast);
+
+  auto DataSizeInBytes = DataSize * sizeof(RawInstrProf::ProfileData<IntPtrT>);
+  auto PaddingSize = getNumPaddingBytes(NamesSize);
 
   ptrdiff_t DataOffset = sizeof(RawInstrProf::Header);
-  ptrdiff_t CountersOffset =
-      DataOffset + sizeof(RawInstrProf::ProfileData<IntPtrT>) * DataSize;
+  ptrdiff_t CountersOffset = DataOffset + DataSizeInBytes;
   ptrdiff_t NamesOffset = CountersOffset + sizeof(uint64_t) * CountersSize;
-  size_t ProfileSize = NamesOffset + sizeof(char) * NamesSize;
+  ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize;
+  size_t ProfileSize = ValueDataOffset + ValueDataSize;
 
   auto *Start = reinterpret_cast<const char *>(&Header);
   if (Start + ProfileSize > DataBuffer->getBufferEnd())
@@ -225,8 +231,23 @@ std::error_code RawInstrProfReader<IntPtrT>::readHeader(
   DataEnd = Data + DataSize;
   CountersStart = reinterpret_cast<const uint64_t *>(Start + CountersOffset);
   NamesStart = Start + NamesOffset;
+  ValueDataStart = reinterpret_cast<const uint8_t*>(Start + ValueDataOffset);
   ProfileEnd = Start + ProfileSize;
 
+  FunctionPtrToNameMap.clear();
+  for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) {
+    const IntPtrT FPtr = swap(I->FunctionPointer);
+    if (!FPtr)
+      continue;
+    StringRef FunctionName(getName(I->NamePtr), swap(I->NameSize));
+    const char* NameEntryPtr = StringTable.insertString(FunctionName);
+    FunctionPtrToNameMap.push_back(std::pair<const IntPtrT, const char*>
+                                   (FPtr, NameEntryPtr));
+  }
+  std::sort(FunctionPtrToNameMap.begin(), FunctionPtrToNameMap.end(), less_first());
+  FunctionPtrToNameMap.erase(std::unique(FunctionPtrToNameMap.begin(),
+                                         FunctionPtrToNameMap.end()),
+                                         FunctionPtrToNameMap.end());
   return success();
 }
 
@@ -234,9 +255,8 @@ template <class IntPtrT>
 std::error_code RawInstrProfReader<IntPtrT>::readName(InstrProfRecord &Record) {
   Record.Name = StringRef(getName(Data->NamePtr), swap(Data->NameSize));
   if (Record.Name.data() < NamesStart ||
-      Record.Name.data() + Record.Name.size() > DataBuffer->getBufferEnd())
+      Record.Name.data() + Record.Name.size() > (char*)ValueDataStart)
     return error(instrprof_error::malformed);
-
   return success();
 }
 
@@ -275,8 +295,54 @@ std::error_code RawInstrProfReader<IntPtrT>::readRawCounts(
 }
 
 template <class IntPtrT>
-std::error_code
-RawInstrProfReader<IntPtrT>::readNextRecord(InstrProfRecord &Record) {
+std::error_code RawInstrProfReader<IntPtrT>::readValueData(
+    InstrProfRecord &Record) {
+
+  Record.clearValueData();
+  if (!Data->Values || (ValueDataDelta == 0))
+    return success();
+
+  // Read value data.
+  uint64_t NumVSites = 0;
+  for (uint32_t Kind = IPVK_First; Kind <= ValueKindLast; ++Kind)
+    NumVSites += swap(Data->NumValueSites[Kind]);
+  NumVSites += getNumPaddingBytes(NumVSites);
+
+  auto VDataCounts = makeArrayRef(getValueDataCounts(Data->Values), NumVSites);
+  // Check bounds.
+  if (VDataCounts.data() < ValueDataStart ||
+      VDataCounts.data() + VDataCounts.size() > (const uint8_t *)ProfileEnd)
+    return error(instrprof_error::malformed);
+
+  const InstrProfValueData *VDataPtr =
+      getValueData(swap(Data->Values) + NumVSites);
+  for (uint32_t Kind = IPVK_First; Kind <= ValueKindLast; ++Kind) {
+    NumVSites = swap(Data->NumValueSites[Kind]);
+    Record.reserveSites(Kind, NumVSites);
+    for (uint32_t VSite = 0; VSite < NumVSites; ++VSite) {
+
+      uint32_t VDataCount = VDataCounts[VSite];
+      if ((const char *)(VDataPtr + VDataCount) > ProfileEnd)
+        return error(instrprof_error::malformed);
+
+      std::vector<InstrProfValueData> CurrentValues;
+      CurrentValues.reserve(VDataCount);
+      for (uint32_t VIndex = 0; VIndex < VDataCount; ++VIndex) {
+        uint64_t TargetValue = swap(VDataPtr->Value);
+        uint64_t Count = swap(VDataPtr->Count);
+        CurrentValues.push_back({TargetValue, Count});
+        ++VDataPtr;
+      }
+      Record.addValueData(Kind, VSite, CurrentValues.data(),
+                          VDataCount, &FunctionPtrToNameMap);
+    }
+  }
+  return success();
+}
+
+template <class IntPtrT>
+std::error_code RawInstrProfReader<IntPtrT>::readNextRecord(
+    InstrProfRecord &Record) {
   if (atEnd())
     if (std::error_code EC = readNextHeader(ProfileEnd))
       return EC;
@@ -293,6 +359,9 @@ RawInstrProfReader<IntPtrT>::readNextRecord(InstrProfRecord &Record) {
   if (std::error_code EC = readRawCounts(Record))
     return EC;
 
+  // Read value data and set Record.
+  if (std::error_code EC = readValueData(Record)) return EC;
+
   // Iterate.
   advanceData();
   return success();
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 5f0ae54df23..334c41f3abc 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -7,19 +7,18 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This pass lowers instrprof_increment intrinsics emitted by a frontend for
-// profiling. It also builds the data structures and initialization code needed
-// for updating execution counts and emitting the profile at runtime.
+// This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
+// It also builds the data structures and initialization code needed for
+// updating execution counts and emitting the profile at runtime.
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/ProfileData/InstrProf.h"
-#include "llvm/Transforms/Instrumentation.h"
-
 #include "llvm/ADT/Triple.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 
 using namespace llvm;
@@ -50,7 +49,15 @@ public:
 private:
   InstrProfOptions Options;
   Module *M;
-  DenseMap<GlobalVariable *, GlobalVariable *> RegionCounters;
+  typedef struct PerFunctionProfileData {
+    uint32_t NumValueSites[IPVK_Last+1];
+    GlobalVariable* RegionCounters;
+    GlobalVariable* DataVar;
+    PerFunctionProfileData() : RegionCounters(nullptr), DataVar(nullptr) {
+      memset(NumValueSites, 0, sizeof(uint32_t) * (IPVK_Last+1));
+    }
+  } PerFunctionProfileData;
+  DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
   std::vector<Value *> UsedVars;
 
   bool isMachO() const {
@@ -77,6 +84,12 @@ private:
     return getInstrProfCoverageSectionName(isMachO());
   }
 
+  /// Count the number of instrumented value sites for the function.
+  void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
+
+  /// Replace instrprof_value_profile with a call to runtime library.
+  void lowerValueProfileInst(InstrProfValueProfileInst *Ins);
+
   /// Replace instrprof_increment with an increment of the appropriate value.
   void lowerIncrement(InstrProfIncrementInst *Inc);
 
@@ -118,21 +131,37 @@ bool InstrProfiling::runOnModule(Module &M) {
   bool MadeChange = false;
 
   this->M = &M;
-  RegionCounters.clear();
+  ProfileDataMap.clear();
   UsedVars.clear();
 
+  // We did not know how many value sites there would be inside
+  // the instrumented function. This is counting the number of instrumented
+  // target value sites to enter it as field in the profile data variable.
   for (Function &F : M)
     for (BasicBlock &BB : F)
       for (auto I = BB.begin(), E = BB.end(); I != E;)
-        if (auto *Inc = dyn_cast<InstrProfIncrementInst>(I++)) {
+        if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I++))
+          computeNumValueSiteCounts(Ind);
+
+  for (Function &F : M)
+    for (BasicBlock &BB : F)
+      for (auto I = BB.begin(), E = BB.end(); I != E;) {
+        auto Instr = I++;
+        if (auto *Inc = dyn_cast<InstrProfIncrementInst>(Instr)) {
           lowerIncrement(Inc);
           MadeChange = true;
+        } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
+          lowerValueProfileInst(Ind);
+          MadeChange = true;
         }
+      }
+
   if (GlobalVariable *Coverage =
           M.getNamedGlobal(getCoverageMappingVarName())) {
     lowerCoverageData(Coverage);
     MadeChange = true;
   }
+
   if (!MadeChange)
     return false;
 
@@ -143,6 +172,54 @@ bool InstrProfiling::runOnModule(Module &M) {
   return true;
 }
 
+static Constant *getOrInsertValueProfilingCall(Module &M) {
+  auto *VoidTy = Type::getVoidTy(M.getContext());
+  auto *VoidPtrTy = Type::getInt8PtrTy(M.getContext());
+  auto *Int32Ty = Type::getInt32Ty(M.getContext());
+  auto *Int64Ty = Type::getInt64Ty(M.getContext());
+  Type *ArgTypes[] = {Int64Ty, VoidPtrTy, Int32Ty};
+  auto *ValueProfilingCallTy =
+      FunctionType::get(VoidTy, makeArrayRef(ArgTypes), false);
+  return M.getOrInsertFunction("__llvm_profile_instrument_target",
+                               ValueProfilingCallTy);
+}
+
+void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
+
+  GlobalVariable *Name = Ind->getName();
+  uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
+  uint64_t Index = Ind->getIndex()->getZExtValue();
+  auto It = ProfileDataMap.find(Name);
+  if (It == ProfileDataMap.end()) {
+    PerFunctionProfileData PD;
+    PD.NumValueSites[ValueKind] = Index + 1;
+    ProfileDataMap[Name] = PD;
+  } else if (It->second.NumValueSites[ValueKind] <= Index)
+    It->second.NumValueSites[ValueKind] = Index + 1;
+}
+
+void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
+
+  GlobalVariable *Name = Ind->getName();
+  auto It = ProfileDataMap.find(Name);
+  assert(It != ProfileDataMap.end() && It->second.DataVar &&
+    "value profiling detected in function with no counter incerement");
+
+  GlobalVariable *DataVar = It->second.DataVar;
+  uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
+  uint64_t Index = Ind->getIndex()->getZExtValue();
+  for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
+    Index += It->second.NumValueSites[Kind];
+
+  IRBuilder<> Builder(Ind);
+  Value* Args[3] = {Ind->getTargetValue(),
+      Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
+      Builder.getInt32(Index)};
+  Ind->replaceAllUsesWith(
+      Builder.CreateCall(getOrInsertValueProfilingCall(*M), Args));
+  Ind->eraseFromParent();
+}
+
 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
   GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
 
@@ -174,9 +251,10 @@ void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageData) {
     GlobalVariable *Name = cast<GlobalVariable>(V);
 
     // If we have region counters for this name, we've already handled it.
-    auto It = RegionCounters.find(Name);
-    if (It != RegionCounters.end())
-      continue;
+    auto It = ProfileDataMap.find(Name);
+    if (It != ProfileDataMap.end())
+      if (It->second.RegionCounters)
+        continue;
 
     // Move the name variable to the right section.
     Name->setSection(getNameSection());
@@ -191,12 +269,25 @@ static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
   return (Prefix + Name).str();
 }
 
+static inline bool shouldRecordFunctionAddr(Function *F) {
+  // Check the linkage
+  if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
+      !F->hasAvailableExternallyLinkage())
+    return true;
+  // Check uses of this function for other than direct calls or invokes to it.
+  return F->hasAddressTaken();
+}
+
 GlobalVariable *
 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
   GlobalVariable *NamePtr = Inc->getName();
-  auto It = RegionCounters.find(NamePtr);
-  if (It != RegionCounters.end())
-    return It->second;
+  auto It = ProfileDataMap.find(NamePtr);
+  PerFunctionProfileData PD;
+  if (It != ProfileDataMap.end()) {
+    if (It->second.RegionCounters)
+      return It->second.RegionCounters;
+    PD = It->second;
+  }
 
   // Move the name variable to the right section. Place them in a COMDAT group
   // if the associated function is a COMDAT. This will make sure that
@@ -225,22 +316,29 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
   CounterPtr->setAlignment(8);
   CounterPtr->setComdat(ProfileVarsComdat);
 
-  RegionCounters[Inc->getName()] = CounterPtr;
-
   // Create data variable.
-
+  auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
+  auto *Int16Ty = Type::getInt16Ty(Ctx);
+  auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last+1);
   Type *DataTypes[] = {
     #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
     #include "llvm/ProfileData/InstrProfData.inc"
   };
   auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
 
+  Constant *FunctionAddr = shouldRecordFunctionAddr(Fn) ?
+                           ConstantExpr::getBitCast(Fn, Int8PtrTy) :
+                           ConstantPointerNull::get(Int8PtrTy);
+
+  Constant *Int16ArrayVals[IPVK_Last+1];
+  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
+    Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
+
   Constant *DataVals[] = {
     #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
     #include "llvm/ProfileData/InstrProfData.inc"
   };
-
-  auto *Data = new GlobalVariable(*M, DataTy, true, NamePtr->getLinkage(),
+  auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(),
                                   ConstantStruct::get(DataTy, DataVals),
                                   getVarName(Inc, getInstrProfDataVarPrefix()));
   Data->setVisibility(NamePtr->getVisibility());
@@ -248,6 +346,10 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
   Data->setAlignment(8);
   Data->setComdat(ProfileVarsComdat);
 
+  PD.RegionCounters = CounterPtr;
+  PD.DataVar = Data;
+  ProfileDataMap[NamePtr] = PD;
+
   // Mark the data variable as used so that it isn't stripped out.
   UsedVars.push_back(Data);
 
@@ -341,7 +443,6 @@ void InstrProfiling::emitUses() {
   LLVMUsed =
       new GlobalVariable(*M, ATy, false, GlobalValue::AppendingLinkage,
                          ConstantArray::get(ATy, MergedVars), "llvm.used");
-
   LLVMUsed->setSection("llvm.metadata");
 }