Separate the ICP total threshold and remaining threshold.

Summary: In the current implementation, isPromotionProfitable only checks if the call count to a direct target is no less than a certain percentage threshold of the remaining call counts that have not been promoted. This causes code size problems when the target count is small but greater than a large portion of remaining counts. E.g. target1 takes 99.9%, while target2 takes 0.1%. Both targets will be promoted and inlined, makes the function size too large, which potentially prevents it from further inlining into its callers. This patch adds another percentage threshold against the total indirect call count. If the target count needs to be no less than both thresholds in order to be promoted speculatively.

Reviewers: davidxl, tejohnson

Reviewed By: tejohnson

Subscribers: sanjoy, llvm-commits

Differential Revision: https://reviews.llvm.org/D35962

llvm-svn: 309345

1 files changed, 20 insertions, 12 deletions

diff --git a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp
index ed233d20153..7c16b301b47 100644
--- a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp
+++ b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp
@@ -41,11 +41,19 @@ static cl::opt<unsigned>
                                "for the promotion"));
 
 // The percent threshold for the direct-call target (this call site vs the
+// remaining call count) for it to be considered as the promotion target.
+static cl::opt<unsigned> ICPRemainingPercentThreshold(
+    "icp-remaining-percent-threshold", cl::init(30), cl::Hidden, cl::ZeroOrMore,
+    cl::desc("The percentage threshold against remaining unpromoted indirect "
+             "call count for the promotion"));
+
+// The percent threshold for the direct-call target (this call site vs the
 // total call count) for it to be considered as the promotion target.
 static cl::opt<unsigned>
-    ICPPercentThreshold("icp-percent-threshold", cl::init(30), cl::Hidden,
-                        cl::ZeroOrMore,
-                        cl::desc("The percentage threshold for the promotion"));
+    ICPTotalPercentThreshold("icp-total-percent-threshold", cl::init(5),
+                             cl::Hidden, cl::ZeroOrMore,
+                             cl::desc("The percentage threshold against total "
+                                      "count for the promotion"));
 
 // Set the maximum number of targets to promote for a single indirect-call
 // callsite.
@@ -59,12 +67,11 @@ ICallPromotionAnalysis::ICallPromotionAnalysis() {
 }
 
 bool ICallPromotionAnalysis::isPromotionProfitable(uint64_t Count,
-                                                   uint64_t TotalCount) {
-  if (Count < ICPCountThreshold)
-    return false;
-
-  unsigned Percentage = (Count * 100) / TotalCount;
-  return (Percentage >= ICPPercentThreshold);
+                                                   uint64_t TotalCount,
+                                                   uint64_t RemainingCount) {
+  return Count >= ICPCountThreshold &&
+         Count * 100 >= ICPRemainingPercentThreshold * RemainingCount &&
+         Count * 100 >= ICPTotalPercentThreshold * TotalCount;
 }
 
 // Indirect-call promotion heuristic. The direct targets are sorted based on
@@ -78,17 +85,18 @@ uint32_t ICallPromotionAnalysis::getProfitablePromotionCandidates(
                << "\n");
 
   uint32_t I = 0;
+  uint64_t RemainingCount = TotalCount;
   for (; I < MaxNumPromotions && I < NumVals; I++) {
     uint64_t Count = ValueDataRef[I].Count;
-    assert(Count <= TotalCount);
+    assert(Count <= RemainingCount);
     DEBUG(dbgs() << " Candidate " << I << " Count=" << Count
                  << "  Target_func: " << ValueDataRef[I].Value << "\n");
 
-    if (!isPromotionProfitable(Count, TotalCount)) {
+    if (!isPromotionProfitable(Count, TotalCount, RemainingCount)) {
       DEBUG(dbgs() << " Not promote: Cold target.\n");
       return I;
     }
-    TotalCount -= Count;
+    RemainingCount -= Count;
   }
   return I;
 }