summaryrefslogtreecommitdiffstats
path: root/clang/lib/CodeGen
diff options
context:
space:
mode:
Diffstat (limited to 'clang/lib/CodeGen')
-rw-r--r--clang/lib/CodeGen/CGExpr.cpp5
-rw-r--r--clang/lib/CodeGen/CGExprAgg.cpp4
-rw-r--r--clang/lib/CodeGen/CGExprComplex.cpp4
-rw-r--r--clang/lib/CodeGen/CGExprScalar.cpp8
-rw-r--r--clang/lib/CodeGen/CGObjC.cpp2
-rw-r--r--clang/lib/CodeGen/CGStmt.cpp141
-rw-r--r--clang/lib/CodeGen/CodeGenFunction.cpp34
-rw-r--r--clang/lib/CodeGen/CodeGenFunction.h12
-rw-r--r--clang/lib/CodeGen/CodeGenPGO.cpp414
-rw-r--r--clang/lib/CodeGen/CodeGenPGO.h80
10 files changed, 500 insertions, 204 deletions
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 8ed30045291..40ca5889398 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -85,6 +85,7 @@ llvm::AllocaInst *CodeGenFunction::CreateMemTemp(QualType Ty,
/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
/// expression and compare the result against zero, returning an Int1Ty value.
llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
+ PGO.setCurrentStmt(E);
if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
llvm::Value *MemPtr = EmitScalarExpr(E);
return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT);
@@ -2680,7 +2681,6 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
eval.begin(*this);
LValue lhs = EmitLValue(expr->getTrueExpr());
eval.end(*this);
- Cnt.adjustForControlFlow();
if (!lhs.isSimple())
return EmitUnsupportedLValue(expr, "conditional operator");
@@ -2690,17 +2690,14 @@ EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
// Any temporaries created here are conditional.
EmitBlock(rhsBlock);
- Cnt.beginElseRegion();
eval.begin(*this);
LValue rhs = EmitLValue(expr->getFalseExpr());
eval.end(*this);
- Cnt.adjustForControlFlow();
if (!rhs.isSimple())
return EmitUnsupportedLValue(expr, "conditional operator");
rhsBlock = Builder.GetInsertBlock();
EmitBlock(contBlock);
- Cnt.applyAdjustmentsToRegion();
llvm::PHINode *phi = Builder.CreatePHI(lhs.getAddress()->getType(), 2,
"cond-lvalue");
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index ef56c0b06f2..313b8d2afcb 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -903,7 +903,6 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
CGF.EmitBlock(LHSBlock);
Cnt.beginRegion(Builder);
Visit(E->getTrueExpr());
- Cnt.adjustForControlFlow();
eval.end(CGF);
assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!");
@@ -917,13 +916,10 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
eval.begin(CGF);
CGF.EmitBlock(RHSBlock);
- Cnt.beginElseRegion();
Visit(E->getFalseExpr());
- Cnt.adjustForControlFlow();
eval.end(CGF);
CGF.EmitBlock(ContBlock);
- Cnt.applyAdjustmentsToRegion();
}
void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) {
diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp
index 938b0b331d7..1f84c86ad6e 100644
--- a/clang/lib/CodeGen/CGExprComplex.cpp
+++ b/clang/lib/CodeGen/CGExprComplex.cpp
@@ -760,20 +760,16 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
CGF.EmitBlock(LHSBlock);
Cnt.beginRegion(Builder);
ComplexPairTy LHS = Visit(E->getTrueExpr());
- Cnt.adjustForControlFlow();
LHSBlock = Builder.GetInsertBlock();
CGF.EmitBranch(ContBlock);
eval.end(CGF);
eval.begin(CGF);
CGF.EmitBlock(RHSBlock);
- Cnt.beginElseRegion();
ComplexPairTy RHS = Visit(E->getFalseExpr());
- Cnt.adjustForControlFlow();
RHSBlock = Builder.GetInsertBlock();
CGF.EmitBlock(ContBlock);
eval.end(CGF);
- Cnt.applyAdjustmentsToRegion();
// Create a PHI node for the real part.
llvm::PHINode *RealPN = Builder.CreatePHI(LHS.first->getType(), 2, "cond.r");
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 0a49ea5a5e9..78bd037e83d 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -2934,7 +2934,6 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
CGF.EmitBlock(RHSBlock);
Cnt.beginRegion(Builder);
Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
- Cnt.adjustForControlFlow();
eval.end(CGF);
// Reaquire the RHS block, as there may be subblocks inserted.
@@ -2947,7 +2946,6 @@ Value *ScalarExprEmitter::VisitBinLAnd(const BinaryOperator *E) {
Builder.SetCurrentDebugLocation(llvm::DebugLoc());
CGF.EmitBlock(ContBlock);
PN->addIncoming(RHSCond, RHSBlock);
- Cnt.applyAdjustmentsToRegion();
// ZExt result to int.
return Builder.CreateZExtOrBitCast(PN, ResTy, "land.ext");
@@ -3017,7 +3015,6 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
CGF.EmitBlock(RHSBlock);
Cnt.beginRegion(Builder);
Value *RHSCond = CGF.EvaluateExprAsBool(E->getRHS());
- Cnt.adjustForControlFlow();
eval.end(CGF);
@@ -3028,7 +3025,6 @@ Value *ScalarExprEmitter::VisitBinLOr(const BinaryOperator *E) {
// into the phi node for the edge with the value of RHSCond.
CGF.EmitBlock(ContBlock);
PN->addIncoming(RHSCond, RHSBlock);
- Cnt.applyAdjustmentsToRegion();
// ZExt result to int.
return Builder.CreateZExtOrBitCast(PN, ResTy, "lor.ext");
@@ -3171,21 +3167,17 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
eval.begin(CGF);
Value *LHS = Visit(lhsExpr);
eval.end(CGF);
- Cnt.adjustForControlFlow();
LHSBlock = Builder.GetInsertBlock();
Builder.CreateBr(ContBlock);
CGF.EmitBlock(RHSBlock);
- Cnt.beginElseRegion();
eval.begin(CGF);
Value *RHS = Visit(rhsExpr);
eval.end(CGF);
- Cnt.adjustForControlFlow();
RHSBlock = Builder.GetInsertBlock();
CGF.EmitBlock(ContBlock);
- Cnt.applyAdjustmentsToRegion();
// If the LHS or RHS is a throw expression, it will be legitimately null.
if (!LHS)
diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp
index 9a0b15d434c..7ca10575c12 100644
--- a/clang/lib/CodeGen/CGObjC.cpp
+++ b/clang/lib/CodeGen/CGObjC.cpp
@@ -1628,7 +1628,7 @@ void CodeGenFunction::EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S){
EmitAutoVarCleanups(variable);
// Perform the loop body, setting up break and continue labels.
- BreakContinueStack.push_back(BreakContinue(LoopEnd, AfterBody, &Cnt));
+ BreakContinueStack.push_back(BreakContinue(LoopEnd, AfterBody));
{
RunCleanupsScope Scope(*this);
EmitStmt(S.getBody());
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 353d8f075c1..918103fcbd5 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -43,6 +43,7 @@ void CodeGenFunction::EmitStopPoint(const Stmt *S) {
void CodeGenFunction::EmitStmt(const Stmt *S) {
assert(S && "Null statement?");
+ PGO.setCurrentStmt(S);
// These statements have their own debug info handling.
if (EmitSimpleStmt(S))
@@ -404,14 +405,12 @@ void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {
EmitStopPoint(&S);
EmitBranchThroughCleanup(getJumpDestForLabel(S.getLabel()));
- PGO.setCurrentRegionUnreachable();
}
void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
if (const LabelDecl *Target = S.getConstantTarget()) {
EmitBranchThroughCleanup(getJumpDestForLabel(Target));
- PGO.setCurrentRegionUnreachable();
return;
}
@@ -428,7 +427,6 @@ void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
cast<llvm::PHINode>(IndGotoBB->begin())->addIncoming(V, CurBB);
EmitBranch(IndGotoBB);
- PGO.setCurrentRegionUnreachable();
}
void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
@@ -480,7 +478,6 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
RunCleanupsScope ThenScope(*this);
EmitStmt(S.getThen());
}
- Cnt.adjustForControlFlow();
EmitBranch(ContBlock);
// Emit the 'else' code if present.
@@ -489,12 +486,10 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
if (getDebugInfo())
Builder.SetCurrentDebugLocation(llvm::DebugLoc());
EmitBlock(ElseBlock);
- Cnt.beginElseRegion();
{
RunCleanupsScope ElseScope(*this);
EmitStmt(Else);
}
- Cnt.adjustForControlFlow();
// There is no need to emit line number for unconditional branch.
if (getDebugInfo())
Builder.SetCurrentDebugLocation(llvm::DebugLoc());
@@ -503,7 +498,6 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
// Emit the continuation block for code after the if.
EmitBlock(ContBlock, true);
- Cnt.applyAdjustmentsToRegion();
}
void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
@@ -519,7 +513,7 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
JumpDest LoopExit = getJumpDestInCurrentScope("while.end");
// Store the blocks to use for break and continue.
- BreakContinueStack.push_back(BreakContinue(LoopExit, LoopHeader, &Cnt));
+ BreakContinueStack.push_back(BreakContinue(LoopExit, LoopHeader));
// C++ [stmt.while]p2:
// When the condition of a while statement is a declaration, the
@@ -541,7 +535,6 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
// while(1) is common, avoid extra exit blocks. Be sure
// to correctly handle break/continue though.
bool EmitBoolCondBranch = true;
- llvm::BranchInst *CondBr = NULL;
if (llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal))
if (C->isOne())
EmitBoolCondBranch = false;
@@ -552,8 +545,8 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
if (ConditionScope.requiresCleanups())
ExitBlock = createBasicBlock("while.exit");
-
- CondBr = Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
+ Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock,
+ PGO.createLoopWeights(S.getCond(), Cnt));
if (ExitBlock != LoopExit.getBlock()) {
EmitBlock(ExitBlock);
@@ -569,16 +562,9 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
Cnt.beginRegion(Builder);
EmitStmt(S.getBody());
}
- Cnt.adjustForControlFlow();
BreakContinueStack.pop_back();
- uint64_t LoopCount = Cnt.getCount();
- uint64_t ExitCount = Cnt.getLoopExitCount();
- if (EmitBoolCondBranch)
- CondBr->setMetadata(llvm::LLVMContext::MD_prof,
- PGO.createBranchWeights(LoopCount, ExitCount));
-
// Immediately force cleanup.
ConditionScope.ForceCleanup();
@@ -587,7 +573,6 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) {
// Emit the exit block.
EmitBlock(LoopExit.getBlock(), true);
- PGO.setCurrentRegionCount(ExitCount + Cnt.getBreakCounter().getCount());
// The LoopHeader typically is just a branch if we skipped emitting
// a branch, try to erase it.
@@ -602,17 +587,15 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S) {
RegionCounter Cnt = getPGORegionCounter(&S);
// Store the blocks to use for break and continue.
- BreakContinueStack.push_back(BreakContinue(LoopExit, LoopCond, &Cnt));
+ BreakContinueStack.push_back(BreakContinue(LoopExit, LoopCond));
// Emit the body of the loop.
llvm::BasicBlock *LoopBody = createBasicBlock("do.body");
- EmitBlock(LoopBody);
- Cnt.beginRegion(Builder);
+ EmitBlockWithFallThrough(LoopBody, Cnt);
{
RunCleanupsScope BodyScope(*this);
EmitStmt(S.getBody());
}
- Cnt.adjustForControlFlow();
EmitBlock(LoopCond.getBlock());
@@ -633,17 +616,13 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S) {
if (C->isZero())
EmitBoolCondBranch = false;
- uint64_t LoopCount = Cnt.getCount() - Cnt.getParentCount();
- uint64_t ExitCount = Cnt.getLoopExitCount();
-
// As long as the condition is true, iterate the loop.
if (EmitBoolCondBranch)
Builder.CreateCondBr(BoolCondVal, LoopBody, LoopExit.getBlock(),
- PGO.createBranchWeights(LoopCount, ExitCount));
+ PGO.createLoopWeights(S.getCond(), Cnt));
// Emit the exit block.
EmitBlock(LoopExit.getBlock());
- PGO.setCurrentRegionCount(ExitCount + Cnt.getBreakCounter().getCount());
// The DoCond block typically is just a branch if we skipped
// emitting a branch, try to erase it.
@@ -652,8 +631,6 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S) {
}
void CodeGenFunction::EmitForStmt(const ForStmt &S) {
- RegionCounter Cnt = getPGORegionCounter(&S);
-
JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
RunCleanupsScope ForScope(*this);
@@ -666,6 +643,8 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
if (S.getInit())
EmitStmt(S.getInit());
+ RegionCounter Cnt = getPGORegionCounter(&S);
+
// Start the loop with a block that tests the condition.
// If there's an increment, the continue scope will be overwritten
// later.
@@ -681,12 +660,11 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
Continue = getJumpDestInCurrentScope("for.inc");
// Store the blocks to use for break and continue.
- BreakContinueStack.push_back(BreakContinue(LoopExit, Continue, &Cnt));
+ BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
// Create a cleanup scope for the condition variable cleanups.
RunCleanupsScope ConditionScope(*this);
- llvm::BranchInst *CondBr = NULL;
if (S.getCond()) {
// If the for statement has a condition scope, emit the local variable
// declaration.
@@ -706,7 +684,8 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
// C99 6.8.5p2/p4: The first substatement is executed if the expression
// compares unequal to 0. The condition must be a scalar type.
llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
- CondBr = Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock);
+ Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock,
+ PGO.createLoopWeights(S.getCond(), Cnt));
if (ExitBlock != LoopExit.getBlock()) {
EmitBlock(ExitBlock);
@@ -732,16 +711,9 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
EmitBlock(Continue.getBlock());
EmitStmt(S.getInc());
}
- Cnt.adjustForControlFlow();
BreakContinueStack.pop_back();
- uint64_t LoopCount = Cnt.getCount();
- uint64_t ExitCount = Cnt.getLoopExitCount();
- if (S.getCond())
- CondBr->setMetadata(llvm::LLVMContext::MD_prof,
- PGO.createBranchWeights(LoopCount, ExitCount));
-
ConditionScope.ForceCleanup();
EmitBranch(CondBlock);
@@ -752,12 +724,9 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) {
// Emit the fall-through block.
EmitBlock(LoopExit.getBlock(), true);
- PGO.setCurrentRegionCount(ExitCount + Cnt.getBreakCounter().getCount());
}
void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) {
- RegionCounter Cnt = getPGORegionCounter(&S);
-
JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
RunCleanupsScope ForScope(*this);
@@ -770,6 +739,8 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) {
EmitStmt(S.getRangeStmt());
EmitStmt(S.getBeginEndStmt());
+ RegionCounter Cnt = getPGORegionCounter(&S);
+
// Start the loop with a block that tests the condition.
// If there's an increment, the continue scope will be overwritten
// later.
@@ -788,8 +759,8 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) {
// The body is executed if the expression, contextually converted
// to bool, is true.
llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
- llvm::BranchInst *CondBr = Builder.CreateCondBr(BoolCondVal,
- ForBody, ExitBlock);
+ Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock,
+ PGO.createLoopWeights(S.getCond(), Cnt));
if (ExitBlock != LoopExit.getBlock()) {
EmitBlock(ExitBlock);
@@ -803,7 +774,7 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) {
JumpDest Continue = getJumpDestInCurrentScope("for.inc");
// Store the blocks to use for break and continue.
- BreakContinueStack.push_back(BreakContinue(LoopExit, Continue, &Cnt));
+ BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
{
// Create a separate cleanup scope for the loop variable and body.
@@ -815,15 +786,9 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) {
// If there is an increment, emit it next.
EmitBlock(Continue.getBlock());
EmitStmt(S.getInc());
- Cnt.adjustForControlFlow();
BreakContinueStack.pop_back();
- uint64_t LoopCount = Cnt.getCount();
- uint64_t ExitCount = Cnt.getLoopExitCount();
- CondBr->setMetadata(llvm::LLVMContext::MD_prof,
- PGO.createBranchWeights(LoopCount, ExitCount));
-
EmitBranch(CondBlock);
ForScope.ForceCleanup();
@@ -833,7 +798,6 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) {
// Emit the fall-through block.
EmitBlock(LoopExit.getBlock(), true);
- PGO.setCurrentRegionCount(ExitCount + Cnt.getBreakCounter().getCount());
}
void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
@@ -847,7 +811,6 @@ void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
/*init*/ true);
}
EmitBranchThroughCleanup(ReturnBlock);
- PGO.setCurrentRegionUnreachable();
}
/// EmitReturnStmt - Note that due to GCC extensions, this can have an operand
@@ -920,7 +883,6 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
cleanupScope.ForceCleanup();
EmitBranchThroughCleanup(ReturnBlock);
- PGO.setCurrentRegionUnreachable();
}
void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) {
@@ -943,14 +905,7 @@ void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
if (HaveInsertPoint())
EmitStopPoint(&S);
- BreakContinue &BC = BreakContinueStack.back();
- // We keep track of breaks from the loop so we can differentiate them from
- // non-local exits in PGO instrumentation. This only applies to loops, not
- // breaks from switch statements.
- if (BC.CountBreak)
- BC.LoopCnt->getBreakCounter().beginRegion(Builder);
- EmitBranchThroughCleanup(BC.BreakBlock);
- PGO.setCurrentRegionUnreachable();
+ EmitBranchThroughCleanup(BreakContinueStack.back().BreakBlock);
}
void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
@@ -962,12 +917,7 @@ void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
if (HaveInsertPoint())
EmitStopPoint(&S);
- BreakContinue &BC = BreakContinueStack.back();
- // We keep track of continues in the loop so we can differentiate them from
- // non-local exits in PGO instrumentation.
- BC.LoopCnt->getContinueCounter().beginRegion(Builder);
- EmitBranchThroughCleanup(BC.ContinueBlock);
- PGO.setCurrentRegionUnreachable();
+ EmitBranchThroughCleanup(BreakContinueStack.back().ContinueBlock);
}
/// EmitCaseStmtRange - If case statement range is not too big then
@@ -984,9 +934,8 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) {
// Emit the code for this case. We do this first to make sure it is
// properly chained from our predecessor before generating the
// switch machinery to enter this block.
- EmitBlock(createBasicBlock("sw.bb"));
- llvm::BasicBlock *CaseDest = Builder.GetInsertBlock();
- CaseCnt.beginRegion(Builder);
+ llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");
+ EmitBlockWithFallThrough(CaseDest, CaseCnt);
EmitStmt(S.getSubStmt());
// If range is empty, do nothing.
@@ -997,7 +946,7 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) {
// FIXME: parameters such as this should not be hardcoded.
if (Range.ult(llvm::APInt(Range.getBitWidth(), 64))) {
// Range is small enough to add multiple switch instruction cases.
- uint64_t Total = CaseCnt.getCount() - CaseCnt.getParentCount();
+ uint64_t Total = CaseCnt.getCount();
unsigned NCases = Range.getZExtValue() + 1;
// We only have one region counter for the entire set of cases here, so we
// need to divide the weights evenly between the generated cases, ensuring
@@ -1036,7 +985,7 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) {
llvm::MDNode *Weights = 0;
if (SwitchWeights) {
- uint64_t ThisCount = CaseCnt.getCount() - CaseCnt.getParentCount();
+ uint64_t ThisCount = CaseCnt.getCount();
uint64_t DefaultCount = (*SwitchWeights)[0];
Weights = PGO.createBranchWeights(ThisCount, DefaultCount);
@@ -1086,7 +1035,7 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) {
// Only do this optimization if there are no cleanups that need emitting.
if (isObviouslyBranchWithoutCleanups(Block)) {
if (SwitchWeights)
- SwitchWeights->push_back(CaseCnt.getCount() - CaseCnt.getParentCount());
+ SwitchWeights->push_back(CaseCnt.getCount());
SwitchInsn->addCase(CaseVal, Block.getBlock());
// If there was a fallthrough into this case, make sure to redirect it to
@@ -1099,11 +1048,10 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) {
}
}
- EmitBlock(createBasicBlock("sw.bb"));
- llvm::BasicBlock *CaseDest = Builder.GetInsertBlock();
+ llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");
+ EmitBlockWithFallThrough(CaseDest, CaseCnt);
if (SwitchWeights)
- SwitchWeights->push_back(CaseCnt.getCount() - CaseCnt.getParentCount());
- CaseCnt.beginRegion(Builder);
+ SwitchWeights->push_back(CaseCnt.getCount());
SwitchInsn->addCase(CaseVal, CaseDest);
// Recursively emitting the statement is acceptable, but is not wonderful for
@@ -1126,8 +1074,11 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) {
CaseCnt = getPGORegionCounter(NextCase);
if (SwitchWeights)
- SwitchWeights->push_back(CaseCnt.getCount() - CaseCnt.getParentCount());
- CaseCnt.beginRegion(Builder);
+ SwitchWeights->push_back(CaseCnt.getCount());
+ if (CGM.getCodeGenOpts().ProfileInstrGenerate) {
+ CaseDest = createBasicBlock("sw.bb");
+ EmitBlockWithFallThrough(CaseDest, CaseCnt);
+ }
SwitchInsn->addCase(CaseVal, CaseDest);
NextCase = dyn_cast<CaseStmt>(CurCase->getSubStmt());
@@ -1142,21 +1093,9 @@ void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S) {
assert(DefaultBlock->empty() &&
"EmitDefaultStmt: Default block already defined?");
- llvm::BasicBlock *SkipCountBB = 0;
- if (CGM.getCodeGenOpts().ProfileInstrGenerate) {
- // The PGO region here needs to count the number of times the edge occurs,
- // so fallthrough into this case will jump past the region counter to the
- // skipcount basic block.
- SkipCountBB = createBasicBlock("skipcount");
- EmitBranch(SkipCountBB);
- }
- EmitBlock(DefaultBlock);
-
RegionCounter Cnt = getPGORegionCounter(&S);
- Cnt.beginRegion(Builder, /*AddIncomingFallThrough=*/true);
+ EmitBlockWithFallThrough(DefaultBlock, Cnt);
- if (SkipCountBB)
- EmitBlock(SkipCountBB);
EmitStmt(S.getSubStmt());
}
@@ -1439,20 +1378,14 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
// Clear the insertion point to indicate we are in unreachable code.
Builder.ClearInsertionPoint();
- PGO.setCurrentRegionUnreachable();
// All break statements jump to NextBlock. If BreakContinueStack is non-empty
- // then reuse last ContinueBlock and that block's counter.
+ // then reuse last ContinueBlock.
JumpDest OuterContinue;
- RegionCounter *OuterCount = 0;
- if (!BreakContinueStack.empty()) {
- BreakContinue &BC = BreakContinueStack.back();
- OuterContinue = BC.ContinueBlock;
- OuterCount = BC.LoopCnt;
- }
+ if (!BreakContinueStack.empty())
+ OuterContinue = BreakContinueStack.back().ContinueBlock;
- BreakContinueStack.push_back(BreakContinue(SwitchExit, OuterContinue,
- OuterCount, /*CountBreak=*/false));
+ BreakContinueStack.push_back(BreakContinue(SwitchExit, OuterContinue));
// Emit switch body.
EmitStmt(S.getBody());
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 715af365c38..409d1141b81 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -688,6 +688,26 @@ void CodeGenFunction::EmitFunctionBody(FunctionArgList &Args,
EmitStmt(Body);
}
+/// When instrumenting to collect profile data, the counts for some blocks
+/// such as switch cases need to not include the fall-through counts, so
+/// emit a branch around the instrumentation code. When not instrumenting,
+/// this just calls EmitBlock().
+void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
+ RegionCounter &Cnt) {
+ llvm::BasicBlock *SkipCountBB = 0;
+ if (HaveInsertPoint() && CGM.getCodeGenOpts().ProfileInstrGenerate) {
+ // When instrumenting for profiling, the fallthrough to certain
+ // statements needs to skip over the instrumentation code so that we
+ // get an accurate count.
+ SkipCountBB = createBasicBlock("skipcount");
+ EmitBranch(SkipCountBB);
+ }
+ EmitBlock(BB);
+ Cnt.beginRegion(Builder, /*AddIncomingFallThrough=*/true);
+ if (SkipCountBB)
+ EmitBlock(SkipCountBB);
+}
+
/// Tries to mark the given function nounwind based on the
/// non-existence of any throwing calls within it. We believe this is
/// lightweight enough to do at -O0.
@@ -917,10 +937,11 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
Cond = Cond->IgnoreParens();
if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Cond)) {
- RegionCounter Cnt = getPGORegionCounter(CondBOp);
// Handle X && Y in a condition.
if (CondBOp->getOpcode() == BO_LAnd) {
+ RegionCounter Cnt = getPGORegionCounter(CondBOp);
+
// If we have "1 && X", simplify the code. "0 && X" would have constant
// folded if the case was simple enough.
bool ConstantBool = false;
@@ -957,13 +978,13 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
eval.begin(*this);
EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, TrueCount);
eval.end(*this);
- Cnt.adjustForControlFlow();
- Cnt.applyAdjustmentsToRegion();
return;
}
if (CondBOp->getOpcode() == BO_LOr) {
+ RegionCounter Cnt = getPGORegionCounter(CondBOp);
+
// If we have "0 || X", simplify the code. "1 || X" would have constant
// folded if the case was simple enough.
bool ConstantBool = false;
@@ -1003,8 +1024,6 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, RHSCount);
eval.end(*this);
- Cnt.adjustForControlFlow();
- Cnt.applyAdjustmentsToRegion();
return;
}
@@ -1037,7 +1056,7 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
// the conditional operator.
uint64_t LHSScaledTrueCount = 0;
if (TrueCount) {
- double LHSRatio = Cnt.getCount() / (double) PGO.getCurrentRegionCount();
+ double LHSRatio = Cnt.getCount() / (double) Cnt.getParentCount();
LHSScaledTrueCount = TrueCount * LHSRatio;
}
@@ -1050,7 +1069,6 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
cond.begin(*this);
EmitBlock(RHSBlock);
- Cnt.beginElseRegion();
EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock,
TrueCount - LHSScaledTrueCount);
cond.end(*this);
@@ -1070,7 +1088,7 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
// Create branch weights based on the number of times we get here and the
// number of times the condition should be true.
- uint64_t CurrentCount = PGO.getCurrentRegionCountWithMin(TrueCount);
+ uint64_t CurrentCount = std::max(PGO.getCurrentRegionCount(), TrueCount);
llvm::MDNode *Weights = PGO.createBranchWeights(TrueCount,
CurrentCount - TrueCount);
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index a1f6721ae81..185e25ccb29 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -818,18 +818,13 @@ private:
llvm::DenseMap<const LabelDecl*, JumpDest> LabelMap;
// BreakContinueStack - This keeps track of where break and continue
- // statements should jump to and the associated base counter for
- // instrumentation.
+ // statements should jump to.
struct BreakContinue {
- BreakContinue(JumpDest Break, JumpDest Continue, RegionCounter *LoopCnt,
- bool CountBreak = true)
- : BreakBlock(Break), ContinueBlock(Continue), LoopCnt(LoopCnt),
- CountBreak(CountBreak) {}
+ BreakContinue(JumpDest Break, JumpDest Continue)
+ : BreakBlock(Break), ContinueBlock(Continue) {}
JumpDest BreakBlock;
JumpDest ContinueBlock;
- RegionCounter *LoopCnt;
- bool CountBreak;
};
SmallVector<BreakContinue, 8> BreakContinueStack;
@@ -1156,6 +1151,7 @@ public:
void EmitDestructorBody(FunctionArgList &Args);
void emitImplicitAssignmentOperatorBody(FunctionArgList &Args);
void EmitFunctionBody(FunctionArgList &Args, const Stmt *Body);
+ void EmitBlockWithFallThrough(llvm::BasicBlock *BB, RegionCounter &Cnt);
void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator,
CallArgList &CallArgs);
diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp
index f81cbecd5d3..ec53fdb0225 100644
--- a/clang/lib/CodeGen/CodeGenPGO.cpp
+++ b/clang/lib/CodeGen/CodeGenPGO.cpp
@@ -284,56 +284,43 @@ namespace {
(*CounterMap)[S] = NextCounter++;
Visit(S->getSubStmt());
}
- /// Assign three counters - one for the body of the loop, one for breaks
- /// from the loop, and one for continues.
- ///
- /// The break and continue counters cover all such statements in this loop,
- /// and are used in calculations to find the number of times the condition
- /// and exit of the loop occur. They are needed so we can differentiate
- /// these statements from non-local exits like return and goto.
+ /// Assign a counter for the body of a while loop.
void VisitWhileStmt(const WhileStmt *S) {
- (*CounterMap)[S] = NextCounter;
- NextCounter += 3;
+ (*CounterMap)[S] = NextCounter++;
Visit(S->getCond());
Visit(S->getBody());
}
- /// Assign counters for the body of the loop, and for breaks and
- /// continues. See VisitWhileStmt.
+ /// Assign a counter for the body of a do-while loop.
void VisitDoStmt(const DoStmt *S) {
- (*CounterMap)[S] = NextCounter;
- NextCounter += 3;
+ (*CounterMap)[S] = NextCounter++;
Visit(S->getBody());
Visit(S->getCond());
}
- /// Assign counters for the body of the loop, and for breaks and
- /// continues. See VisitWhileStmt.
+ /// Assign a counter for the body of a for loop.
void VisitForStmt(const ForStmt *S) {
- (*CounterMap)[S] = NextCounter;
- NextCounter += 3;
+ (*CounterMap)[S] = NextCounter++;
+ if (S->getInit())
+ Visit(S->getInit());
const Expr *E;
if ((E = S->getCond()))
Visit(E);
- Visit(S->getBody());
if ((E = S->getInc()))
Visit(E);
+ Visit(S->getBody());
}
- /// Assign counters for the body of the loop, and for breaks and
- /// continues. See VisitWhileStmt.
+ /// Assign a counter for the body of a for-range loop.
void VisitCXXForRangeStmt(const CXXForRangeStmt *S) {
- (*CounterMap)[S] = NextCounter;
- NextCounter += 3;
- const Expr *E;
- if ((E = S->getCond()))
- Visit(E);
+ (*CounterMap)[S] = NextCounter++;
+ Visit(S->getRangeStmt());
+ Visit(S->getBeginEndStmt());
+ Visit(S->getCond());
+ Visit(S->getLoopVarStmt());
Visit(S->getBody());
- if ((E = S->getInc()))
- Visit(E);
+ Visit(S->getInc());
}
- /// Assign counters for the body of the loop, and for breaks and
- /// continues. See VisitWhileStmt.
+ /// Assign a counter for the body of a for-collection loop.
void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S) {
- (*CounterMap)[S] = NextCounter;
- NextCounter += 3;
+ (*CounterMap)[S] = NextCounter++;
Visit(S->getElement());
Visit(S->getBody());
}
@@ -402,6 +389,343 @@ namespace {
Visit(E->getRHS());
}
};
+
+ /// A StmtVisitor that propagates the raw counts through the AST and
+ /// records the count at statements where the value may change.
+ struct ComputeRegionCounts : public ConstStmtVisitor<ComputeRegionCounts> {
+ /// PGO state.
+ CodeGenPGO &PGO;
+
+ /// A flag that is set when the current count should be recorded on the
+ /// next statement, such as at the exit of a loop.
+ bool RecordNextStmtCount;
+
+ /// The map of statements to count values.
+ llvm::DenseMap<const Stmt*, uint64_t> *CountMap;
+
+ /// BreakContinueStack - Keep counts of breaks and continues inside loops.
+ struct BreakContinue {
+ uint64_t BreakCount;
+ uint64_t ContinueCount;
+ BreakContinue() : BreakCount(0), ContinueCount(0) {}
+ };
+ SmallVector<BreakContinue, 8> BreakContinueStack;
+
+ ComputeRegionCounts(llvm::DenseMap<const Stmt*, uint64_t> *CountMap,
+ CodeGenPGO &PGO) :
+ PGO(PGO), RecordNextStmtCount(false), CountMap(CountMap) {
+ }
+
+ void RecordStmtCount(const Stmt *S) {
+ if (RecordNextStmtCount) {
+ (*CountMap)[S] = PGO.getCurrentRegionCount();
+ RecordNextStmtCount = false;
+ }
+ }
+
+ void VisitStmt(const Stmt *S) {
+ RecordStmtCount(S);
+ for (Stmt::const_child_range I = S->children(); I; ++I) {
+ if (*I)
+ this->Visit(*I);
+ }
+ }
+
+ void VisitFunctionDecl(const FunctionDecl *S) {
+ RegionCounter Cnt(PGO, S->getBody());
+ Cnt.beginRegion();
+ (*CountMap)[S->getBody()] = PGO.getCurrentRegionCount();
+ Visit(S->getBody());
+ }
+
+ void VisitReturnStmt(const ReturnStmt *S) {
+ RecordStmtCount(S);
+ if (S->getRetValue())
+ Visit(S->getRetValue());
+ PGO.setCurrentRegionUnreachable();
+ RecordNextStmtCount = true;
+ }
+
+ void VisitGotoStmt(const GotoStmt *S) {
+ RecordStmtCount(S);
+ PGO.setCurrentRegionUnreachable();
+ RecordNextStmtCount = true;
+ }
+
+ void VisitLabelStmt(const LabelStmt *S) {
+ RecordNextStmtCount = false;
+ RegionCounter Cnt(PGO, S);
+ Cnt.beginRegion();
+ (*CountMap)[S] = PGO.getCurrentRegionCount();
+ Visit(S->getSubStmt());
+ }
+
+ void VisitBreakStmt(const BreakStmt *S) {
+ RecordStmtCount(S);
+ assert(!BreakContinueStack.empty() && "break not in a loop or switch!");
+ BreakContinueStack.back().BreakCount += PGO.getCurrentRegionCount();
+ PGO.setCurrentRegionUnreachable();
+ RecordNextStmtCount = true;
+ }
+
+ void VisitContinueStmt(const ContinueStmt *S) {
+ RecordStmtCount(S);
+ assert(!BreakContinueStack.empty() && "continue stmt not in a loop!");
+ BreakContinueStack.back().ContinueCount += PGO.getCurrentRegionCount();
+ PGO.setCurrentRegionUnreachable();
+ RecordNextStmtCount = true;
+ }
+
+ void VisitWhileStmt(const WhileStmt *S) {
+ RecordStmtCount(S);
+ RegionCounter Cnt(PGO, S);
+ BreakContinueStack.push_back(BreakContinue());
+ // Visit the body region first so the break/continue adjustments can be
+ // included when visiting the condition.
+ Cnt.beginRegion();
+ (*CountMap)[S->getBody()] = PGO.getCurrentRegionCount();
+ Visit(S->getBody());
+ Cnt.adjustForControlFlow();
+
+ // ...then go back and propagate counts through the condition. The count
+ // at the start of the condition is the sum of the incoming edges,
+ // the backedge from the end of the loop body, and the edges from
+ // continue statements.
+ BreakContinue BC = BreakContinueStack.pop_back_val();
+ Cnt.setCurrentRegionCount(Cnt.getParentCount() +
+ Cnt.getAdjustedCount() + BC.ContinueCount);
+ (*CountMap)[S->getCond()] = PGO.getCurrentRegionCount();
+ Visit(S->getCond());
+ Cnt.adjustForControlFlow();
+ Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount);
+ RecordNextStmtCount = true;
+ }
+
+ void VisitDoStmt(const DoStmt *S) {
+ RecordStmtCount(S);
+ RegionCounter Cnt(PGO, S);
+ BreakContinueStack.push_back(BreakContinue());
+ Cnt.beginRegion(/*AddIncomingFallThrough=*/true);
+ (*CountMap)[S->getBody()] = PGO.getCurrentRegionCount();
+ Visit(S->getBody());
+ Cnt.adjustForControlFlow();
+
+ BreakContinue BC = BreakContinueStack.pop_back_val();
+ // The count at the start of the condition is equal to the count at the
+ // end of the body. The adjusted count does not include either the
+ // fall-through count coming into the loop or the continue count, so add
+ // both of those separately. This is coincidentally the same equation as
+ // with while loops but for different reasons.
+ Cnt.setCurrentRegionCount(Cnt.getParentCount() +
+ Cnt.getAdjustedCount() + BC.ContinueCount);
+ (*CountMap)[S->getCond()] = PGO.getCurrentRegionCount();
+ Visit(S->getCond());
+ Cnt.adjustForControlFlow();
+ Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount);
+ RecordNextStmtCount = true;
+ }
+
+ void VisitForStmt(const ForStmt *S) {
+ RecordStmtCount(S);
+ if (S->getInit())
+ Visit(S->getInit());
+ RegionCounter Cnt(PGO, S);
+ BreakContinueStack.push_back(BreakContinue());
+ // Visit the body region first. (This is basically the same as a while
+ // loop; see further comments in VisitWhileStmt.)
+ Cnt.beginRegion();
+ (*CountMap)[S->getBody()] = PGO.getCurrentRegionCount();
+ Visit(S->getBody());
+ Cnt.adjustForControlFlow();
+
+ // The increment is essentially part of the body but it needs to include
+ // the count for all the continue statements.
+ if (S->getInc()) {
+ Cnt.setCurrentRegionCount(PGO.getCurrentRegionCount() +
+ BreakContinueStack.back().ContinueCount);
+ (*CountMap)[S->getInc()] = PGO.getCurrentRegionCount();
+ Visit(S->getInc());
+ Cnt.adjustForControlFlow();
+ }
+
+ BreakContinue BC = BreakContinueStack.pop_back_val();
+
+ // ...then go back and propagate counts through the condition.
+ if (S->getCond()) {
+ Cnt.setCurrentRegionCount(Cnt.getParentCount() +
+ Cnt.getAdjustedCount() +
+ BC.ContinueCount);
+ (*CountMap)[S->getCond()] = PGO.getCurrentRegionCount();
+ Visit(S->getCond());
+ Cnt.adjustForControlFlow();
+ }
+ Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount);
+ RecordNextStmtCount = true;
+ }
+
+ void VisitCXXForRangeStmt(const CXXForRangeStmt *S) {
+ RecordStmtCount(S);
+ Visit(S->getRangeStmt());
+ Visit(S->getBeginEndStmt());
+ RegionCounter Cnt(PGO, S);
+ BreakContinueStack.push_back(BreakContinue());
+ // Visit the body region first. (This is basically the same as a while
+ // loop; see further comments in VisitWhileStmt.)
+ Cnt.beginRegion();
+ (*CountMap)[S->getLoopVarStmt()] = PGO.getCurrentRegionCount();
+ Visit(S->getLoopVarStmt());
+ Visit(S->getBody());
+ Cnt.adjustForControlFlow();
+
+ // The increment is essentially part of the body but it needs to include
+ // the count for all the continue statements.
+ Cnt.setCurrentRegionCount(PGO.getCurrentRegionCount() +
+ BreakContinueStack.back().ContinueCount);
+ (*CountMap)[S->getInc()] = PGO.getCurrentRegionCount();
+ Visit(S->getInc());
+ Cnt.adjustForControlFlow();
+
+ BreakContinue BC = BreakContinueStack.pop_back_val();
+
+ // ...then go back and propagate counts through the condition.
+ Cnt.setCurrentRegionCount(Cnt.getParentCount() +
+ Cnt.getAdjustedCount() +
+ BC.ContinueCount);
+ (*CountMap)[S->getCond()] = PGO.getCurrentRegionCount();
+ Visit(S->getCond());
+ Cnt.adjustForControlFlow();
+ Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount);
+ RecordNextStmtCount = true;
+ }
+
+ void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S) {
+ RecordStmtCount(S);
+ Visit(S->getElement());
+ RegionCounter Cnt(PGO, S);
+ BreakContinueStack.push_back(BreakContinue());
+ Cnt.beginRegion();
+ (*CountMap)[S->getBody()] = PGO.getCurrentRegionCount();
+ Visit(S->getBody());
+ BreakContinue BC = BreakContinueStack.pop_back_val();
+ Cnt.adjustForControlFlow();
+ Cnt.applyAdjustmentsToRegion(BC.BreakCount + BC.ContinueCount);
+ RecordNextStmtCount = true;
+ }
+
+ void VisitSwitchStmt(const SwitchStmt *S) {
+ RecordStmtCount(S);
+ Visit(S->getCond());
+ PGO.setCurrentRegionUnreachable();
+ BreakContinueStack.push_back(BreakContinue());
+ Visit(S->getBody());
+ // If the switch is inside a loop, add the continue counts.
+ BreakContinue BC = BreakContinueStack.pop_back_val();
+ if (!BreakContinueStack.empty())
+ BreakContinueStack.back().ContinueCount += BC.ContinueCount;
+ RegionCounter ExitCnt(PGO, S);
+ ExitCnt.beginRegion();
+ RecordNextStmtCount = true;
+ }
+
+ void VisitCaseStmt(const CaseStmt *S) {
+ RecordNextStmtCount = false;
+ RegionCounter Cnt(PGO, S);
+ Cnt.beginRegion(/*AddIncomingFallThrough=*/true);
+ (*CountMap)[S] = Cnt.getCount();
+ RecordNextStmtCount = true;
+ Visit(S->getSubStmt());
+ }
+
+ void VisitDefaultStmt(const DefaultStmt *S) {
+ RecordNextStmtCount = false;
+ RegionCounter Cnt(PGO, S);
+ Cnt.beginRegion(/*AddIncomingFallThrough=*/true);
+ (*CountMap)[S] = Cnt.getCount();
+ RecordNextStmtCount = true;
+ Visit(S->getSubStmt());
+ }
+
+ void VisitIfStmt(const IfStmt *S) {
+ RecordStmtCount(S);
+ RegionCounter Cnt(PGO, S);
+ Visit(S->getCond());
+
+ Cnt.beginRegion();
+ (*CountMap)[S->getThen()] = PGO.getCurrentRegionCount();
+ Visit(S->getThen());
+ Cnt.adjustForControlFlow();
+
+ if (S->getElse()) {
+ Cnt.beginElseRegion();
+ (*CountMap)[S->getElse()] = PGO.getCurrentRegionCount();
+ Visit(S->getElse());
+ Cnt.adjustForControlFlow();
+ }
+ Cnt.applyAdjustmentsToRegion(0);
+ RecordNextStmtCount = true;
+ }
+
+ void VisitCXXTryStmt(const CXXTryStmt *S) {
+ RecordStmtCount(S);
+ Visit(S->getTryBlock());
+ for (unsigned I = 0, E = S->getNumHandlers(); I < E; ++I)
+ Visit(S->getHandler(I));
+ RegionCounter Cnt(PGO, S);
+ Cnt.beginRegion();
+ RecordNextStmtCount = true;
+ }
+
+ void VisitCXXCatchStmt(const CXXCatchStmt *S) {
+ RecordNextStmtCount = false;
+ RegionCounter Cnt(PGO, S);
+ Cnt.beginRegion();
+ (*CountMap)[S] = PGO.getCurrentRegionCount();
+ Visit(S->getHandlerBlock());
+ }
+
+ void VisitConditionalOperator(const ConditionalOperator *E) {
+ RecordStmtCount(E);
+ RegionCounter Cnt(PGO, E);
+ Visit(E->getCond());
+
+ Cnt.beginRegion();
+ (*CountMap)[E->getTrueExpr()] = PGO.getCurrentRegionCount();
+ Visit(E->getTrueExpr());
+ Cnt.adjustForControlFlow();
+
+ Cnt.beginElseRegion();
+ (*CountMap)[E->getFalseExpr()] = PGO.getCurrentRegionCount();
+ Visit(E->getFalseExpr());
+ Cnt.adjustForControlFlow();
+
+ Cnt.applyAdjustmentsToRegion(0);
+ RecordNextStmtCount = true;
+ }
+
+ void VisitBinLAnd(const BinaryOperator *E) {
+ RecordStmtCount(E);
+ RegionCounter Cnt(PGO, E);
+ Visit(E->getLHS());
+ Cnt.beginRegion();
+ (*CountMap)[E->getRHS()] = PGO.getCurrentRegionCount();
+ Visit(E->getRHS());
+ Cnt.adjustForControlFlow();
+ Cnt.applyAdjustmentsToRegion(0);
+ RecordNextStmtCount = true;
+ }
+
+ void VisitBinLOr(const BinaryOperator *E) {
+ RecordStmtCount(E);
+ RegionCounter Cnt(PGO, E);
+ Visit(E->getLHS());
+ Cnt.beginRegion();
+ (*CountMap)[E->getRHS()] = PGO.getCurrentRegionCount();
+ Visit(E->getRHS());
+ Cnt.adjustForControlFlow();
+ Cnt.applyAdjustmentsToRegion(0);
+ RecordNextStmtCount = true;
+ }
+ };
}
void CodeGenPGO::assignRegionCounters(GlobalDecl &GD) {
@@ -415,8 +739,10 @@ void CodeGenPGO::assignRegionCounters(GlobalDecl &GD) {
mapRegionCounters(D);
if (InstrumentRegions)
emitCounterVariables();
- if (PGOData)
+ if (PGOData) {
loadRegionCounts(GD, PGOData);
+ computeRegionCounts(D);
+ }
}
void CodeGenPGO::mapRegionCounters(const Decl *D) {
@@ -427,6 +753,13 @@ void CodeGenPGO::mapRegionCounters(const Decl *D) {
NumRegionCounters = Walker.NextCounter;
}
+void CodeGenPGO::computeRegionCounts(const Decl *D) {
+ StmtCountMap = new llvm::DenseMap<const Stmt*, uint64_t>();
+ ComputeRegionCounts Walker(StmtCountMap, *this);
+ if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
+ Walker.VisitFunctionDecl(FD);
+}
+
void CodeGenPGO::emitCounterVariables() {
llvm::LLVMContext &Ctx = CGM.getLLVMContext();
llvm::ArrayType *CounterTy = llvm::ArrayType::get(llvm::Type::getInt64Ty(Ctx),
@@ -464,6 +797,8 @@ void CodeGenPGO::loadRegionCounts(GlobalDecl &GD, PGOProfileData *PGOData) {
void CodeGenPGO::destroyRegionCounters() {
if (RegionCounterMap != 0)
delete RegionCounterMap;
+ if (StmtCountMap != 0)
+ delete StmtCountMap;
if (RegionCounts != 0)
delete RegionCounts;
}
@@ -493,3 +828,18 @@ llvm::MDNode *CodeGenPGO::createBranchWeights(ArrayRef<uint64_t> Weights) {
}
return MDHelper.createBranchWeights(ScaledWeights);
}
+
+llvm::MDNode *CodeGenPGO::createLoopWeights(const Stmt *Cond,
+ RegionCounter &Cnt) {
+ if (!haveRegionCounts())
+ return 0;
+ uint64_t LoopCount = Cnt.getCount();
+ uint64_t CondCount = 0;
+ bool Found = getStmtCount(Cond, CondCount);
+ assert(Found && "missing expected loop condition count");
+ (void)Found;
+ if (CondCount == 0)
+ return 0;
+ return createBranchWeights(LoopCount,
+ std::max(CondCount, LoopCount) - LoopCount);
+}
diff --git a/clang/lib/CodeGen/CodeGenPGO.h b/clang/lib/CodeGen/CodeGenPGO.h
index 38923d1b98f..083e5466c66 100644
--- a/clang/lib/CodeGen/CodeGenPGO.h
+++ b/clang/lib/CodeGen/CodeGenPGO.h
@@ -60,13 +60,14 @@ private:
unsigned NumRegionCounters;
llvm::GlobalVariable *RegionCounters;
llvm::DenseMap<const Stmt*, unsigned> *RegionCounterMap;
+ llvm::DenseMap<const Stmt*, uint64_t> *StmtCountMap;
std::vector<uint64_t> *RegionCounts;
uint64_t CurrentRegionCount;
public:
CodeGenPGO(CodeGenModule &CGM)
: CGM(CGM), NumRegionCounters(0), RegionCounters(0), RegionCounterMap(0),
- RegionCounts(0), CurrentRegionCount(0) {}
+ StmtCountMap(0), RegionCounts(0), CurrentRegionCount(0) {}
~CodeGenPGO() {}
/// Whether or not we have PGO region data for the current function. This is
@@ -76,22 +77,42 @@ public:
/// Return the counter value of the current region.
uint64_t getCurrentRegionCount() const { return CurrentRegionCount; }
- /// Return the counter value of the current region, or \p Min if it is larger.
- uint64_t getCurrentRegionCountWithMin(uint64_t Min) {
- return std::max(Min, CurrentRegionCount);
- }
+
/// Set the counter value for the current region. This is used to keep track
/// of changes to the most recent counter from control flow and non-local
/// exits.
void setCurrentRegionCount(uint64_t Count) { CurrentRegionCount = Count; }
+
/// Indicate that the current region is never reached, and thus should have a
/// counter value of zero. This is important so that subsequent regions can
/// correctly track their parent counts.
void setCurrentRegionUnreachable() { setCurrentRegionCount(0); }
+ /// Check if an execution count is known for a given statement. If so, return
+ /// true and put the value in Count; else return false.
+ bool getStmtCount(const Stmt *S, uint64_t &Count) {
+ if (!StmtCountMap)
+ return false;
+ llvm::DenseMap<const Stmt*, uint64_t>::const_iterator
+ I = StmtCountMap->find(S);
+ if (I == StmtCountMap->end())
+ return false;
+ Count = I->second;
+ return true;
+ }
+
+ /// If the execution count for the current statement is known, record that
+ /// as the current count.
+ void setCurrentStmt(const Stmt *S) {
+ uint64_t Count;
+ if (getStmtCount(S, Count))
+ setCurrentRegionCount(Count);
+ }
+
/// Calculate branch weights appropriate for PGO data
llvm::MDNode *createBranchWeights(uint64_t TrueCount, uint64_t FalseCount);
llvm::MDNode *createBranchWeights(ArrayRef<uint64_t> Weights);
+ llvm::MDNode *createLoopWeights(const Stmt *Cond, RegionCounter &Cnt);
/// Assign counters to regions and configure them for PGO of a given
/// function. Does nothing if instrumentation is not enabled and either
@@ -109,6 +130,7 @@ public:
private:
void mapRegionCounters(const Decl *D);
+ void computeRegionCounts(const Decl *D);
void loadRegionCounts(GlobalDecl &GD, PGOProfileData *PGOData);
void emitCounterVariables();
@@ -157,6 +179,7 @@ public:
/// the region of the counter was entered, but for switch labels it's the
/// number of direct jumps to that label.
uint64_t getCount() const { return Count; }
+
/// Get the value of the counter with adjustments applied. Adjustments occur
/// when control enters or leaves the region abnormally; i.e., if there is a
/// jump to a label within the region, or if the function can return from
@@ -166,43 +189,27 @@ public:
assert((Adjust > 0 || (uint64_t)(-Adjust) <= Count) && "Negative count");
return Count + Adjust;
}
+
/// Get the value of the counter in this region's parent, i.e., the region
/// that was active when this region began. This is useful for deriving
/// counts in implicitly counted regions, like the false case of a condition
/// or the normal exits of a loop.
uint64_t getParentCount() const { return ParentCount; }
- /// Get the number of times the condition of a loop will evaluate false. This
- /// is the number of times we enter the loop, adjusted by the difference
- /// between entering and exiting the loop body normally, excepting that
- /// 'continue' statements also bring us back here.
- ///
- /// Undefined if this counter is not counting a loop.
- uint64_t getLoopExitCount() const {
- return getParentCount() + getContinueCounter().getCount() +
- getAdjustedCount() - getCount();
- }
- /// Get the associated break counter. Undefined if this counter is not
- /// counting a loop.
- RegionCounter getBreakCounter() const {
- return RegionCounter(*PGO, Counter + 1);
- }
- /// Get the associated continue counter. Undefined if this counter is not
- /// counting a loop.
- RegionCounter getContinueCounter() const {
- return RegionCounter(*PGO, Counter + 2);
- }
-
/// Activate the counter by emitting an increment and starting to track
/// adjustments. If AddIncomingFallThrough is true, the current region count
/// will be added to the counter for the purposes of tracking the region.
void beginRegion(CGBuilderTy &Builder, bool AddIncomingFallThrough=false) {
+ beginRegion(AddIncomingFallThrough);
+ PGO->emitCounterIncrement(Builder, Counter);
+ }
+ void beginRegion(bool AddIncomingFallThrough=false) {
RegionCount = Count;
if (AddIncomingFallThrough)
RegionCount += PGO->getCurrentRegionCount();
PGO->setCurrentRegionCount(RegionCount);
- PGO->emitCounterIncrement(Builder, Counter);
}
+
/// For counters on boolean branches, begins tracking adjustments for the
/// uncounted path.
void beginElseRegion() {
@@ -210,17 +217,28 @@ public:
PGO->setCurrentRegionCount(RegionCount);
}
+ /// Reset the current region count.
+ void setCurrentRegionCount(uint64_t CurrentCount) {
+ RegionCount = CurrentCount;
+ PGO->setCurrentRegionCount(RegionCount);
+ }
+
/// Adjust for non-local control flow after emitting a subexpression or
/// substatement. This must be called to account for constructs such as gotos,
/// labels, and returns, so that we can ensure that our region's count is
/// correct in the code that follows.
void adjustForControlFlow() {
Adjust += PGO->getCurrentRegionCount() - RegionCount;
+ // Reset the region count in case this is called again later.
+ RegionCount = PGO->getCurrentRegionCount();
}
- /// Commit all adjustments to the current region. This should be called after
- /// all blocks that adjust for control flow count have been emitted.
- void applyAdjustmentsToRegion() {
- PGO->setCurrentRegionCount(ParentCount + Adjust);
+
+ /// Commit all adjustments to the current region. If the region is a loop,
+ /// the LoopAdjust value should be the count of all the breaks and continues
+ /// from the loop, to compensate for those counts being deducted from the
+ /// adjustments for the body of the loop.
+ void applyAdjustmentsToRegion(uint64_t LoopAdjust) {
+ PGO->setCurrentRegionCount(ParentCount + Adjust + LoopAdjust);
}
};
OpenPOWER on IntegriCloud