diff options
Diffstat (limited to 'clang/lib/CodeGen/CGStmt.cpp')
-rw-r--r-- | clang/lib/CodeGen/CGStmt.cpp | 227 |
1 files changed, 199 insertions, 28 deletions
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp index 6224839ff7c..5031d60cbb3 100644 --- a/clang/lib/CodeGen/CGStmt.cpp +++ b/clang/lib/CodeGen/CGStmt.cpp @@ -358,7 +358,9 @@ void CodeGenFunction::EmitLabel(const LabelDecl *D) { ResolveBranchFixups(Dest.getBlock()); } + RegionCounter Cnt = getPGORegionCounter(D->getStmt()); EmitBlock(Dest.getBlock()); + Cnt.beginRegion(Builder); } /// Change the cleanup scope of the labels in this lexical scope to @@ -402,12 +404,14 @@ void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) { EmitStopPoint(&S); EmitBranchThroughCleanup(getJumpDestForLabel(S.getLabel())); + PGO.setCurrentRegionCount(0); } void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) { if (const LabelDecl *Target = S.getConstantTarget()) { EmitBranchThroughCleanup(getJumpDestForLabel(Target)); + PGO.setCurrentRegionCount(0); return; } @@ -424,12 +428,14 @@ void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) { cast<llvm::PHINode>(IndGotoBB->begin())->addIncoming(V, CurBB); EmitBranch(IndGotoBB); + PGO.setCurrentRegionCount(0); } void CodeGenFunction::EmitIfStmt(const IfStmt &S) { // C99 6.8.4.1: The first substatement is executed if the expression compares // unequal to 0. The condition must be a scalar type. LexicalScope ConditionScope(*this, S.getSourceRange()); + RegionCounter Cnt = getPGORegionCounter(&S); if (S.getConditionVariable()) EmitAutoVarDecl(*S.getConditionVariable()); @@ -447,6 +453,8 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { // If the skipped block has no labels in it, just emit the executed block. // This avoids emitting dead code and simplifies the CFG substantially. if (!ContainsLabel(Skipped)) { + if (CondConstant) + Cnt.beginRegion(Builder); if (Executed) { RunCleanupsScope ExecutedScope(*this); EmitStmt(Executed); @@ -462,14 +470,17 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { llvm::BasicBlock *ElseBlock = ContBlock; if (S.getElse()) ElseBlock = createBasicBlock("if.else"); - EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock); + + EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, Cnt.getCount()); // Emit the 'then' code. - EmitBlock(ThenBlock); + EmitBlock(ThenBlock); + Cnt.beginRegion(Builder); { RunCleanupsScope ThenScope(*this); EmitStmt(S.getThen()); } + Cnt.adjustFallThroughCount(); EmitBranch(ContBlock); // Emit the 'else' code if present. @@ -478,10 +489,12 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { if (getDebugInfo()) Builder.SetCurrentDebugLocation(llvm::DebugLoc()); EmitBlock(ElseBlock); + Cnt.beginElseRegion(); { RunCleanupsScope ElseScope(*this); EmitStmt(Else); } + Cnt.adjustFallThroughCount(); // There is no need to emit line number for unconditional branch. if (getDebugInfo()) Builder.SetCurrentDebugLocation(llvm::DebugLoc()); @@ -490,9 +503,12 @@ void CodeGenFunction::EmitIfStmt(const IfStmt &S) { // Emit the continuation block for code after the if. EmitBlock(ContBlock, true); + Cnt.applyAdjustmentsToRegion(); } void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) { + RegionCounter Cnt = getPGORegionCounter(&S); + // Emit the header for the loop, which will also become // the continue target. JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond"); @@ -503,7 +519,7 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) { JumpDest LoopExit = getJumpDestInCurrentScope("while.end"); // Store the blocks to use for break and continue. - BreakContinueStack.push_back(BreakContinue(LoopExit, LoopHeader)); + BreakContinueStack.push_back(BreakContinue(LoopExit, LoopHeader, &Cnt)); // C++ [stmt.while]p2: // When the condition of a while statement is a declaration, the @@ -525,6 +541,7 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) { // while(1) is common, avoid extra exit blocks. Be sure // to correctly handle break/continue though. bool EmitBoolCondBranch = true; + llvm::BranchInst *CondBr = NULL; if (llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal)) if (C->isOne()) EmitBoolCondBranch = false; @@ -536,7 +553,7 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) { if (ConditionScope.requiresCleanups()) ExitBlock = createBasicBlock("while.exit"); - Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock); + CondBr = Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); @@ -549,11 +566,19 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) { { RunCleanupsScope BodyScope(*this); EmitBlock(LoopBody); + Cnt.beginRegion(Builder); EmitStmt(S.getBody()); } + Cnt.adjustFallThroughCount(); BreakContinueStack.pop_back(); + uint64_t LoopCount = Cnt.getCount(); + uint64_t ExitCount = Cnt.getLoopExitCount(); + if (EmitBoolCondBranch) + CondBr->setMetadata(llvm::LLVMContext::MD_prof, + PGO.createBranchWeights(LoopCount, ExitCount)); + // Immediately force cleanup. ConditionScope.ForceCleanup(); @@ -562,6 +587,7 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) { // Emit the exit block. EmitBlock(LoopExit.getBlock(), true); + PGO.setCurrentRegionCount(ExitCount + Cnt.getBreakCounter().getCount()); // The LoopHeader typically is just a branch if we skipped emitting // a branch, try to erase it. @@ -573,16 +599,20 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S) { JumpDest LoopExit = getJumpDestInCurrentScope("do.end"); JumpDest LoopCond = getJumpDestInCurrentScope("do.cond"); + RegionCounter Cnt = getPGORegionCounter(&S); + // Store the blocks to use for break and continue. - BreakContinueStack.push_back(BreakContinue(LoopExit, LoopCond)); + BreakContinueStack.push_back(BreakContinue(LoopExit, LoopCond, &Cnt)); // Emit the body of the loop. llvm::BasicBlock *LoopBody = createBasicBlock("do.body"); EmitBlock(LoopBody); + Cnt.beginRegion(Builder); { RunCleanupsScope BodyScope(*this); EmitStmt(S.getBody()); } + Cnt.adjustFallThroughCount(); BreakContinueStack.pop_back(); @@ -603,12 +633,18 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S) { if (C->isZero()) EmitBoolCondBranch = false; + uint64_t LoopCount = Cnt.getCount() - Cnt.getParentCount(); + uint64_t ExitCount = Cnt.getLoopExitCount(); + // As long as the condition is true, iterate the loop. - if (EmitBoolCondBranch) - Builder.CreateCondBr(BoolCondVal, LoopBody, LoopExit.getBlock()); + if (EmitBoolCondBranch) { + Builder.CreateCondBr(BoolCondVal, LoopBody, LoopExit.getBlock(), + PGO.createBranchWeights(LoopCount, ExitCount)); + } // Emit the exit block. EmitBlock(LoopExit.getBlock()); + PGO.setCurrentRegionCount(ExitCount + Cnt.getBreakCounter().getCount()); // The DoCond block typically is just a branch if we skipped // emitting a branch, try to erase it. @@ -617,6 +653,8 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S) { } void CodeGenFunction::EmitForStmt(const ForStmt &S) { + RegionCounter Cnt = getPGORegionCounter(&S); + JumpDest LoopExit = getJumpDestInCurrentScope("for.end"); RunCleanupsScope ForScope(*this); @@ -639,6 +677,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) { // Create a cleanup scope for the condition variable cleanups. RunCleanupsScope ConditionScope(*this); + llvm::BranchInst *CondBr = NULL; if (S.getCond()) { // If the for statement has a condition scope, emit the local variable // declaration. @@ -658,7 +697,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) { // C99 6.8.5p2/p4: The first substatement is executed if the expression // compares unequal to 0. The condition must be a scalar type. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); - Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock); + CondBr = Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); @@ -670,6 +709,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) { // Treat it as a non-zero constant. Don't even create a new block for the // body, just fall into it. } + Cnt.beginRegion(Builder); // If the for loop doesn't have an increment we can just use the // condition as the continue block. Otherwise we'll need to create @@ -679,7 +719,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) { Continue = getJumpDestInCurrentScope("for.inc"); // Store the blocks to use for break and continue. - BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); + BreakContinueStack.push_back(BreakContinue(LoopExit, Continue, &Cnt)); { // Create a separate cleanup scope for the body, in case it is not @@ -693,9 +733,16 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) { EmitBlock(Continue.getBlock()); EmitStmt(S.getInc()); } + Cnt.adjustFallThroughCount(); BreakContinueStack.pop_back(); + uint64_t LoopCount = Cnt.getCount(); + uint64_t ExitCount = Cnt.getLoopExitCount(); + if (S.getCond()) + CondBr->setMetadata(llvm::LLVMContext::MD_prof, + PGO.createBranchWeights(LoopCount, ExitCount)); + ConditionScope.ForceCleanup(); EmitBranch(CondBlock); @@ -706,9 +753,12 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S) { // Emit the fall-through block. EmitBlock(LoopExit.getBlock(), true); + PGO.setCurrentRegionCount(ExitCount + Cnt.getBreakCounter().getCount()); } void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) { + RegionCounter Cnt = getPGORegionCounter(&S); + JumpDest LoopExit = getJumpDestInCurrentScope("for.end"); RunCleanupsScope ForScope(*this); @@ -739,7 +789,8 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) { // The body is executed if the expression, contextually converted // to bool, is true. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); - Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock); + llvm::BranchInst *CondBr = Builder.CreateCondBr(BoolCondVal, + ForBody, ExitBlock); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); @@ -747,12 +798,13 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) { } EmitBlock(ForBody); + Cnt.beginRegion(Builder); // Create a block for the increment. In case of a 'continue', we jump there. JumpDest Continue = getJumpDestInCurrentScope("for.inc"); // Store the blocks to use for break and continue. - BreakContinueStack.push_back(BreakContinue(LoopExit, Continue)); + BreakContinueStack.push_back(BreakContinue(LoopExit, Continue, &Cnt)); { // Create a separate cleanup scope for the loop variable and body. @@ -764,9 +816,15 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) { // If there is an increment, emit it next. EmitBlock(Continue.getBlock()); EmitStmt(S.getInc()); + Cnt.adjustFallThroughCount(); BreakContinueStack.pop_back(); + uint64_t LoopCount = Cnt.getCount(); + uint64_t ExitCount = Cnt.getLoopExitCount(); + CondBr->setMetadata(llvm::LLVMContext::MD_prof, + PGO.createBranchWeights(LoopCount, ExitCount)); + EmitBranch(CondBlock); ForScope.ForceCleanup(); @@ -776,6 +834,7 @@ void CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S) { // Emit the fall-through block. EmitBlock(LoopExit.getBlock(), true); + PGO.setCurrentRegionCount(ExitCount + Cnt.getBreakCounter().getCount()); } void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) { @@ -789,6 +848,7 @@ void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) { /*init*/ true); } EmitBranchThroughCleanup(ReturnBlock); + PGO.setCurrentRegionCount(0); } /// EmitReturnStmt - Note that due to GCC extensions, this can have an operand @@ -860,6 +920,7 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) { cleanupScope.ForceCleanup(); EmitBranchThroughCleanup(ReturnBlock); + PGO.setCurrentRegionCount(0); } void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) { @@ -882,8 +943,14 @@ void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) { if (HaveInsertPoint()) EmitStopPoint(&S); - JumpDest Block = BreakContinueStack.back().BreakBlock; - EmitBranchThroughCleanup(Block); + BreakContinue &BC = BreakContinueStack.back(); + // We keep track of breaks from the loop so we can differentiate them from + // non-local exits in PGO instrumentation. This only applies to loops, not + // breaks from switch statements. + if (BC.CountBreak) + BC.LoopCnt->getBreakCounter().beginRegion(Builder); + EmitBranchThroughCleanup(BC.BreakBlock); + PGO.setCurrentRegionCount(0); } void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) { @@ -895,8 +962,12 @@ void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) { if (HaveInsertPoint()) EmitStopPoint(&S); - JumpDest Block = BreakContinueStack.back().ContinueBlock; - EmitBranchThroughCleanup(Block); + BreakContinue &BC = BreakContinueStack.back(); + // We keep track of continues in the loop so we can differentiate them from + // non-local exits in PGO instrumentation. + BC.LoopCnt->getContinueCounter().beginRegion(Builder); + EmitBranchThroughCleanup(BC.ContinueBlock); + PGO.setCurrentRegionCount(0); } /// EmitCaseStmtRange - If case statement range is not too big then @@ -908,11 +979,14 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) { llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(getContext()); llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(getContext()); + RegionCounter CaseCnt = getPGORegionCounter(&S); + // Emit the code for this case. We do this first to make sure it is // properly chained from our predecessor before generating the // switch machinery to enter this block. EmitBlock(createBasicBlock("sw.bb")); llvm::BasicBlock *CaseDest = Builder.GetInsertBlock(); + CaseCnt.beginRegion(Builder); EmitStmt(S.getSubStmt()); // If range is empty, do nothing. @@ -923,7 +997,17 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) { // FIXME: parameters such as this should not be hardcoded. if (Range.ult(llvm::APInt(Range.getBitWidth(), 64))) { // Range is small enough to add multiple switch instruction cases. - for (unsigned i = 0, e = Range.getZExtValue() + 1; i != e; ++i) { + uint64_t Total = CaseCnt.getCount() - CaseCnt.getParentCount(); + unsigned NCases = Range.getZExtValue() + 1; + // Divide the weights evenly between the cases, ensuring that the total + // weight is preserved. Ie, a weight of 5 over three cases will be + // distributed as weights of 2, 2, and 1. + uint64_t Weight = Total / NCases, Rem = Total % NCases; + for (unsigned I = 0; I != NCases; ++I) { + if (SwitchWeights) + SwitchWeights->push_back(Weight + (Rem ? 1 : 0)); + if (Rem) + Rem--; SwitchInsn->addCase(Builder.getInt(LHS), CaseDest); LHS++; } @@ -948,7 +1032,19 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S) { Builder.CreateSub(SwitchInsn->getCondition(), Builder.getInt(LHS)); llvm::Value *Cond = Builder.CreateICmpULE(Diff, Builder.getInt(Range), "inbounds"); - Builder.CreateCondBr(Cond, CaseDest, FalseDest); + + llvm::MDNode *Weights = 0; + if (SwitchWeights) { + uint64_t ThisCount = CaseCnt.getCount() - CaseCnt.getParentCount(); + uint64_t DefaultCount = (*SwitchWeights)[0]; + Weights = PGO.createBranchWeights(ThisCount, DefaultCount); + + // Since we're chaining the switch default through each large case range, we + // need to update the weight for the default, ie, the first case, to include + // this case. + (*SwitchWeights)[0] += ThisCount; + } + Builder.CreateCondBr(Cond, CaseDest, FalseDest, Weights); // Restore the appropriate insertion point. if (RestoreBB) @@ -974,17 +1070,22 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) { return; } + RegionCounter CaseCnt = getPGORegionCounter(&S); llvm::ConstantInt *CaseVal = Builder.getInt(S.getLHS()->EvaluateKnownConstInt(getContext())); - // If the body of the case is just a 'break', and if there was no fallthrough, - // try to not emit an empty block. - if ((CGM.getCodeGenOpts().OptimizationLevel > 0) && + // If the body of the case is just a 'break', try to not emit an empty block. + // If we're profiling or we're not optimizing, leave the block in for better + // debug and coverage analysis. + if (!CGM.getCodeGenOpts().ProfileInstrGenerate && + CGM.getCodeGenOpts().OptimizationLevel > 0 && isa<BreakStmt>(S.getSubStmt())) { JumpDest Block = BreakContinueStack.back().BreakBlock; // Only do this optimization if there are no cleanups that need emitting. if (isObviouslyBranchWithoutCleanups(Block)) { + if (SwitchWeights) + SwitchWeights->push_back(CaseCnt.getCount() - CaseCnt.getParentCount()); SwitchInsn->addCase(CaseVal, Block.getBlock()); // If there was a fallthrough into this case, make sure to redirect it to @@ -999,6 +1100,9 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) { EmitBlock(createBasicBlock("sw.bb")); llvm::BasicBlock *CaseDest = Builder.GetInsertBlock(); + if (SwitchWeights) + SwitchWeights->push_back(CaseCnt.getCount() - CaseCnt.getParentCount()); + CaseCnt.beginRegion(Builder); SwitchInsn->addCase(CaseVal, CaseDest); // Recursively emitting the statement is acceptable, but is not wonderful for @@ -1016,8 +1120,14 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) { // Otherwise, iteratively add consecutive cases to this switch stmt. while (NextCase && NextCase->getRHS() == 0) { CurCase = NextCase; - llvm::ConstantInt *CaseVal = + llvm::ConstantInt *CaseVal = Builder.getInt(CurCase->getLHS()->EvaluateKnownConstInt(getContext())); + + CaseCnt = getPGORegionCounter(NextCase); + if (SwitchWeights) + SwitchWeights->push_back(CaseCnt.getCount() - CaseCnt.getParentCount()); + CaseCnt.beginRegion(Builder); + SwitchInsn->addCase(CaseVal, CaseDest); NextCase = dyn_cast<CaseStmt>(CurCase->getSubStmt()); } @@ -1030,7 +1140,22 @@ void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S) { llvm::BasicBlock *DefaultBlock = SwitchInsn->getDefaultDest(); assert(DefaultBlock->empty() && "EmitDefaultStmt: Default block already defined?"); + + llvm::BasicBlock *SkipCountBB = 0; + if (CGM.getCodeGenOpts().ProfileInstrGenerate) { + // The PGO region here needs to count the number of times the edge occurs, + // so fallthrough into this case will jump past the region counter to the + // skipcount basic block. + SkipCountBB = createBasicBlock("skipcount"); + EmitBranch(SkipCountBB); + } EmitBlock(DefaultBlock); + + RegionCounter Cnt = getPGORegionCounter(&S); + Cnt.beginRegion(Builder, /*AddIncomingFallThrough=*/true); + + if (SkipCountBB) + EmitBlock(SkipCountBB); EmitStmt(S.getSubStmt()); } @@ -1187,7 +1312,8 @@ static CSFC_Result CollectStatementsForCase(const Stmt *S, static bool FindCaseStatementsForValue(const SwitchStmt &S, const llvm::APSInt &ConstantCondValue, SmallVectorImpl<const Stmt*> &ResultStmts, - ASTContext &C) { + ASTContext &C, + const SwitchCase *&ResultCase) { // First step, find the switch case that is being branched to. We can do this // efficiently by scanning the SwitchCase list. const SwitchCase *Case = S.getSwitchCaseList(); @@ -1230,6 +1356,7 @@ static bool FindCaseStatementsForValue(const SwitchStmt &S, // while (1) { // case 4: ... bool FoundCase = false; + ResultCase = Case; return CollectStatementsForCase(S.getBody(), Case, FoundCase, ResultStmts) != CSFC_Failure && FoundCase; @@ -1245,6 +1372,7 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { // Handle nested switch statements. llvm::SwitchInst *SavedSwitchInsn = SwitchInsn; + SmallVector<uint64_t, 16> *SavedSwitchWeights = SwitchWeights; llvm::BasicBlock *SavedCRBlock = CaseRangeBlock; // See if we can constant fold the condition of the switch and therefore only @@ -1252,8 +1380,14 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { llvm::APSInt ConstantCondValue; if (ConstantFoldsToSimpleInteger(S.getCond(), ConstantCondValue)) { SmallVector<const Stmt*, 4> CaseStmts; + const SwitchCase *Case = 0; if (FindCaseStatementsForValue(S, ConstantCondValue, CaseStmts, - getContext())) { + getContext(), Case)) { + PGO.setCurrentRegionCount(0); + if (Case) { + RegionCounter CaseCnt = getPGORegionCounter(Case); + CaseCnt.beginRegion(Builder); + } RunCleanupsScope ExecutedScope(*this); // At this point, we are no longer "within" a switch instance, so @@ -1265,6 +1399,8 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { // specified series of statements and we're good. for (unsigned i = 0, e = CaseStmts.size(); i != e; ++i) EmitStmt(CaseStmts[i]); + RegionCounter ExitCnt = getPGORegionCounter(&S); + ExitCnt.beginRegion(Builder); // Now we want to restore the saved switch instance so that nested // switches continue to function properly @@ -1282,18 +1418,41 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { // failure. llvm::BasicBlock *DefaultBlock = createBasicBlock("sw.default"); SwitchInsn = Builder.CreateSwitch(CondV, DefaultBlock); + if (PGO.haveRegionCounts()) { + // Walk the SwitchCase list to find how many there are. + uint64_t DefaultCount = 0; + unsigned NumCases = 0; + for (const SwitchCase *Case = S.getSwitchCaseList(); + Case; + Case = Case->getNextSwitchCase()) { + if (isa<DefaultStmt>(Case)) + DefaultCount = getPGORegionCounter(Case).getCount(); + NumCases += 1; + } + SwitchWeights = new SmallVector<uint64_t, 16>(); + SwitchWeights->reserve(NumCases); + // The default needs to be first. We store the edge count, so we already + // know the right weight. + SwitchWeights->push_back(DefaultCount); + } CaseRangeBlock = DefaultBlock; // Clear the insertion point to indicate we are in unreachable code. Builder.ClearInsertionPoint(); + PGO.setCurrentRegionCount(0); // All break statements jump to NextBlock. If BreakContinueStack is non-empty - // then reuse last ContinueBlock. + // then reuse last ContinueBlock and that block's counter. JumpDest OuterContinue; - if (!BreakContinueStack.empty()) - OuterContinue = BreakContinueStack.back().ContinueBlock; + RegionCounter *OuterCount = 0; + if (!BreakContinueStack.empty()) { + BreakContinue &BC = BreakContinueStack.back(); + OuterContinue = BC.ContinueBlock; + OuterCount = BC.LoopCnt; + } - BreakContinueStack.push_back(BreakContinue(SwitchExit, OuterContinue)); + BreakContinueStack.push_back(BreakContinue(SwitchExit, OuterContinue, + OuterCount, /*CountBreak=*/false)); // Emit switch body. EmitStmt(S.getBody()); @@ -1322,8 +1481,20 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { // Emit continuation. EmitBlock(SwitchExit.getBlock(), true); - + RegionCounter ExitCnt = getPGORegionCounter(&S); + ExitCnt.beginRegion(Builder); + + if (SwitchWeights) { + assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() && + "switch weights do not match switch cases"); + // If there's only one jump destination there's no sense weighting it. + if (SwitchWeights->size() > 1) + SwitchInsn->setMetadata(llvm::LLVMContext::MD_prof, + PGO.createBranchWeights(*SwitchWeights)); + delete SwitchWeights; + } SwitchInsn = SavedSwitchInsn; + SwitchWeights = SavedSwitchWeights; CaseRangeBlock = SavedCRBlock; } |