[X86] Correct dwarf unwind information in function epilogue

CFI instructions that set appropriate cfa offset and cfa register are now
inserted in emitEpilogue() in X86FrameLowering.

Majority of the changes in this patch:

1. Ensure that CFI instructions do not affect code generation.
2. Enable maintaining correct information about cfa offset and cfa register
in a function when basic blocks are reordered, merged, split, duplicated.

These changes are target independent and described below.

Changed CFI instructions so that they:

1. are duplicable
2. are not counted as instructions when tail duplicating or tail merging
3. can be compared as equal

Add information to each MachineBasicBlock about cfa offset and cfa register
that are valid at its entry and exit (incoming and outgoing CFI info). Add
support for updating this information when basic blocks are merged, split,
duplicated, created. Add a verification pass (CFIInfoVerifier) that checks
that outgoing cfa offset and register of predecessor blocks match incoming
values of their successors.

Incoming and outgoing CFI information is used by a late pass
(CFIInstrInserter) that corrects CFA calculation rule for a basic block if
needed. That means that additional CFI instructions get inserted at basic
block beginning to correct the rule for calculating CFA. Having CFI
instructions in function epilogue can cause incorrect CFA calculation rule
for some basic blocks. This can happen if, due to basic block reordering,
or the existence of multiple epilogue blocks, some of the blocks have wrong
cfa offset and register values set by the epilogue block above them.

Patch by Violeta Vukobrat.

Differential Revision: https://reviews.llvm.org/D18046

llvm-svn: 306529

10 files changed, 585 insertions, 14 deletions

diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index 53095497629..ed7e22d3eef 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -304,9 +304,9 @@ static unsigned ComputeCommonTailLength(MachineBasicBlock *MBB1,
   while (I1 != MBB1->begin() && I2 != MBB2->begin()) {
     --I1; --I2;
     // Skip debugging pseudos; necessary to avoid changing the code.
-    while (I1->isDebugValue()) {
+    while (I1->isDirective()) {
       if (I1==MBB1->begin()) {
-        while (I2->isDebugValue()) {
+        while (I2->isDirective()) {
           if (I2==MBB2->begin())
             // I1==DBG at begin; I2==DBG at begin
             return TailLen;
@@ -319,7 +319,7 @@ static unsigned ComputeCommonTailLength(MachineBasicBlock *MBB1,
       --I1;
     }
     // I1==first (untested) non-DBG preceding known match
-    while (I2->isDebugValue()) {
+    while (I2->isDirective()) {
       if (I2==MBB2->begin()) {
         ++I1;
         // I1==non-DBG, or first of DBGs not at begin; I2==DBG at begin
@@ -362,6 +362,35 @@ static unsigned ComputeCommonTailLength(MachineBasicBlock *MBB1,
     }
     ++I1;
   }
+
+  // Ensure that I1 and I2 do not point to a CFI_INSTRUCTION. This can happen if
+  // I1 and I2 are non-identical when compared and then one or both of them ends
+  // up pointing to a CFI instruction after being incremented. For example:
+  /*
+    BB1:
+    ...
+    INSTRUCTION_A
+    ADD32ri8  <- last common instruction
+    ...
+    BB2:
+    ...
+    INSTRUCTION_B
+    CFI_INSTRUCTION
+    ADD32ri8  <- last common instruction
+    ...
+  */
+  // When INSTRUCTION_A and INSTRUCTION_B are compared as not equal, after
+  // incrementing the iterators, I1 will point to ADD, however I2 will point to
+  // the CFI instruction. Later on, this leads to BB2 being 'hacked off' at the
+  // wrong place (in ReplaceTailWithBranchTo()) which results in losing this CFI
+  // instruction.
+  while (I1 != MBB1->end() && I1->isCFIInstruction()) {
+    ++I1;
+  }
+
+  while (I2 != MBB2->end() && I2->isCFIInstruction()) {
+    ++I2;
+  }
   return TailLen;
 }
 
@@ -417,6 +446,14 @@ MachineBasicBlock *BranchFolder::SplitMBBAt(MachineBasicBlock &CurMBB,
     FuncletMembership[NewMBB] = n;
   }
 
+  // Recalculate CFI info for CurMBB. Use existing incoming cfa offset and
+  // register.
+  CurMBB.recalculateCFIInfo(true);
+  // Recalculate CFI info for NewMBB. Use CurMBB's outgoing cfa offset and
+  // register as NewMBB's incoming.
+  NewMBB->recalculateCFIInfo(false, CurMBB.getOutgoingCFAOffset(),
+                             CurMBB.getOutgoingCFARegister());
+
   return NewMBB;
 }
 
@@ -426,7 +463,7 @@ static unsigned EstimateRuntime(MachineBasicBlock::iterator I,
                                 MachineBasicBlock::iterator E) {
   unsigned Time = 0;
   for (; I != E; ++I) {
-    if (I->isDebugValue())
+    if (I->isDirective())
       continue;
     if (I->isCall())
       Time += 10;
@@ -780,7 +817,7 @@ void BranchFolder::MergeCommonTailDebugLocs(unsigned commonTailIndex) {
   }
 
   for (auto &MI : *MBB) {
-    if (MI.isDebugValue())
+    if (MI.isDirective())
       continue;
     DebugLoc DL = MI.getDebugLoc();
     for (unsigned int i = 0 ; i < NextCommonInsts.size() ; i++) {
@@ -790,7 +827,7 @@ void BranchFolder::MergeCommonTailDebugLocs(unsigned commonTailIndex) {
       auto &Pos = NextCommonInsts[i];
       assert(Pos != SameTails[i].getBlock()->end() &&
           "Reached BB end within common tail");
-      while (Pos->isDebugValue()) {
+      while (Pos->isDirective()) {
         ++Pos;
         assert(Pos != SameTails[i].getBlock()->end() &&
             "Reached BB end within common tail");
@@ -823,12 +860,12 @@ mergeOperations(MachineBasicBlock::iterator MBBIStartPos,
     assert(MBBI != MBBIE && "Reached BB end within common tail length!");
     (void)MBBIE;
 
-    if (MBBI->isDebugValue()) {
+    if (MBBI->isDirective()) {
       ++MBBI;
       continue;
     }
 
-    while ((MBBICommon != MBBIECommon) && MBBICommon->isDebugValue())
+    while ((MBBICommon != MBBIECommon) && MBBICommon->isDirective())
       ++MBBICommon;
 
     assert(MBBICommon != MBBIECommon &&
@@ -971,6 +1008,11 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB,
       mergeOperations(SameTails[i].getTailStartPos(), *MBB);
       // Hack the end off BB i, making it jump to BB commonTailIndex instead.
       ReplaceTailWithBranchTo(SameTails[i].getTailStartPos(), MBB);
+
+      // Recalculate CFI info for BB. Use existing incoming cfa offset and
+      // register.
+      SameTails[i].getBlock()->recalculateCFIInfo(true);
+
       // BB i is no longer a predecessor of SuccBB; remove it from the worklist.
       MergePotentials.erase(SameTails[i].getMPIter());
     }
@@ -1381,6 +1423,10 @@ ReoptimizeBlock:
       assert(PrevBB.succ_empty());
       PrevBB.transferSuccessors(MBB);
       MadeChange = true;
+
+      // Update CFI info for PrevBB.
+      PrevBB.mergeCFIInfo(MBB);
+
       return MadeChange;
     }
 
diff --git a/llvm/lib/CodeGen/CFIInfoVerifier.cpp b/llvm/lib/CodeGen/CFIInfoVerifier.cpp
new file mode 100644
index 00000000000..ff5a953c242
--- /dev/null
+++ b/llvm/lib/CodeGen/CFIInfoVerifier.cpp
@@ -0,0 +1,123 @@
+//===----------- CFIInfoVerifier.cpp - CFI Information Verifier -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass verifies incoming and outgoing CFI information of basic blocks. CFI
+// information is information about offset and register set by CFI directives,
+// valid at the start and end of a basic block. This pass checks that outgoing
+// information of predecessors matches incoming information of their successors.
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/Target/TargetMachine.h"
+using namespace llvm;
+
+namespace {
+class CFIInfoVerifier : public MachineFunctionPass {
+ public:
+  static char ID;
+
+  CFIInfoVerifier() : MachineFunctionPass(ID) {
+    initializeCFIInfoVerifierPass(*PassRegistry::getPassRegistry());
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    bool NeedsDwarfCFI = (MF.getMMI().hasDebugInfo() ||
+                          MF.getFunction()->needsUnwindTableEntry()) &&
+                         (!MF.getTarget().getTargetTriple().isOSDarwin() &&
+                          !MF.getTarget().getTargetTriple().isOSWindows());
+    if (!NeedsDwarfCFI) return false;
+    verify(MF);
+    return false;
+  }
+
+ private:
+  // Go through each MBB in a function and check that outgoing offset and
+  // register of its predecessors match incoming offset and register of that
+  // MBB, as well as that incoming offset and register of its successors match
+  // outgoing offset and register of the MBB.
+  void verify(MachineFunction &MF);
+  void report(const char *msg, MachineBasicBlock &MBB);
+};
+}
+
+char CFIInfoVerifier::ID = 0;
+INITIALIZE_PASS(CFIInfoVerifier, "cfiinfoverifier",
+                "Verify that corresponding in/out CFI info matches", false,
+                false)
+FunctionPass *llvm::createCFIInfoVerifier() { return new CFIInfoVerifier(); }
+
+void CFIInfoVerifier::verify(MachineFunction &MF) {
+  for (auto &CurrMBB : MF) {
+    for (auto Pred : CurrMBB.predecessors()) {
+      // Check that outgoing offset values of predecessors match the incoming
+      // offset value of CurrMBB
+      if (Pred->getOutgoingCFAOffset() != CurrMBB.getIncomingCFAOffset()) {
+        report("The outgoing offset of a predecessor is inconsistent.",
+               CurrMBB);
+        errs() << "Predecessor BB#" << Pred->getNumber()
+               << " has outgoing offset (" << Pred->getOutgoingCFAOffset()
+               << "), while BB#" << CurrMBB.getNumber()
+               << " has incoming offset (" << CurrMBB.getIncomingCFAOffset()
+               << ").\n";
+      }
+      // Check that outgoing register values of predecessors match the incoming
+      // register value of CurrMBB
+      if (Pred->getOutgoingCFARegister() != CurrMBB.getIncomingCFARegister()) {
+        report("The outgoing register of a predecessor is inconsistent.",
+               CurrMBB);
+        errs() << "Predecessor BB#" << Pred->getNumber()
+               << " has outgoing register (" << Pred->getOutgoingCFARegister()
+               << "), while BB#" << CurrMBB.getNumber()
+               << " has incoming register (" << CurrMBB.getIncomingCFARegister()
+               << ").\n";
+      }
+    }
+
+    for (auto Succ : CurrMBB.successors()) {
+      // Check that incoming offset values of successors match the outgoing
+      // offset value of CurrMBB
+      if (Succ->getIncomingCFAOffset() != CurrMBB.getOutgoingCFAOffset()) {
+        report("The incoming offset of a successor is inconsistent.", CurrMBB);
+        errs() << "Successor BB#" << Succ->getNumber()
+               << " has incoming offset (" << Succ->getIncomingCFAOffset()
+               << "), while BB#" << CurrMBB.getNumber()
+               << " has outgoing offset (" << CurrMBB.getOutgoingCFAOffset()
+               << ").\n";
+      }
+      // Check that incoming register values of successors match the outgoing
+      // register value of CurrMBB
+      if (Succ->getIncomingCFARegister() != CurrMBB.getOutgoingCFARegister()) {
+        report("The incoming register of a successor is inconsistent.",
+               CurrMBB);
+        errs() << "Successor BB#" << Succ->getNumber()
+               << " has incoming register (" << Succ->getIncomingCFARegister()
+               << "), while BB#" << CurrMBB.getNumber()
+               << " has outgoing register (" << CurrMBB.getOutgoingCFARegister()
+               << ").\n";
+      }
+    }
+  }
+}
+
+void CFIInfoVerifier::report(const char *msg, MachineBasicBlock &MBB) {
+  assert(&MBB);
+  errs() << '\n';
+  errs() << "*** " << msg << " ***\n"
+         << "- function:    " << MBB.getParent()->getName() << "\n";
+  errs() << "- basic block: BB#" << MBB.getNumber() << ' ' << MBB.getName()
+         << " (" << (const void *)&MBB << ')';
+  errs() << '\n';
+}
diff --git a/llvm/lib/CodeGen/CFIInstrInserter.cpp b/llvm/lib/CodeGen/CFIInstrInserter.cpp
new file mode 100644
index 00000000000..b83c957d7c4
--- /dev/null
+++ b/llvm/lib/CodeGen/CFIInstrInserter.cpp
@@ -0,0 +1,124 @@
+//===------ CFIInstrInserter.cpp - Insert additional CFI instructions -----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Insert CFI instructions at the beginnings of basic blocks if needed. CFI
+// instructions are inserted if basic blocks have incorrect offset or register
+// set by prevoius blocks.
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
+using namespace llvm;
+
+namespace {
+class CFIInstrInserter : public MachineFunctionPass {
+ public:
+  CFIInstrInserter() : MachineFunctionPass(ID) {
+    initializeCFIInstrInserterPass(*PassRegistry::getPassRegistry());
+  }
+  bool runOnMachineFunction(MachineFunction &MF) override;
+  static char ID;
+
+ private:
+  StringRef getPassName() const override { return "CFI Instruction Inserter"; }
+
+  // Check if incoming CFI information of a basic block matches outgoing CFI
+  // information of the previous block. If it doesn't, insert CFI instruction at
+  // the beginning of the block that corrects the CFA calculation rule for that
+  // block.
+  void CorrectCFA(MachineFunction &MF);
+
+  // Return the cfa offset value that should be set at the beginning of MBB if
+  // needed. The negated value is needed when creating CFI instructions that set
+  // absolute offset.
+  int getCorrectCFAOffset(MachineBasicBlock &MBB) {
+    return -MBB.getIncomingCFAOffset();
+  }
+
+  // Were any CFI instructions inserted
+  bool InsertedCFIInstr = false;
+};
+}
+
+char CFIInstrInserter::ID = 0;
+INITIALIZE_PASS(CFIInstrInserter, "cfiinstrinserter",
+                "Check CFI info and insert CFI instructions if needed", false,
+                false)
+
+FunctionPass *llvm::createCFIInstrInserter() { return new CFIInstrInserter(); }
+
+bool CFIInstrInserter::runOnMachineFunction(MachineFunction &MF) {
+  bool NeedsDwarfCFI = (MF.getMMI().hasDebugInfo() ||
+                        MF.getFunction()->needsUnwindTableEntry()) &&
+                       (!MF.getTarget().getTargetTriple().isOSDarwin() &&
+                        !MF.getTarget().getTargetTriple().isOSWindows());
+
+  if (!NeedsDwarfCFI) return false;
+
+  // Insert appropriate CFI instructions for each MBB if CFA calculation rule
+  // needs to be corrected for that MBB.
+  CorrectCFA(MF);
+
+  return InsertedCFIInstr;
+}
+
+void CFIInstrInserter::CorrectCFA(MachineFunction &MF) {
+
+  MachineBasicBlock &FirstMBB = MF.front();
+  MachineBasicBlock *PrevMBB = &FirstMBB;
+  const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
+  InsertedCFIInstr = false;
+
+  for (auto &MBB : MF) {
+    // Skip the first MBB in a function
+    if (MBB.getNumber() == FirstMBB.getNumber()) continue;
+
+    auto MBBI = MBB.begin();
+    DebugLoc DL = MBB.findDebugLoc(MBBI);
+
+    if (PrevMBB->getOutgoingCFAOffset() != MBB.getIncomingCFAOffset()) {
+      // If both outgoing offset and register of a previous block don't match
+      // incoming offset and register of this block, add a def_cfa instruction
+      // with the correct offset and register for this block.
+      if (PrevMBB->getOutgoingCFARegister() != MBB.getIncomingCFARegister()) {
+        unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
+            nullptr, MBB.getIncomingCFARegister(), getCorrectCFAOffset(MBB)));
+        BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+            .addCFIIndex(CFIIndex);
+        // If outgoing offset of a previous block doesn't match incoming offset
+        // of this block, add a def_cfa_offset instruction with the correct
+        // offset for this block.
+      } else {
+        unsigned CFIIndex =
+            MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(
+                nullptr, getCorrectCFAOffset(MBB)));
+        BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+            .addCFIIndex(CFIIndex);
+      }
+      InsertedCFIInstr = true;
+      // If outgoing register of a previous block doesn't match incoming
+      // register of this block, add a def_cfa_register instruction with the
+      // correct register for this block.
+    } else if (PrevMBB->getOutgoingCFARegister() !=
+               MBB.getIncomingCFARegister()) {
+      unsigned CFIIndex =
+          MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(
+              nullptr, MBB.getIncomingCFARegister()));
+      BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+          .addCFIIndex(CFIIndex);
+      InsertedCFIInstr = true;
+    }
+    PrevMBB = &MBB;
+  }
+}
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index 7f3c6da9126..be6dcbfee0a 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -10,6 +10,8 @@ add_llvm_library(LLVMCodeGen
   BuiltinGCs.cpp
   CalcSpillWeights.cpp
   CallingConvLower.cpp
+  CFIInfoVerifier.cpp
+  CFIInstrInserter.cpp
   CodeGen.cpp
   CodeGenPrepare.cpp
   CountingFunctionInserter.cpp
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index faa5f139cf7..453e17e575d 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -24,6 +24,8 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeBranchCoalescingPass(Registry);
   initializeBranchFolderPassPass(Registry);
   initializeBranchRelaxationPass(Registry);
+  initializeCFIInfoVerifierPass(Registry);
+  initializeCFIInstrInserterPass(Registry);
   initializeCodeGenPreparePass(Registry);
   initializeCountingFunctionInserterPass(Registry);
   initializeDeadMachineInstructionElimPass(Registry);
diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp
index 81597afe6b0..c9976d7f2ff 100644
--- a/llvm/lib/CodeGen/MachineBasicBlock.cpp
+++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp
@@ -35,6 +35,8 @@
 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
 #include <algorithm>
+#include <queue>
+#include <set>
 using namespace llvm;
 
 #define DEBUG_TYPE "codegen"
@@ -1343,3 +1345,228 @@ MachineBasicBlock::livein_iterator MachineBasicBlock::livein_begin() const {
       "Liveness information is accurate");
   return LiveIns.begin();
 }
+
+void MachineBasicBlock::updateCFIInfo(MachineBasicBlock::iterator Pos) {
+  // Used for calculating outgoing cfa offset when CFI instruction added at Pos
+  // is def_cfa or def_cfa_offset.
+  /* For example:
+     ...
+     .cfi_adjust_cfa_offset 4
+     ...
+     .cfi_adjust_cfa_offset 4
+     ...
+     .cfi_def_cfa_offset 16  <---- newly added CFI instruction at Pos
+     ...
+     .cfi_adjust_cfa_offset 4
+     ...
+     Once def_cfa_offset is inserted, outgoing cfa offset is no longer
+     calculated as incoming offset incremented by the sum of all adjustments
+     (12). It becomes equal to the offset set by the added CFI instruction (16)
+     incremented by the sum of adjustments below it (4). Adjustments above the
+     added def_cfa_offset directive don't have effect below it anymore and
+     therefore don't affect the value of outgoing cfa offset.
+  */
+  int AdjustAmount = 0;
+  // Used to check if outgoing cfa offset should be updated or not (when def_cfa
+  // is inserted).
+  bool ShouldSetOffset = true;
+  // Used to check if outgoing cfa register should be updated or not (when
+  // def_cfa is inserted).
+  bool ShouldSetRegister = true;
+  const std::vector<MCCFIInstruction> CFIInstructions =
+      getParent()->getFrameInstructions();
+  MCCFIInstruction CFI = CFIInstructions[Pos->getOperand(0).getCFIIndex()];
+  // Type of the CFI instruction that was inserted.
+  MCCFIInstruction::OpType CFIType = CFI.getOperation();
+
+  // Check if there are already existing CFI instructions below Pos and see if
+  // outgoing CFI info should be updated or not.
+  for (MachineBasicBlock::reverse_iterator RI = rbegin();
+       RI != Pos.getReverse(); ++RI) {
+    if (RI->isCFIInstruction()) {
+      MCCFIInstruction::OpType RIType =
+          CFIInstructions[RI->getOperand(0).getCFIIndex()].getOperation();
+      switch (RIType) {
+        case MCCFIInstruction::OpAdjustCfaOffset:
+          AdjustAmount +=
+              CFIInstructions[RI->getOperand(0).getCFIIndex()].getOffset();
+          break;
+        case MCCFIInstruction::OpDefCfaOffset:
+          // CFI instruction doesn't affect outgoing cfa offset if there is
+          // already a def_cfa_offset instruction below it.
+          if (CFIType == MCCFIInstruction::OpDefCfaOffset ||
+              CFIType == MCCFIInstruction::OpAdjustCfaOffset)
+            return;
+          if (CFIType == MCCFIInstruction::OpDefCfa) {
+            // CFI instruction doesn't affect outgoing cfa offset and register
+            // if there are both def_cfa_offset and def_cfa_register
+            // instructions below it.
+            if (!ShouldSetRegister) return;
+            ShouldSetOffset = false;
+          }
+          break;
+        case MCCFIInstruction::OpDefCfaRegister:
+          // CFI instruction doesn't affect outgoing cfa register if there is
+          // already a def_cfa_register instruction below it.
+          if (CFIType == MCCFIInstruction::OpDefCfaRegister) return;
+          if (CFIType == MCCFIInstruction::OpDefCfa) {
+            // CFI instruction doesn't affect outgoing cfa offset and register
+            // if there are both def_cfa_offset and def_cfa_register
+            // instructions below it.
+            if (!ShouldSetOffset) return;
+            ShouldSetRegister = false;
+          }
+          break;
+        case MCCFIInstruction::OpDefCfa:
+          // CFI instruction doesn't affect outgoing cfa offset and register if
+          // there is already a def_cfa instruction below it.
+          if (CFIType == MCCFIInstruction::OpDefCfaRegister ||
+              CFIType == MCCFIInstruction::OpDefCfaOffset ||
+              CFIType == MCCFIInstruction::OpDefCfa ||
+              CFIType == MCCFIInstruction::OpAdjustCfaOffset)
+            return;
+          break;
+        default:
+          break;
+      }
+    }
+  }
+
+  // Update the outgoing CFI info based on the added CFI instruction.
+  switch (CFIType) {
+    case MCCFIInstruction::OpAdjustCfaOffset:
+      setOutgoingCFAOffset(getOutgoingCFAOffset() + CFI.getOffset());
+      break;
+    case MCCFIInstruction::OpDefCfaOffset:
+      setOutgoingCFAOffset(CFI.getOffset() + AdjustAmount);
+      break;
+    case MCCFIInstruction::OpDefCfaRegister:
+      setOutgoingCFARegister(CFI.getRegister());
+      break;
+    case MCCFIInstruction::OpDefCfa:
+      if (ShouldSetOffset) setOutgoingCFAOffset(CFI.getOffset() + AdjustAmount);
+      if (ShouldSetRegister) setOutgoingCFARegister(CFI.getRegister());
+      break;
+    default:
+      break;
+  }
+}
+
+void MachineBasicBlock::updateCFIInfoSucc() {
+  // Blocks whose successors' CFI info should be updated.
+  std::queue<MachineBasicBlock *> Successors;
+  // Keep track of basic blocks that have already been put in the Successors
+  // queue.
+  std::set<MachineBasicBlock *> ProcessedMBBs;
+  // Start with updating CFI info for direct successors of this block.
+  Successors.push(this);
+  ProcessedMBBs.insert(this);
+
+  // Go through the successors and update their CFI info if needed.
+  while (!Successors.empty()) {
+    MachineBasicBlock *CurrSucc = Successors.front();
+    Successors.pop();
+
+    // Update CFI info for CurrSucc's successors.
+    for (auto Succ : CurrSucc->successors()) {
+      if (ProcessedMBBs.find(Succ) != ProcessedMBBs.end()) continue;
+      if (Succ->getIncomingCFAOffset() == CurrSucc->getOutgoingCFAOffset() &&
+          Succ->getIncomingCFARegister() == CurrSucc->getOutgoingCFARegister())
+        continue;
+      bool ChangedOutgoingInfo = false;
+      // Do not update cfa offset if the existing value matches the new.
+      if (Succ->getIncomingCFAOffset() != CurrSucc->getOutgoingCFAOffset()) {
+        // If the block doesn't have a def_cfa_offset or def_cfa directive,
+        // update its outgoing offset.
+        if (!Succ->hasDefOffset()) {
+          // Succ block doesn't set absolute offset, so the difference between
+          // outgoing and incoming offset remains the same. This difference is
+          // the sum of offsets set by adjust_cfa_offset directives.
+          int AdjustAmount =
+              Succ->getOutgoingCFAOffset() - Succ->getIncomingCFAOffset();
+          Succ->setOutgoingCFAOffset(CurrSucc->getOutgoingCFAOffset() +
+                                     AdjustAmount);
+          ChangedOutgoingInfo = true;
+        }
+        Succ->setIncomingCFAOffset(CurrSucc->getOutgoingCFAOffset());
+      }
+      // Do not update cfa register if the existing value matches the new.
+      if (Succ->getIncomingCFARegister() !=
+          CurrSucc->getOutgoingCFARegister()) {
+        Succ->setIncomingCFARegister(CurrSucc->getOutgoingCFARegister());
+        // If the block doesn't have a def_cfa_register or def_cfa directive,
+        // update its outgoing register.
+        if (!Succ->hasDefRegister()) {
+          Succ->setOutgoingCFARegister(Succ->getIncomingCFARegister());
+          ChangedOutgoingInfo = true;
+        }
+      }
+      // If Succ's outgoing CFI info has been changed, it's successors should be
+      // updated as well.
+      if (ChangedOutgoingInfo) {
+        Successors.push(Succ);
+        ProcessedMBBs.insert(Succ);
+      }
+    }
+  }
+}
+
+void MachineBasicBlock::recalculateCFIInfo(bool UseExistingIncoming,
+                                           int NewIncomingOffset,
+                                           unsigned NewIncomingRegister) {
+  // Outgoing cfa offset set by the block.
+  int SetOffset;
+  // Outgoing cfa register set by the block.
+  unsigned SetRegister;
+  const std::vector<MCCFIInstruction> &Instrs =
+      getParent()->getFrameInstructions();
+
+  // Set initial values to SetOffset and SetRegister. Use existing incoming
+  // values or values passed as arguments.
+  if (!UseExistingIncoming) {
+    // Set new incoming cfa offset and register values.
+    setIncomingCFAOffset(NewIncomingOffset);
+    setIncomingCFARegister(NewIncomingRegister);
+  }
+
+  SetOffset = getIncomingCFAOffset();
+  SetRegister = getIncomingCFARegister();
+
+  setDefOffset(false);
+  setDefRegister(false);
+
+  // Determine cfa offset and register set by the block.
+  for (MachineBasicBlock::iterator MI = begin(); MI != end(); ++MI) {
+    if (MI->isCFIInstruction()) {
+      unsigned CFIIndex = MI->getOperand(0).getCFIIndex();
+      const MCCFIInstruction &CFI = Instrs[CFIIndex];
+      if (CFI.getOperation() == MCCFIInstruction::OpDefCfaRegister) {
+        SetRegister = CFI.getRegister();
+        setDefRegister(true);
+      } else if (CFI.getOperation() == MCCFIInstruction::OpDefCfaOffset) {
+        SetOffset = CFI.getOffset();
+        setDefOffset(true);
+      } else if (CFI.getOperation() == MCCFIInstruction::OpAdjustCfaOffset) {
+        SetOffset = SetOffset + CFI.getOffset();
+      } else if (CFI.getOperation() == MCCFIInstruction::OpDefCfa) {
+        SetRegister = CFI.getRegister();
+        SetOffset = CFI.getOffset();
+        setDefOffset(true);
+        setDefRegister(true);
+      }
+    }
+  }
+
+  // Update outgoing CFI info.
+  setOutgoingCFAOffset(SetOffset);
+  setOutgoingCFARegister(SetRegister);
+}
+
+void MachineBasicBlock::mergeCFIInfo(MachineBasicBlock *MBB) {
+  // Update CFI info. This basic block acquires MBB's outgoing cfa offset and
+  // register values.
+  setOutgoingCFAOffset(MBB->getOutgoingCFAOffset());
+  setOutgoingCFARegister(MBB->getOutgoingCFARegister());
+  setDefOffset(hasDefOffset() || MBB->hasDefOffset());
+  setDefRegister(hasDefRegister() || MBB->hasDefRegister());
+}
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 81c6dace92e..01fb2a19779 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -305,8 +305,33 @@ bool MachineOperand::isIdenticalTo(const MachineOperand &Other) const {
   }
   case MachineOperand::MO_MCSymbol:
     return getMCSymbol() == Other.getMCSymbol();
-  case MachineOperand::MO_CFIIndex:
-    return getCFIIndex() == Other.getCFIIndex();
+  case MachineOperand::MO_CFIIndex: {
+    const MachineFunction *MF = getParent()->getParent()->getParent();
+    const MachineFunction *OtherMF =
+        Other.getParent()->getParent()->getParent();
+    MCCFIInstruction Inst = MF->getFrameInstructions()[getCFIIndex()];
+    MCCFIInstruction OtherInst =
+        OtherMF->getFrameInstructions()[Other.getCFIIndex()];
+    MCCFIInstruction::OpType op = Inst.getOperation();
+    if (op != OtherInst.getOperation()) return false;
+    if (op == MCCFIInstruction::OpDefCfa || op == MCCFIInstruction::OpOffset ||
+        op == MCCFIInstruction::OpRestore ||
+        op == MCCFIInstruction::OpUndefined ||
+        op == MCCFIInstruction::OpSameValue ||
+        op == MCCFIInstruction::OpDefCfaRegister ||
+        op == MCCFIInstruction::OpRelOffset ||
+        op == MCCFIInstruction::OpRegister)
+      if (Inst.getRegister() != OtherInst.getRegister()) return false;
+    if (op == MCCFIInstruction::OpRegister)
+      if (Inst.getRegister2() != OtherInst.getRegister2()) return false;
+    if (op == MCCFIInstruction::OpDefCfa || op == MCCFIInstruction::OpOffset ||
+        op == MCCFIInstruction::OpRelOffset ||
+        op == MCCFIInstruction::OpDefCfaOffset ||
+        op == MCCFIInstruction::OpAdjustCfaOffset ||
+        op == MCCFIInstruction::OpGnuArgsSize)
+      if (Inst.getOffset() != OtherInst.getOffset()) return false;
+    return true;
+  }
   case MachineOperand::MO_Metadata:
     return getMetadata() == Other.getMetadata();
   case MachineOperand::MO_IntrinsicID:
@@ -355,8 +380,13 @@ hash_code llvm::hash_value(const MachineOperand &MO) {
     return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getMetadata());
   case MachineOperand::MO_MCSymbol:
     return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getMCSymbol());
-  case MachineOperand::MO_CFIIndex:
-    return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getCFIIndex());
+  case MachineOperand::MO_CFIIndex: {
+    const MachineFunction *MF = MO.getParent()->getParent()->getParent();
+    MCCFIInstruction Inst = MF->getFrameInstructions()[MO.getCFIIndex()];
+    return hash_combine(MO.getType(), MO.getTargetFlags(), Inst.getOperation(),
+                        Inst.getRegister(), Inst.getRegister2(),
+                        Inst.getOffset());
+  }
   case MachineOperand::MO_IntrinsicID:
     return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getIntrinsicID());
   case MachineOperand::MO_Predicate:
diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
index e9f8d43fe64..d528b01cdff 100644
--- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -977,6 +977,10 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) {
 void PEI::insertPrologEpilogCode(MachineFunction &Fn) {
   const TargetFrameLowering &TFI = *Fn.getSubtarget().getFrameLowering();
 
+  // Set initial incoming and outgoing cfa offset and register values for basic
+  // blocks.
+  TFI.initializeCFIInfo(Fn);
+
   // Add prologue to the function...
   for (MachineBasicBlock *SaveBlock : SaveBlocks)
     TFI.emitPrologue(Fn, *SaveBlock);
diff --git a/llvm/lib/CodeGen/TailDuplicator.cpp b/llvm/lib/CodeGen/TailDuplicator.cpp
index dc7265dcf6c..62ce903535b 100644
--- a/llvm/lib/CodeGen/TailDuplicator.cpp
+++ b/llvm/lib/CodeGen/TailDuplicator.cpp
@@ -604,8 +604,8 @@ bool TailDuplicator::shouldTailDuplicate(bool IsSimple,
     if (PreRegAlloc && MI.isCall())
       return false;
 
-    if (!MI.isPHI() && !MI.isDebugValue())
-      InstrCount += 1;
+    if (!MI.isPHI() && !MI.isDirective())
+        InstrCount += 1;
 
     if (InstrCount > MaxDuplicateCount)
       return false;
@@ -857,6 +857,9 @@ bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
     for (MachineBasicBlock *Succ : TailBB->successors())
       PredBB->addSuccessor(Succ, MBPI->getEdgeProbability(TailBB, Succ));
 
+    // Update the CFI info for PredBB.
+    PredBB->mergeCFIInfo(TailBB);
+
     Changed = true;
     ++NumTailDups;
   }
@@ -917,6 +920,9 @@ bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
     PrevBB->transferSuccessors(TailBB);
     TDBBs.push_back(PrevBB);
     Changed = true;
+
+    // Update the CFI info for PrevBB.
+    PrevBB->mergeCFIInfo(TailBB);
   }
 
   // If this is after register allocation, there are no phis to fix.
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 817e58ce59e..b95ef25d7ee 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -769,8 +769,15 @@ void TargetPassConfig::addMachinePasses() {
   if (getOptLevel() != CodeGenOpt::None)
     addBlockPlacement();
 
+  // Verify basic block incoming and outgoing cfa offset and register values.
+  addPass(createCFIInfoVerifier());
+
   addPreEmitPass();
 
+  // Correct CFA calculation rule where needed by inserting appropriate CFI
+  // instructions.
+  addPass(createCFIInstrInserter(), false);
+
   if (TM->Options.EnableIPRA)
     // Collect register usage information and produce a register mask of
     // clobbered registers, to be used to optimize call sites.