10 files changed, 585 insertions, 14 deletions

diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index 53095497629..ed7e22d3eef 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -304,9 +304,9 @@ static unsigned ComputeCommonTailLength(MachineBasicBlock *MBB1,
   while (I1 != MBB1->begin() && I2 != MBB2->begin()) {
     --I1; --I2;
     // Skip debugging pseudos; necessary to avoid changing the code.
-    while (I1->isDebugValue()) {
+    while (I1->isDirective()) {
       if (I1==MBB1->begin()) {
-        while (I2->isDebugValue()) {
+        while (I2->isDirective()) {
           if (I2==MBB2->begin())
             // I1==DBG at begin; I2==DBG at begin
             return TailLen;
@@ -319,7 +319,7 @@ static unsigned ComputeCommonTailLength(MachineBasicBlock *MBB1,
       --I1;
     }
     // I1==first (untested) non-DBG preceding known match
-    while (I2->isDebugValue()) {
+    while (I2->isDirective()) {
       if (I2==MBB2->begin()) {
         ++I1;
         // I1==non-DBG, or first of DBGs not at begin; I2==DBG at begin
@@ -362,6 +362,35 @@ static unsigned ComputeCommonTailLength(MachineBasicBlock *MBB1,
     }
     ++I1;
   }
+
+  // Ensure that I1 and I2 do not point to a CFI_INSTRUCTION. This can happen if
+  // I1 and I2 are non-identical when compared and then one or both of them ends
+  // up pointing to a CFI instruction after being incremented. For example:
+  /*
+    BB1:
+    ...
+    INSTRUCTION_A
+    ADD32ri8  <- last common instruction
+    ...
+    BB2:
+    ...
+    INSTRUCTION_B
+    CFI_INSTRUCTION
+    ADD32ri8  <- last common instruction
+    ...
+  */
+  // When INSTRUCTION_A and INSTRUCTION_B are compared as not equal, after
+  // incrementing the iterators, I1 will point to ADD, however I2 will point to
+  // the CFI instruction. Later on, this leads to BB2 being 'hacked off' at the
+  // wrong place (in ReplaceTailWithBranchTo()) which results in losing this CFI
+  // instruction.
+  while (I1 != MBB1->end() && I1->isCFIInstruction()) {
+    ++I1;
+  }
+
+  while (I2 != MBB2->end() && I2->isCFIInstruction()) {
+    ++I2;
+  }
   return TailLen;
 }
 
@@ -417,6 +446,14 @@ MachineBasicBlock *BranchFolder::SplitMBBAt(MachineBasicBlock &CurMBB,
     FuncletMembership[NewMBB] = n;
   }
 
+  // Recalculate CFI info for CurMBB. Use existing incoming cfa offset and
+  // register.
+  CurMBB.recalculateCFIInfo(true);
+  // Recalculate CFI info for NewMBB. Use CurMBB's outgoing cfa offset and
+  // register as NewMBB's incoming.
+  NewMBB->recalculateCFIInfo(false, CurMBB.getOutgoingCFAOffset(),
+                             CurMBB.getOutgoingCFARegister());
+
   return NewMBB;
 }
 
@@ -426,7 +463,7 @@ static unsigned EstimateRuntime(MachineBasicBlock::iterator I,
                                 MachineBasicBlock::iterator E) {
   unsigned Time = 0;
   for (; I != E; ++I) {
-    if (I->isDebugValue())
+    if (I->isDirective())
       continue;
     if (I->isCall())
       Time += 10;
@@ -780,7 +817,7 @@ void BranchFolder::MergeCommonTailDebugLocs(unsigned commonTailIndex) {
   }
 
   for (auto &MI : *MBB) {
-    if (MI.isDebugValue())
+    if (MI.isDirective())
       continue;
     DebugLoc DL = MI.getDebugLoc();
     for (unsigned int i = 0 ; i < NextCommonInsts.size() ; i++) {
@@ -790,7 +827,7 @@ void BranchFolder::MergeCommonTailDebugLocs(unsigned commonTailIndex) {
       auto &Pos = NextCommonInsts[i];
       assert(Pos != SameTails[i].getBlock()->end() &&
           "Reached BB end within common tail");
-      while (Pos->isDebugValue()) {
+      while (Pos->isDirective()) {
         ++Pos;
         assert(Pos != SameTails[i].getBlock()->end() &&
             "Reached BB end within common tail");
@@ -823,12 +860,12 @@ mergeOperations(MachineBasicBlock::iterator MBBIStartPos,
     assert(MBBI != MBBIE && "Reached BB end within common tail length!");
     (void)MBBIE;
 
-    if (MBBI->isDebugValue()) {
+    if (MBBI->isDirective()) {
       ++MBBI;
       continue;
     }
 
-    while ((MBBICommon != MBBIECommon) && MBBICommon->isDebugValue())
+    while ((MBBICommon != MBBIECommon) && MBBICommon->isDirective())
       ++MBBICommon;
 
     assert(MBBICommon != MBBIECommon &&
@@ -971,6 +1008,11 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB,
       mergeOperations(SameTails[i].getTailStartPos(), *MBB);
       // Hack the end off BB i, making it jump to BB commonTailIndex instead.
       ReplaceTailWithBranchTo(SameTails[i].getTailStartPos(), MBB);
+
+      // Recalculate CFI info for BB. Use existing incoming cfa offset and
+      // register.
+      SameTails[i].getBlock()->recalculateCFIInfo(true);
+
       // BB i is no longer a predecessor of SuccBB; remove it from the worklist.
       MergePotentials.erase(SameTails[i].getMPIter());
     }
@@ -1381,6 +1423,10 @@ ReoptimizeBlock:
       assert(PrevBB.succ_empty());
       PrevBB.transferSuccessors(MBB);
       MadeChange = true;
+
+      // Update CFI info for PrevBB.
+      PrevBB.mergeCFIInfo(MBB);
+
       return MadeChange;
     }
 
diff --git a/llvm/lib/CodeGen/CFIInfoVerifier.cpp b/llvm/lib/CodeGen/CFIInfoVerifier.cpp
new file mode 100644
index 00000000000..ff5a953c242
--- /dev/null
+++ b/llvm/lib/CodeGen/CFIInfoVerifier.cpp
@@ -0,0 +1,123 @@
+//===----------- CFIInfoVerifier.cpp - CFI Information Verifier -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass verifies incoming and outgoing CFI information of basic blocks. CFI
+// information is information about offset and register set by CFI directives,
+// valid at the start and end of a basic block. This pass checks that outgoing
+// information of predecessors matches incoming information of their successors.
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/Target/TargetMachine.h"
+using namespace llvm;
+
+namespace {
+class CFIInfoVerifier : public MachineFunctionPass {
+ public:
+  static char ID;
+
+  CFIInfoVerifier() : MachineFunctionPass(ID) {
+    initializeCFIInfoVerifierPass(*PassRegistry::getPassRegistry());
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    bool NeedsDwarfCFI = (MF.getMMI().hasDebugInfo() ||
+                          MF.getFunction()->needsUnwindTableEntry()) &&
+                         (!MF.getTarget().getTargetTriple().isOSDarwin() &&
+                          !MF.getTarget().getTargetTriple().isOSWindows());
+    if (!NeedsDwarfCFI) return false;
+    verify(MF);
+    return false;
+  }
+
+ private:
+  // Go through each MBB in a function and check that outgoing offset and
+  // register of its predecessors match incoming offset and register of that
+  // MBB, as well as that incoming offset and register of its successors match
+  // outgoing offset and register of the MBB.
+  void verify(MachineFunction &MF);
+  void report(const char *msg, MachineBasicBlock &MBB);
+};
+}
+
+char CFIInfoVerifier::ID = 0;
+INITIALIZE_PASS(CFIInfoVerifier, "cfiinfoverifier",
+                "Verify that corresponding in/out CFI info matches", false,
+                false)
+FunctionPass *llvm::createCFIInfoVerifier() { return new CFIInfoVerifier(); }
+
+void CFIInfoVerifier::verify(MachineFunction &MF) {
+  for (auto &CurrMBB : MF) {
+    for (auto Pred : CurrMBB.predecessors()) {
+      // Check that outgoing offset values of predecessors match the incoming
+      // offset value of CurrMBB
+      if (Pred->getOutgoingCFAOffset() != CurrMBB.getIncomingCFAOffset()) {
+        report("The outgoing offset of a predecessor is inconsistent.",
+               CurrMBB);
+        errs() << "Predecessor BB#" << Pred->getNumber()
+               << " has outgoing offset (" << Pred->getOutgoingCFAOffset()
+               << "), while BB#" << CurrMBB.getNumber()
+               << " has incoming offset (" << CurrMBB.getIncomingCFAOffset()
+               << ").\n";
+      }
+      // Check that outgoing register values of predecessors match the incoming
+      // register value of CurrMBB
+      if (Pred->getOutgoingCFARegister() != CurrMBB.getIncomingCFARegister()) {
+        report("The outgoing register of a predecessor is inconsistent.",
+               CurrMBB);
+        errs() << "Predecessor BB#" << Pred->getNumber()
+               << " has outgoing register (" << Pred->getOutgoingCFARegister()
+               << "), while BB#" << CurrMBB.getNumber()
+               << " has incoming register (" << CurrMBB.getIncomingCFARegister()
+               << ").\n";
+      }
+    }
+
+    for (auto Succ : CurrMBB.successors()) {
+      // Check that incoming offset values of successors match the outgoing
+      // offset value of CurrMBB
+      if (Succ->getIncomingCFAOffset() != CurrMBB.getOutgoingCFAOffset()) {
+        report("The incoming offset of a successor is inconsistent.", CurrMBB);
+        errs() << "Successor BB#" << Succ->getNumber()
+               << " has incoming offset (" << Succ->getIncomingCFAOffset()
+               << "), while BB#" << CurrMBB.getNumber()
+               << " has outgoing offset (" << CurrMBB.getOutgoingCFAOffset()
+               << ").\n";
+      }
+      // Check that incoming register values of successors match the outgoing
+      // register value of CurrMBB
+      if (Succ->getIncomingCFARegister() != CurrMBB.getOutgoingCFARegister()) {
+        report("The incoming register of a successor is inconsistent.",
+               CurrMBB);
+        errs() << "Successor BB#" << Succ->getNumber()
+               << " has incoming register (" << Succ->getIncomingCFARegister()
+               << "), while BB#" << CurrMBB.getNumber()
+               << " has outgoing register (" << CurrMBB.getOutgoingCFARegister()
+               << ").\n";
+      }
+    }
+  }
+}
+
+void CFIInfoVerifier::report(const char *msg, MachineBasicBlock &MBB) {
+  assert(&MBB);
+  errs() << '\n';
+  errs() << "*** " << msg << " ***\n"
+         << "- function:    " << MBB.getParent()->getName() << "\n";
+  errs() << "- basic block: BB#" << MBB.getNumber() << ' ' << MBB.getName()
+         << " (" << (const void *)&MBB << ')';
+  errs() << '\n';
+}
diff --git a/llvm/lib/CodeGen/CFIInstrInserter.cpp b/llvm/lib/CodeGen/CFIInstrInserter.cpp
new file mode 100644
index 00000000000..b83c957d7c4
--- /dev/null
+++ b/llvm/lib/CodeGen/CFIInstrInserter.cpp
@@ -0,0 +1,124 @@
+//===------ CFIInstrInserter.cpp - Insert additional CFI instructions -----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Insert CFI instructions at the beginnings of basic blocks if needed. CFI
+// instructions are inserted if basic blocks have incorrect offset or register
+// set by prevoius blocks.
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
+using namespace llvm;
+
+namespace {
+class CFIInstrInserter : public MachineFunctionPass {
+ public:
+  CFIInstrInserter() : MachineFunctionPass(ID) {
+    initializeCFIInstrInserterPass(*PassRegistry::getPassRegistry());
+  }
+  bool runOnMachineFunction(MachineFunction &MF) override;
+  static char ID;
+
+ private:
+  StringRef getPassName() const override { return "CFI Instruction Inserter"; }
+
+  // Check if incoming CFI information of a basic block matches outgoing CFI
+  // information of the previous block. If it doesn't, insert CFI instruction at
+  // the beginning of the block that corrects the CFA calculation rule for that
+  // block.
+  void CorrectCFA(MachineFunction &MF);
+
+  // Return the cfa offset value that should be set at the beginning of MBB if
+  // needed. The negated value is needed when creating CFI instructions that set
+  // absolute offset.
+  int getCorrectCFAOffset(MachineBasicBlock &MBB) {
+    return -MBB.getIncomingCFAOffset();
+  }
+
+  // Were any CFI instructions inserted
+  bool InsertedCFIInstr = false;
+};
+}
+
+char CFIInstrInserter::ID = 0;
+INITIALIZE_PASS(CFIInstrInserter, "cfiinstrinserter",
+                "Check CFI info and insert CFI instructions if needed", false,
+                false)
+
+FunctionPass *llvm::createCFIInstrInserter() { return new CFIInstrInserter(); }
+
+bool CFIInstrInserter::runOnMachineFunction(MachineFunction &MF) {
+  bool NeedsDwarfCFI = (MF.getMMI().hasDebugInfo() ||
+                        MF.getFunction()->needsUnwindTableEntry()) &&
+                       (!MF.getTarget().getTargetTriple().isOSDarwin() &&
+                        !MF.getTarget().getTargetTriple().isOSWindows());
+
+  if (!NeedsDwarfCFI) return false;
+
+  // Insert appropriate CFI instructions for each MBB if CFA calculation rule
+  // needs to be corrected for that MBB.
+  CorrectCFA(MF);
+
+  return InsertedCFIInstr;
+}
+
+void CFIInstrInserter::CorrectCFA(MachineFunction &MF) {
+
+  MachineBasicBlock &FirstMBB = MF.front();
+  MachineBasicBlock *PrevMBB = &FirstMBB;
+  const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
+  InsertedCFIInstr = false;
+
+  for (auto &MBB : MF) {
+    // Skip the first MBB in a function
+    if (MBB.getNumber() == FirstMBB.getNumber()) continue;
+
+    auto MBBI = MBB.begin();
+    DebugLoc DL = MBB.findDebugLoc(MBBI);
+
+    if (PrevMBB->getOutgoingCFAOffset() != MBB.getIncomingCFAOffset()) {
+      // If both outgoing offset and register of a previous block don't match
+      // incoming offset and register of this block, add a def_cfa instruction
+      // with the correct offset and register for this block.
+      if (PrevMBB->getOutgoingCFARegister() != MBB.getIncomingCFARegister()) {
+        unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
+            nullptr, MBB.getIncomingCFARegister(), getCorrectCFAOffset(MBB)));
+        BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+            .addCFIIndex(CFIIndex);
+        // If outgoing offset of a previous block doesn't match incoming offset
+        // of this block, add a def_cfa_offset instruction with the correct
+        // offset for this block.
+      } else {
+        unsigned CFIIndex =
+            MF.addFrameInst(MCCFIInstruction::createDefCfaOffset(
+                nullptr, getCorrectCFAOffset(MBB)));
+        BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+            .addCFIIndex(CFIIndex);
+      }
+      InsertedCFIInstr = true;
+      // If outgoing register of a previous block doesn't match incoming
+      // register of this block, add a def_cfa_register instruction with the
+      // correct register for this block.
+    } else if (PrevMBB->getOutgoingCFARegister() !=
+               MBB.getIncomingCFARegister()) {
+      unsigned CFIIndex =
+          MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(
+              nullptr, MBB.getIncomingCFARegister()));
+      BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+          .addCFIIndex(CFIIndex);
+      InsertedCFIInstr = true;
+    }
+    PrevMBB = &MBB;
+  }
+}
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index 7f3c6da9126..be6dcbfee0a 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -10,6 +10,8 @@ add_llvm_library(LLVMCodeGen
   BuiltinGCs.cpp
   CalcSpillWeights.cpp
   CallingConvLower.cpp
+  CFIInfoVerifier.cpp
+  CFIInstrInserter.cpp
   CodeGen.cpp
   CodeGenPrepare.cpp
   CountingFunctionInserter.cpp
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index faa5f139cf7..453e17e575d 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -24,6 +24,8 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeBranchCoalescingPass(Registry);
   initializeBranchFolderPassPass(Registry);
   initializeBranchRelaxationPass(Registry);
+  initializeCFIInfoVerifierPass(Registry);
+  initializeCFIInstrInserterPass(Registry);
   initializeCodeGenPreparePass(Registry);
   initializeCountingFunctionInserterPass(Registry);
   initializeDeadMachineInstructionElimPass(Registry);
diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp
index 81597afe6b0..c9976d7f2ff 100644
--- a/llvm/lib/CodeGen/MachineBasicBlock.cpp
+++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp
@@ -35,6 +35,8 @@
 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
 #include <algorithm>
+#include <queue>
+#include <set>
 using namespace llvm;
 
 #define DEBUG_TYPE "codegen"
@@ -1343,3 +1345,228 @@ MachineBasicBlock::livein_iterator MachineBasicBlock::livein_begin() const {
       "Liveness information is accurate");
   return LiveIns.begin();
 }
+
+void MachineBasicBlock::updateCFIInfo(MachineBasicBlock::iterator Pos) {
+  // Used for calculating outgoing cfa offset when CFI instruction added at Pos
+  // is def_cfa or def_cfa_offset.
+  /* For example:
+     ...
+     .cfi_adjust_cfa_offset 4
+     ...
+     .cfi_adjust_cfa_offset 4
+     ...
+     .cfi_def_cfa_offset 16  <---- newly added CFI instruction at Pos
+     ...
+     .cfi_adjust_cfa_offset 4
+     ...
+     Once def_cfa_offset is inserted, outgoing cfa offset is no longer
+     calculated as incoming offset incremented by the sum of all adjustments
+     (12). It becomes equal to the offset set by the added CFI instruction (16)
+     incremented by the sum of adjustments below it (4). Adjustments above the
+     added def_cfa_offset directive don't have effect below it anymore and
+     therefore don't affect the value of outgoing cfa offset.
+  */
+  int AdjustAmount = 0;
+  // Used to check if outgoing cfa offset should be updated or not (when def_cfa
+  // is inserted).
+  bool ShouldSetOffset = true;
+  // Used to check if outgoing cfa register should be updated or not (when
+  // def_cfa is inserted).
+  bool ShouldSetRegister = true;
+  const std::vector<MCCFIInstruction> CFIInstructions =
+      getParent()->getFrameInstructions();
+  MCCFIInstruction CFI = CFIInstructions[Pos->getOperand(0).getCFIIndex()];
+  // Type of the CFI instruction that was inserted.
+  MCCFIInstruction::OpType CFIType = CFI.getOperation();
+
+  // Check if there are already existing CFI instructions below Pos and see if
+  // outgoing CFI info should be updated or not.
+  for (MachineBasicBlock::reverse_iterator RI = rbegin();
+       RI != Pos.getReverse(); ++RI) {
+    if (RI->isCFIInstruction()) {
+      MCCFIInstruction::OpType RIType =
+          CFIInstructions[RI->getOperand(0).getCFIIndex()].getOperation();
+      switch (RIType) {
+        case MCCFIInstruction::OpAdjustCfaOffset:
+          AdjustAmount +=
+              CFIInstructions[RI->getOperand(0).getCFIIndex()].getOffset();
+          break;
+        case MCCFIInstruction::OpDefCfaOffset:
+          // CFI instruction doesn't affect outgoing cfa offset if there is
+          // already a def_cfa_offset instruction below it.
+          if (CFIType == MCCFIInstruction::OpDefCfaOffset ||
+              CFIType == MCCFIInstruction::OpAdjustCfaOffset)
+            return;
+          if (CFIType == MCCFIInstruction::OpDefCfa) {
+            // CFI instruction doesn't affect outgoing cfa offset and register
+            // if there are both def_cfa_offset and def_cfa_register
+            // instructions below it.
+            if (!ShouldSetRegister) return;
+            ShouldSetOffset = false;
+          }
+          break;
+        case MCCFIInstruction::OpDefCfaRegister:
+          // CFI instruction doesn't affect outgoing cfa register if there is
+          // already a def_cfa_register instruction below it.
+          if (CFIType == MCCFIInstruction::OpDefCfaRegister) return;
+          if (CFIType == MCCFIInstruction::OpDefCfa) {
+            // CFI instruction doesn't affect outgoing cfa offset and register
+            // if there are both def_cfa_offset and def_cfa_register
+            // instructions below it.
+            if (!ShouldSetOffset) return;
+            ShouldSetRegister = false;
+          }
+          break;
+        case MCCFIInstruction::OpDefCfa:
+          // CFI instruction doesn't affect outgoing cfa offset and register if
+          // there is already a def_cfa instruction below it.
+          if (CFIType == MCCFIInstruction::OpDefCfaRegister ||
+              CFIType == MCCFIInstruction::OpDefCfaOffset ||
+              CFIType == MCCFIInstruction::OpDefCfa ||
+              CFIType == MCCFIInstruction::OpAdjustCfaOffset)
+            return;
+          break;
+        default:
+          break;
+      }
+    }
+  }
+
+  // Update the outgoing CFI info based on the added CFI instruction.
+  switch (CFIType) {
+    case MCCFIInstruction::OpAdjustCfaOffset:
+      setOutgoingCFAOffset(getOutgoingCFAOffset() + CFI.getOffset());
+      break;
+    case MCCFIInstruction::OpDefCfaOffset:
+      setOutgoingCFAOffset(CFI.getOffset() + AdjustAmount);
+      break;
+    case MCCFIInstruction::OpDefCfaRegister:
+      setOutgoingCFARegister(CFI.getRegister());
+      break;
+    case MCCFIInstruction::OpDefCfa:
+      if (ShouldSetOffset) setOutgoingCFAOffset(CFI.getOffset() + AdjustAmount);
+      if (ShouldSetRegister) setOutgoingCFARegister(CFI.getRegister());
+      break;
+    default:
+      break;
+  }
+}
+
+void MachineBasicBlock::updateCFIInfoSucc() {
+  // Blocks whose successors' CFI info should be updated.
+  std::queue<MachineBasicBlock *> Successors;
+  // Keep track of basic blocks that have already been put in the Successors
+  // queue.
+  std::set<MachineBasicBlock *> ProcessedMBBs;
+  // Start with updating CFI info for direct successors of this block.
+  Successors.push(this);
+  ProcessedMBBs.insert(this);
+
+  // Go through the successors and update their CFI info if needed.
+  while (!Successors.empty()) {
+    MachineBasicBlock *CurrSucc = Successors.front();
+    Successors.pop();
+
+    // Update CFI info for CurrSucc's successors.
+    for (auto Succ : CurrSucc->successors()) {
+      if (ProcessedMBBs.find(Succ) != ProcessedMBBs.end()) continue;
+      if (Succ->getIncomingCFAOffset() == CurrSucc->getOutgoingCFAOffset() &&
+          Succ->getIncomingCFARegister() == CurrSucc->getOutgoingCFARegister())
+        continue;
+      bool ChangedOutgoingInfo = false;
+      // Do not update cfa offset if the existing value matches the new.
+      if (Succ->getIncomingCFAOffset() != CurrSucc->getOutgoingCFAOffset()) {
+        // If the block doesn't have a def_cfa_offset or def_cfa directive,
+        // update its outgoing offset.
+        if (!Succ->hasDefOffset()) {
+          // Succ block doesn't set absolute offset, so the difference between
+          // outgoing and incoming offset remains the same. This difference is
+          // the sum of offsets set by adjust_cfa_offset directives.
+          int AdjustAmount =
+              Succ->getOutgoingCFAOffset() - Succ->getIncomingCFAOffset();
+          Succ->setOutgoingCFAOffset(CurrSucc->getOutgoingCFAOffset() +
+                                     AdjustAmount);
+          ChangedOutgoingInfo = true;
+        }
+        Succ->setIncomingCFAOffset(CurrSucc->getOutgoingCFAOffset());
+      }
+      // Do not update cfa register if the existing value matches the new.
+      if (Succ->getIncomingCFARegister() !=
+          CurrSucc->getOutgoingCFARegister()) {
+        Succ->setIncomingCFARegister(CurrSucc->getOutgoingCFARegister());
+        // If the block doesn't have a def_cfa_register or def_cfa directive,
+        // update its outgoing register.
+        if (!Succ->hasDefRegister()) {
+          Succ->setOutgoingCFARegister(Succ->getIncomingCFARegister());
+          ChangedOutgoingInfo = true;
+        }
+      }
+      // If Succ's outgoing CFI info has been changed, it's successors should be
+      // updated as well.
+      if (ChangedOutgoingInfo) {
+        Successors.push(Succ);
+        ProcessedMBBs.insert(Succ);
+      }
+    }
+  }
+}
+
+void MachineBasicBlock::recalculateCFIInfo(bool UseExistingIncoming,
+                                           int NewIncomingOffset,
+                                           unsigned NewIncomingRegister) {
+  // Outgoing cfa offset set by the block.
+  int SetOffset;
+  // Outgoing cfa register set by the block.
+  unsigned SetRegister;
+  const std::vector<MCCFIInstruction> &Instrs =
+      getParent()->getFrameInstructions();
+
+  // Set initial values to SetOffset and SetRegister. Use existing incoming
+  // values or values passed as arguments.
+  if (!UseExistingIncoming) {
+    // Set new incoming cfa offset and register values.
+    setIncomingCFAOffset(NewIncomingOffset);
+    setIncomingCFARegister(NewIncomingRegister);
+  }
+
+  SetOffset = getIncomingCFAOffset();
+  SetRegister = getIncomingCFARegister();
+
+  setDefOffset(false);
+  setDefRegister(false);
+
+  // Determine cfa offset and register set by the block.
+  for (MachineBasicBlock::iterator MI = begin(); MI != end(); ++MI) {
+    if (MI->isCFIInstruction()) {
+      unsigned CFIIndex = MI->getOperand(0).getCFIIndex();
+      const MCCFIInstruction &CFI = Instrs[CFIIndex];
+      if (CFI.getOperation() == MCCFIInstruction::OpDefCfaRegister) {
+        SetRegister = CFI.getRegister();
+        setDefRegister(true);
+      } else if (CFI.getOperation() == MCCFIInstruction::OpDefCfaOffset) {
+        SetOffset = CFI.getOffset();
+        setDefOffset(true);
+      } else if (CFI.getOperation() == MCCFIInstruction::OpAdjustCfaOffset) {
+        SetOffset = SetOffset + CFI.getOffset();
+      } else if (CFI.getOperation() == MCCFIInstruction::OpDefCfa) {
+        SetRegister = CFI.getRegister();
+        SetOffset = CFI.getOffset();
+        setDefOffset(true);
+        setDefRegister(true);
+      }
+    }
+  }
+
+  // Update outgoing CFI info.
+  setOutgoingCFAOffset(SetOffset);
+  setOutgoingCFARegister(SetRegister);
+}
+
+void MachineBasicBlock::mergeCFIInfo(MachineBasicBlock *MBB) {
+  // Update CFI info. This basic block acquires MBB's outgoing cfa offset and
+  // register values.
+  setOutgoingCFAOffset(MBB->getOutgoingCFAOffset());
+  setOutgoingCFARegister(MBB->getOutgoingCFARegister());
+  setDefOffset(hasDefOffset() || MBB->hasDefOffset());
+  setDefRegister(hasDefRegister() || MBB->hasDefRegister());
+}
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 81c6dace92e..01fb2a19779 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -305,8 +305,33 @@ bool MachineOperand::isIdenticalTo(const MachineOperand &Other) const {
   }
   case MachineOperand::MO_MCSymbol:
     return getMCSymbol() == Other.getMCSymbol();
-  case MachineOperand::MO_CFIIndex:
-    return getCFIIndex() == Other.getCFIIndex();
+  case MachineOperand::MO_CFIIndex: {
+    const MachineFunction *MF = getParent()->getParent()->getParent();
+    const MachineFunction *OtherMF =
+        Other.getParent()->getParent()->getParent();
+    MCCFIInstruction Inst = MF->getFrameInstructions()[getCFIIndex()];
+    MCCFIInstruction OtherInst =
+        OtherMF->getFrameInstructions()[Other.getCFIIndex()];
+    MCCFIInstruction::OpType op = Inst.getOperation();
+    if (op != OtherInst.getOperation()) return false;
+    if (op == MCCFIInstruction::OpDefCfa || op == MCCFIInstruction::OpOffset ||
+        op == MCCFIInstruction::OpRestore ||
+        op == MCCFIInstruction::OpUndefined ||
+        op == MCCFIInstruction::OpSameValue ||
+        op == MCCFIInstruction::OpDefCfaRegister ||
+        op == MCCFIInstruction::OpRelOffset ||
+        op == MCCFIInstruction::OpRegister)
+      if (Inst.getRegister() != OtherInst.getRegister()) return false;
+    if (op == MCCFIInstruction::OpRegister)
+      if (Inst.getRegister2() != OtherInst.getRegister2()) return false;
+    if (op == MCCFIInstruction::OpDefCfa || op == MCCFIInstruction::OpOffset ||
+        op == MCCFIInstruction::OpRelOffset ||
+        op == MCCFIInstruction::OpDefCfaOffset ||
+        op == MCCFIInstruction::OpAdjustCfaOffset ||
+        op == MCCFIInstruction::OpGnuArgsSize)
+      if (Inst.getOffset() != OtherInst.getOffset()) return false;
+    return true;
+  }
   case MachineOperand::MO_Metadata:
     return getMetadata() == Other.getMetadata();
   case MachineOperand::MO_IntrinsicID:
@@ -355,8 +380,13 @@ hash_code llvm::hash_value(const MachineOperand &MO) {
     return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getMetadata());
   case MachineOperand::MO_MCSymbol:
     return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getMCSymbol());
-  case MachineOperand::MO_CFIIndex:
-    return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getCFIIndex());
+  case MachineOperand::MO_CFIIndex: {
+    const MachineFunction *MF = MO.getParent()->getParent()->getParent();
+    MCCFIInstruction Inst = MF->getFrameInstructions()[MO.getCFIIndex()];
+    return hash_combine(MO.getType(), MO.getTargetFlags(), Inst.getOperation(),
+                        Inst.getRegister(), Inst.getRegister2(),
+                        Inst.getOffset());
+  }
   case MachineOperand::MO_IntrinsicID:
     return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getIntrinsicID());
   case MachineOperand::MO_Predicate:
diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
index e9f8d43fe64..d528b01cdff 100644
--- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -977,6 +977,10 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &Fn) {
 void PEI::insertPrologEpilogCode(MachineFunction &Fn) {
   const TargetFrameLowering &TFI = *Fn.getSubtarget().getFrameLowering();
 
+  // Set initial incoming and outgoing cfa offset and register values for basic
+  // blocks.
+  TFI.initializeCFIInfo(Fn);
+
   // Add prologue to the function...
   for (MachineBasicBlock *SaveBlock : SaveBlocks)
     TFI.emitPrologue(Fn, *SaveBlock);
diff --git a/llvm/lib/CodeGen/TailDuplicator.cpp b/llvm/lib/CodeGen/TailDuplicator.cpp
index dc7265dcf6c..62ce903535b 100644
--- a/llvm/lib/CodeGen/TailDuplicator.cpp
+++ b/llvm/lib/CodeGen/TailDuplicator.cpp
@@ -604,8 +604,8 @@ bool TailDuplicator::shouldTailDuplicate(bool IsSimple,
     if (PreRegAlloc && MI.isCall())
       return false;
 
-    if (!MI.isPHI() && !MI.isDebugValue())
-      InstrCount += 1;
+    if (!MI.isPHI() && !MI.isDirective())
+        InstrCount += 1;
 
     if (InstrCount > MaxDuplicateCount)
       return false;
@@ -857,6 +857,9 @@ bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
     for (MachineBasicBlock *Succ : TailBB->successors())
       PredBB->addSuccessor(Succ, MBPI->getEdgeProbability(TailBB, Succ));
 
+    // Update the CFI info for PredBB.
+    PredBB->mergeCFIInfo(TailBB);
+
     Changed = true;
     ++NumTailDups;
   }
@@ -917,6 +920,9 @@ bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
     PrevBB->transferSuccessors(TailBB);
     TDBBs.push_back(PrevBB);
     Changed = true;
+
+    // Update the CFI info for PrevBB.
+    PrevBB->mergeCFIInfo(TailBB);
   }
 
   // If this is after register allocation, there are no phis to fix.
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 817e58ce59e..b95ef25d7ee 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -769,8 +769,15 @@ void TargetPassConfig::addMachinePasses() {
   if (getOptLevel() != CodeGenOpt::None)
     addBlockPlacement();
 
+  // Verify basic block incoming and outgoing cfa offset and register values.
+  addPass(createCFIInfoVerifier());
+
   addPreEmitPass();
 
+  // Correct CFA calculation rule where needed by inserting appropriate CFI
+  // instructions.
+  addPass(createCFIInstrInserter(), false);
+
   if (TM->Options.EnableIPRA)
     // Collect register usage information and produce a register mask of
     // clobbered registers, to be used to optimize call sites.