[IR] Reformulate LLVM's EH funclet IR

While we have successfully implemented a funclet-oriented EH scheme on
top of LLVM IR, our scheme has some notable deficiencies:
- catchendpad and cleanupendpad are necessary in the current design
  but they are difficult to explain to others, even to seasoned LLVM
  experts.
- catchendpad and cleanupendpad are optimization barriers.  They cannot
  be split and force all potentially throwing call-sites to be invokes.
  This has a noticable effect on the quality of our code generation.
- catchpad, while similar in some aspects to invoke, is fairly awkward.
  It is unsplittable, starts a funclet, and has control flow to other
  funclets.
- The nesting relationship between funclets is currently a property of
  control flow edges.  Because of this, we are forced to carefully
  analyze the flow graph to see if there might potentially exist illegal
  nesting among funclets.  While we have logic to clone funclets when
  they are illegally nested, it would be nicer if we had a
  representation which forbade them upfront.

Let's clean this up a bit by doing the following:
- Instead, make catchpad more like cleanuppad and landingpad: no control
  flow, just a bunch of simple operands;  catchpad would be splittable.
- Introduce catchswitch, a control flow instruction designed to model
  the constraints of funclet oriented EH.
- Make funclet scoping explicit by having funclet instructions consume
  the token produced by the funclet which contains them.
- Remove catchendpad and cleanupendpad.  Their presence can be inferred
  implicitly using coloring information.

N.B.  The state numbering code for the CLR has been updated but the
veracity of it's output cannot be spoken for.  An expert should take a
look to make sure the results are reasonable.

Reviewers: rnk, JosephTremoulet, andrew.w.kaylor

Differential Revision: http://reviews.llvm.org/D15139

llvm-svn: 255422

5 files changed, 249 insertions, 356 deletions

diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index e41815aafa8..fbb27773c11 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -2890,19 +2890,36 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
 
       writeOperand(LPI->getClause(i), true);
     }
-  } else if (const auto *CPI = dyn_cast<CatchPadInst>(&I)) {
+  } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(&I)) {
+    Out << " within ";
+    writeOperand(CatchSwitch->getParentPad(), /*PrintType=*/false);
     Out << " [";
-    for (unsigned Op = 0, NumOps = CPI->getNumArgOperands(); Op < NumOps;
+    unsigned Op = 0;
+    for (const BasicBlock *PadBB : CatchSwitch->handlers()) {
+      if (Op > 0)
+        Out << ", ";
+      writeOperand(PadBB, /*PrintType=*/true);
+      ++Op;
+    }
+    Out << "] unwind ";
+    if (const BasicBlock *UnwindDest = CatchSwitch->getUnwindDest())
+      writeOperand(UnwindDest, /*PrintType=*/true);
+    else
+      Out << "to caller";
+  } else if (const auto *FPI = dyn_cast<FuncletPadInst>(&I)) {
+    Out << " within ";
+    writeOperand(FPI->getParentPad(), /*PrintType=*/false);
+    Out << " [";
+    for (unsigned Op = 0, NumOps = FPI->getNumArgOperands(); Op < NumOps;
          ++Op) {
       if (Op > 0)
         Out << ", ";
-      writeOperand(CPI->getArgOperand(Op), /*PrintType=*/true);
+      writeOperand(FPI->getArgOperand(Op), /*PrintType=*/true);
     }
-    Out << "]\n          to ";
-    writeOperand(CPI->getNormalDest(), /*PrintType=*/true);
-    Out << " unwind ";
-    writeOperand(CPI->getUnwindDest(), /*PrintType=*/true);
+    Out << ']';
   } else if (const auto *TPI = dyn_cast<TerminatePadInst>(&I)) {
+    Out << " within ";
+    writeOperand(TPI->getParentPad(), /*PrintType=*/false);
     Out << " [";
     for (unsigned Op = 0, NumOps = TPI->getNumArgOperands(); Op < NumOps;
          ++Op) {
@@ -2915,44 +2932,21 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
       writeOperand(TPI->getUnwindDest(), /*PrintType=*/true);
     else
       Out << "to caller";
-  } else if (const auto *CPI = dyn_cast<CleanupPadInst>(&I)) {
-    Out << " [";
-    for (unsigned Op = 0, NumOps = CPI->getNumOperands(); Op < NumOps; ++Op) {
-      if (Op > 0)
-        Out << ", ";
-      writeOperand(CPI->getOperand(Op), /*PrintType=*/true);
-    }
-    Out << "]";
   } else if (isa<ReturnInst>(I) && !Operand) {
     Out << " void";
   } else if (const auto *CRI = dyn_cast<CatchReturnInst>(&I)) {
-    Out << ' ';
-    writeOperand(CRI->getCatchPad(), /*PrintType=*/false);
+    Out << " from ";
+    writeOperand(CRI->getOperand(0), /*PrintType=*/false);
 
     Out << " to ";
-    writeOperand(CRI->getSuccessor(), /*PrintType=*/true);
+    writeOperand(CRI->getOperand(1), /*PrintType=*/true);
   } else if (const auto *CRI = dyn_cast<CleanupReturnInst>(&I)) {
-    Out << ' ';
-    writeOperand(CRI->getCleanupPad(), /*PrintType=*/false);
+    Out << " from ";
+    writeOperand(CRI->getOperand(0), /*PrintType=*/false);
 
     Out << " unwind ";
     if (CRI->hasUnwindDest())
-      writeOperand(CRI->getUnwindDest(), /*PrintType=*/true);
-    else
-      Out << "to caller";
-  } else if (const auto *CEPI = dyn_cast<CatchEndPadInst>(&I)) {
-    Out << " unwind ";
-    if (CEPI->hasUnwindDest())
-      writeOperand(CEPI->getUnwindDest(), /*PrintType=*/true);
-    else
-      Out << "to caller";
-  } else if (const auto *CEPI = dyn_cast<CleanupEndPadInst>(&I)) {
-    Out << ' ';
-    writeOperand(CEPI->getCleanupPad(), /*PrintType=*/false);
-
-    Out << " unwind ";
-    if (CEPI->hasUnwindDest())
-      writeOperand(CEPI->getUnwindDest(), /*PrintType=*/true);
+      writeOperand(CRI->getOperand(1), /*PrintType=*/true);
     else
       Out << "to caller";
   } else if (const CallInst *CI = dyn_cast<CallInst>(&I)) {
diff --git a/llvm/lib/IR/Dominators.cpp b/llvm/lib/IR/Dominators.cpp
index d94e31d4875..b9d4fb7de88 100644
--- a/llvm/lib/IR/Dominators.cpp
+++ b/llvm/lib/IR/Dominators.cpp
@@ -91,11 +91,11 @@ bool DominatorTree::dominates(const Instruction *Def,
   if (Def == User)
     return false;
 
-  // The value defined by an invoke/catchpad dominates an instruction only if
-  // it dominates every instruction in UseBB.
-  // A PHI is dominated only if the instruction dominates every possible use
-  // in the UseBB.
-  if (isa<InvokeInst>(Def) || isa<CatchPadInst>(Def) || isa<PHINode>(User))
+  // The value defined by an invoke dominates an instruction only if it
+  // dominates every instruction in UseBB.
+  // A PHI is dominated only if the instruction dominates every possible use in
+  // the UseBB.
+  if (isa<InvokeInst>(Def) || isa<PHINode>(User))
     return dominates(Def, UseBB);
 
   if (DefBB != UseBB)
@@ -126,18 +126,13 @@ bool DominatorTree::dominates(const Instruction *Def,
   if (DefBB == UseBB)
     return false;
 
-  // Invoke/CatchPad results are only usable in the normal destination, not in
-  // the exceptional destination.
+  // Invoke results are only usable in the normal destination, not in the
+  // exceptional destination.
   if (const auto *II = dyn_cast<InvokeInst>(Def)) {
     BasicBlock *NormalDest = II->getNormalDest();
     BasicBlockEdge E(DefBB, NormalDest);
     return dominates(E, UseBB);
   }
-  if (const auto *CPI = dyn_cast<CatchPadInst>(Def)) {
-    BasicBlock *NormalDest = CPI->getNormalDest();
-    BasicBlockEdge E(DefBB, NormalDest);
-    return dominates(E, UseBB);
-  }
 
   return dominates(DefBB, UseBB);
 }
@@ -239,8 +234,8 @@ bool DominatorTree::dominates(const Instruction *Def, const Use &U) const {
   if (!isReachableFromEntry(DefBB))
     return false;
 
-  // Invoke/CatchPad instructions define their return values on the edges
-  // to their normal successors, so we have to handle them specially.
+  // Invoke instructions define their return values on the edges to their normal
+  // successors, so we have to handle them specially.
   // Among other things, this means they don't dominate anything in
   // their own block, except possibly a phi, so we don't need to
   // walk the block in any case.
@@ -249,11 +244,6 @@ bool DominatorTree::dominates(const Instruction *Def, const Use &U) const {
     BasicBlockEdge E(DefBB, NormalDest);
     return dominates(E, U);
   }
-  if (const auto *CPI = dyn_cast<CatchPadInst>(Def)) {
-    BasicBlock *NormalDest = CPI->getNormalDest();
-    BasicBlockEdge E(DefBB, NormalDest);
-    return dominates(E, U);
-  }
 
   // If the def and use are in different blocks, do a simple CFG dominator
   // tree query.
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index 7bd50328b12..ce2e1d8c02b 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -202,11 +202,10 @@ const char *Instruction::getOpcodeName(unsigned OpCode) {
   case Invoke: return "invoke";
   case Resume: return "resume";
   case Unreachable: return "unreachable";
-  case CleanupEndPad: return "cleanupendpad";
   case CleanupRet: return "cleanupret";
-  case CatchEndPad: return "catchendpad";
   case CatchRet: return "catchret";
   case CatchPad: return "catchpad";
+  case CatchSwitch: return "catchswitch";
   case TerminatePad: return "terminatepad";
 
   // Standard binary operators...
@@ -476,10 +475,8 @@ bool Instruction::mayThrow() const {
     return !CI->doesNotThrow();
   if (const auto *CRI = dyn_cast<CleanupReturnInst>(this))
     return CRI->unwindsToCaller();
-  if (const auto *CEPI = dyn_cast<CleanupEndPadInst>(this))
-    return CEPI->unwindsToCaller();
-  if (const auto *CEPI = dyn_cast<CatchEndPadInst>(this))
-    return CEPI->unwindsToCaller();
+  if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(this))
+    return CatchSwitch->unwindsToCaller();
   if (const auto *TPI = dyn_cast<TerminatePadInst>(this))
     return TPI->unwindsToCaller();
   return isa<ResumeInst>(this);
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 11c55865135..82eb1e0f2f7 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -764,61 +764,6 @@ BasicBlock *ResumeInst::getSuccessorV(unsigned idx) const {
 }
 
 //===----------------------------------------------------------------------===//
-//                        CleanupEndPadInst Implementation
-//===----------------------------------------------------------------------===//
-
-CleanupEndPadInst::CleanupEndPadInst(const CleanupEndPadInst &CEPI)
-    : TerminatorInst(CEPI.getType(), Instruction::CleanupEndPad,
-                     OperandTraits<CleanupEndPadInst>::op_end(this) -
-                         CEPI.getNumOperands(),
-                     CEPI.getNumOperands()) {
-  setInstructionSubclassData(CEPI.getSubclassDataFromInstruction());
-  setCleanupPad(CEPI.getCleanupPad());
-  if (BasicBlock *UnwindDest = CEPI.getUnwindDest())
-    setUnwindDest(UnwindDest);
-}
-
-void CleanupEndPadInst::init(CleanupPadInst *CleanupPad, BasicBlock *UnwindBB) {
-  setCleanupPad(CleanupPad);
-  if (UnwindBB) {
-    setInstructionSubclassData(getSubclassDataFromInstruction() | 1);
-    setUnwindDest(UnwindBB);
-  }
-}
-
-CleanupEndPadInst::CleanupEndPadInst(CleanupPadInst *CleanupPad,
-                                     BasicBlock *UnwindBB, unsigned Values,
-                                     Instruction *InsertBefore)
-    : TerminatorInst(Type::getVoidTy(CleanupPad->getContext()),
-                     Instruction::CleanupEndPad,
-                     OperandTraits<CleanupEndPadInst>::op_end(this) - Values,
-                     Values, InsertBefore) {
-  init(CleanupPad, UnwindBB);
-}
-
-CleanupEndPadInst::CleanupEndPadInst(CleanupPadInst *CleanupPad,
-                                     BasicBlock *UnwindBB, unsigned Values,
-                                     BasicBlock *InsertAtEnd)
-    : TerminatorInst(Type::getVoidTy(CleanupPad->getContext()),
-                     Instruction::CleanupEndPad,
-                     OperandTraits<CleanupEndPadInst>::op_end(this) - Values,
-                     Values, InsertAtEnd) {
-  init(CleanupPad, UnwindBB);
-}
-
-BasicBlock *CleanupEndPadInst::getSuccessorV(unsigned Idx) const {
-  assert(Idx == 0);
-  return getUnwindDest();
-}
-unsigned CleanupEndPadInst::getNumSuccessorsV() const {
-  return getNumSuccessors();
-}
-void CleanupEndPadInst::setSuccessorV(unsigned Idx, BasicBlock *B) {
-  assert(Idx == 0);
-  setUnwindDest(B);
-}
-
-//===----------------------------------------------------------------------===//
 //                        CleanupReturnInst Implementation
 //===----------------------------------------------------------------------===//
 
@@ -828,23 +773,22 @@ CleanupReturnInst::CleanupReturnInst(const CleanupReturnInst &CRI)
                          CRI.getNumOperands(),
                      CRI.getNumOperands()) {
   setInstructionSubclassData(CRI.getSubclassDataFromInstruction());
-  Op<-1>() = CRI.Op<-1>();
+  Op<0>() = CRI.Op<0>();
   if (CRI.hasUnwindDest())
-    Op<-2>() = CRI.Op<-2>();
+    Op<1>() = CRI.Op<1>();
 }
 
-void CleanupReturnInst::init(CleanupPadInst *CleanupPad, BasicBlock *UnwindBB) {
+void CleanupReturnInst::init(Value *CleanupPad, BasicBlock *UnwindBB) {
   if (UnwindBB)
     setInstructionSubclassData(getSubclassDataFromInstruction() | 1);
 
-  Op<-1>() = CleanupPad;
+  Op<0>() = CleanupPad;
   if (UnwindBB)
-    Op<-2>() = UnwindBB;
+    Op<1>() = UnwindBB;
 }
 
-CleanupReturnInst::CleanupReturnInst(CleanupPadInst *CleanupPad,
-                                     BasicBlock *UnwindBB, unsigned Values,
-                                     Instruction *InsertBefore)
+CleanupReturnInst::CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB,
+                                     unsigned Values, Instruction *InsertBefore)
     : TerminatorInst(Type::getVoidTy(CleanupPad->getContext()),
                      Instruction::CleanupRet,
                      OperandTraits<CleanupReturnInst>::op_end(this) - Values,
@@ -852,9 +796,8 @@ CleanupReturnInst::CleanupReturnInst(CleanupPadInst *CleanupPad,
   init(CleanupPad, UnwindBB);
 }
 
-CleanupReturnInst::CleanupReturnInst(CleanupPadInst *CleanupPad,
-                                     BasicBlock *UnwindBB, unsigned Values,
-                                     BasicBlock *InsertAtEnd)
+CleanupReturnInst::CleanupReturnInst(Value *CleanupPad, BasicBlock *UnwindBB,
+                                     unsigned Values, BasicBlock *InsertAtEnd)
     : TerminatorInst(Type::getVoidTy(CleanupPad->getContext()),
                      Instruction::CleanupRet,
                      OperandTraits<CleanupReturnInst>::op_end(this) - Values,
@@ -875,58 +818,9 @@ void CleanupReturnInst::setSuccessorV(unsigned Idx, BasicBlock *B) {
 }
 
 //===----------------------------------------------------------------------===//
-//                        CatchEndPadInst Implementation
-//===----------------------------------------------------------------------===//
-
-CatchEndPadInst::CatchEndPadInst(const CatchEndPadInst &CRI)
-    : TerminatorInst(CRI.getType(), Instruction::CatchEndPad,
-                     OperandTraits<CatchEndPadInst>::op_end(this) -
-                         CRI.getNumOperands(),
-                     CRI.getNumOperands()) {
-  setInstructionSubclassData(CRI.getSubclassDataFromInstruction());
-  if (BasicBlock *UnwindDest = CRI.getUnwindDest())
-    setUnwindDest(UnwindDest);
-}
-
-void CatchEndPadInst::init(BasicBlock *UnwindBB) {
-  if (UnwindBB) {
-    setInstructionSubclassData(getSubclassDataFromInstruction() | 1);
-    setUnwindDest(UnwindBB);
-  }
-}
-
-CatchEndPadInst::CatchEndPadInst(LLVMContext &C, BasicBlock *UnwindBB,
-                                 unsigned Values, Instruction *InsertBefore)
-    : TerminatorInst(Type::getVoidTy(C), Instruction::CatchEndPad,
-                     OperandTraits<CatchEndPadInst>::op_end(this) - Values,
-                     Values, InsertBefore) {
-  init(UnwindBB);
-}
-
-CatchEndPadInst::CatchEndPadInst(LLVMContext &C, BasicBlock *UnwindBB,
-                                 unsigned Values, BasicBlock *InsertAtEnd)
-    : TerminatorInst(Type::getVoidTy(C), Instruction::CatchEndPad,
-                     OperandTraits<CatchEndPadInst>::op_end(this) - Values,
-                     Values, InsertAtEnd) {
-  init(UnwindBB);
-}
-
-BasicBlock *CatchEndPadInst::getSuccessorV(unsigned Idx) const {
-  assert(Idx == 0);
-  return getUnwindDest();
-}
-unsigned CatchEndPadInst::getNumSuccessorsV() const {
-  return getNumSuccessors();
-}
-void CatchEndPadInst::setSuccessorV(unsigned Idx, BasicBlock *B) {
-  assert(Idx == 0);
-  setUnwindDest(B);
-}
-
-//===----------------------------------------------------------------------===//
 //                        CatchReturnInst Implementation
 //===----------------------------------------------------------------------===//
-void CatchReturnInst::init(CatchPadInst *CatchPad, BasicBlock *BB) {
+void CatchReturnInst::init(Value *CatchPad, BasicBlock *BB) {
   Op<0>() = CatchPad;
   Op<1>() = BB;
 }
@@ -938,7 +832,7 @@ CatchReturnInst::CatchReturnInst(const CatchReturnInst &CRI)
   Op<1>() = CRI.Op<1>();
 }
 
-CatchReturnInst::CatchReturnInst(CatchPadInst *CatchPad, BasicBlock *BB,
+CatchReturnInst::CatchReturnInst(Value *CatchPad, BasicBlock *BB,
                                  Instruction *InsertBefore)
     : TerminatorInst(Type::getVoidTy(BB->getContext()), Instruction::CatchRet,
                      OperandTraits<CatchReturnInst>::op_begin(this), 2,
@@ -946,7 +840,7 @@ CatchReturnInst::CatchReturnInst(CatchPadInst *CatchPad, BasicBlock *BB,
   init(CatchPad, BB);
 }
 
-CatchReturnInst::CatchReturnInst(CatchPadInst *CatchPad, BasicBlock *BB,
+CatchReturnInst::CatchReturnInst(Value *CatchPad, BasicBlock *BB,
                                  BasicBlock *InsertAtEnd)
     : TerminatorInst(Type::getVoidTy(BB->getContext()), Instruction::CatchRet,
                      OperandTraits<CatchReturnInst>::op_begin(this), 2,
@@ -967,64 +861,136 @@ void CatchReturnInst::setSuccessorV(unsigned Idx, BasicBlock *B) {
 }
 
 //===----------------------------------------------------------------------===//
-//                        CatchPadInst Implementation
+//                       CatchSwitchInst Implementation
 //===----------------------------------------------------------------------===//
-void CatchPadInst::init(BasicBlock *IfNormal, BasicBlock *IfException,
-                        ArrayRef<Value *> Args, const Twine &NameStr) {
-  assert(getNumOperands() == 2 + Args.size() && "NumOperands not set up?");
-  Op<-2>() = IfNormal;
-  Op<-1>() = IfException;
-  std::copy(Args.begin(), Args.end(), op_begin());
+
+CatchSwitchInst::CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest,
+                                 unsigned NumReservedValues,
+                                 const Twine &NameStr,
+                                 Instruction *InsertBefore)
+    : TerminatorInst(ParentPad->getType(), Instruction::CatchSwitch, nullptr, 0,
+                     InsertBefore) {
+  if (UnwindDest)
+    ++NumReservedValues;
+  init(ParentPad, UnwindDest, NumReservedValues + 1);
   setName(NameStr);
 }
 
-CatchPadInst::CatchPadInst(const CatchPadInst &CPI)
-    : TerminatorInst(CPI.getType(), Instruction::CatchPad,
-                     OperandTraits<CatchPadInst>::op_end(this) -
-                         CPI.getNumOperands(),
-                     CPI.getNumOperands()) {
-  std::copy(CPI.op_begin(), CPI.op_end(), op_begin());
+CatchSwitchInst::CatchSwitchInst(Value *ParentPad, BasicBlock *UnwindDest,
+                                 unsigned NumReservedValues,
+                                 const Twine &NameStr, BasicBlock *InsertAtEnd)
+    : TerminatorInst(ParentPad->getType(), Instruction::CatchSwitch, nullptr, 0,
+                     InsertAtEnd) {
+  if (UnwindDest)
+    ++NumReservedValues;
+  init(ParentPad, UnwindDest, NumReservedValues + 1);
+  setName(NameStr);
 }
 
-CatchPadInst::CatchPadInst(BasicBlock *IfNormal, BasicBlock *IfException,
-                           ArrayRef<Value *> Args, unsigned Values,
-                           const Twine &NameStr, Instruction *InsertBefore)
-    : TerminatorInst(Type::getTokenTy(IfNormal->getContext()),
-                     Instruction::CatchPad,
-                     OperandTraits<CatchPadInst>::op_end(this) - Values, Values,
-                     InsertBefore) {
-  init(IfNormal, IfException, Args, NameStr);
+CatchSwitchInst::CatchSwitchInst(const CatchSwitchInst &CSI)
+    : TerminatorInst(CSI.getType(), Instruction::CatchSwitch, nullptr,
+                     CSI.getNumOperands()) {
+  init(CSI.getParentPad(), CSI.getUnwindDest(), CSI.getNumOperands());
+  setNumHungOffUseOperands(ReservedSpace);
+  Use *OL = getOperandList();
+  const Use *InOL = CSI.getOperandList();
+  for (unsigned I = 1, E = ReservedSpace; I != E; ++I)
+    OL[I] = InOL[I];
 }
 
-CatchPadInst::CatchPadInst(BasicBlock *IfNormal, BasicBlock *IfException,
-                           ArrayRef<Value *> Args, unsigned Values,
-                           const Twine &NameStr, BasicBlock *InsertAtEnd)
-    : TerminatorInst(Type::getTokenTy(IfNormal->getContext()),
-                     Instruction::CatchPad,
-                     OperandTraits<CatchPadInst>::op_end(this) - Values, Values,
-                     InsertAtEnd) {
-  init(IfNormal, IfException, Args, NameStr);
+void CatchSwitchInst::init(Value *ParentPad, BasicBlock *UnwindDest,
+                           unsigned NumReservedValues) {
+  assert(ParentPad && NumReservedValues);
+
+  ReservedSpace = NumReservedValues;
+  setNumHungOffUseOperands(UnwindDest ? 2 : 1);
+  allocHungoffUses(ReservedSpace);
+
+  Op<0>() = ParentPad;
+  if (UnwindDest) {
+    setInstructionSubclassData(getSubclassDataFromInstruction() | 1);
+    setUnwindDest(UnwindDest);
+  }
 }
 
-BasicBlock *CatchPadInst::getSuccessorV(unsigned Idx) const {
-  return getSuccessor(Idx);
+/// growOperands - grow operands - This grows the operand list in response to a
+/// push_back style of operation. This grows the number of ops by 2 times.
+void CatchSwitchInst::growOperands(unsigned Size) {
+  unsigned NumOperands = getNumOperands();
+  assert(NumOperands >= 1);
+  if (ReservedSpace >= NumOperands + Size)
+    return;
+  ReservedSpace = (NumOperands + Size / 2) * 2;
+  growHungoffUses(ReservedSpace);
+}
+
+void CatchSwitchInst::addHandler(BasicBlock *Handler) {
+  unsigned OpNo = getNumOperands();
+  growOperands(1);
+  assert(OpNo < ReservedSpace && "Growing didn't work!");
+  setNumHungOffUseOperands(getNumOperands() + 1);
+  getOperandList()[OpNo] = Handler;
+}
+
+BasicBlock *CatchSwitchInst::getSuccessorV(unsigned idx) const {
+  return getSuccessor(idx);
 }
-unsigned CatchPadInst::getNumSuccessorsV() const {
+unsigned CatchSwitchInst::getNumSuccessorsV() const {
   return getNumSuccessors();
 }
-void CatchPadInst::setSuccessorV(unsigned Idx, BasicBlock *B) {
-  return setSuccessor(Idx, B);
+void CatchSwitchInst::setSuccessorV(unsigned idx, BasicBlock *B) {
+  setSuccessor(idx, B);
+}
+
+//===----------------------------------------------------------------------===//
+//                        FuncletPadInst Implementation
+//===----------------------------------------------------------------------===//
+void FuncletPadInst::init(Value *ParentPad, ArrayRef<Value *> Args,
+                          const Twine &NameStr) {
+  assert(getNumOperands() == 1 + Args.size() && "NumOperands not set up?");
+  std::copy(Args.begin(), Args.end(), op_begin());
+  setParentPad(ParentPad);
+  setName(NameStr);
+}
+
+FuncletPadInst::FuncletPadInst(const FuncletPadInst &FPI)
+    : Instruction(FPI.getType(), FPI.getOpcode(),
+                  OperandTraits<FuncletPadInst>::op_end(this) -
+                      FPI.getNumOperands(),
+                  FPI.getNumOperands()) {
+  std::copy(FPI.op_begin(), FPI.op_end(), op_begin());
+  setParentPad(FPI.getParentPad());
+}
+
+FuncletPadInst::FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad,
+                               ArrayRef<Value *> Args, unsigned Values,
+                               const Twine &NameStr, Instruction *InsertBefore)
+    : Instruction(ParentPad->getType(), Op,
+                  OperandTraits<FuncletPadInst>::op_end(this) - Values, Values,
+                  InsertBefore) {
+  init(ParentPad, Args, NameStr);
+}
+
+FuncletPadInst::FuncletPadInst(Instruction::FuncletPadOps Op, Value *ParentPad,
+                               ArrayRef<Value *> Args, unsigned Values,
+                               const Twine &NameStr, BasicBlock *InsertAtEnd)
+    : Instruction(ParentPad->getType(), Op,
+                  OperandTraits<FuncletPadInst>::op_end(this) - Values, Values,
+                  InsertAtEnd) {
+  init(ParentPad, Args, NameStr);
 }
 
 //===----------------------------------------------------------------------===//
 //                        TerminatePadInst Implementation
 //===----------------------------------------------------------------------===//
-void TerminatePadInst::init(BasicBlock *BB, ArrayRef<Value *> Args) {
-  if (BB)
+void TerminatePadInst::init(Value *ParentPad, BasicBlock *BB,
+                            ArrayRef<Value *> Args) {
+  if (BB) {
     setInstructionSubclassData(getSubclassDataFromInstruction() | 1);
-  if (BB)
-    Op<-1>() = BB;
-  std::copy(Args.begin(), Args.end(), op_begin());
+    setUnwindDest(BB);
+  }
+  std::copy(Args.begin(), Args.end(), arg_begin());
+  setParentPad(ParentPad);
 }
 
 TerminatePadInst::TerminatePadInst(const TerminatePadInst &TPI)
@@ -1036,22 +1002,24 @@ TerminatePadInst::TerminatePadInst(const TerminatePadInst &TPI)
   std::copy(TPI.op_begin(), TPI.op_end(), op_begin());
 }
 
-TerminatePadInst::TerminatePadInst(LLVMContext &C, BasicBlock *BB,
+TerminatePadInst::TerminatePadInst(Value *ParentPad, BasicBlock *BB,
                                    ArrayRef<Value *> Args, unsigned Values,
                                    Instruction *InsertBefore)
-    : TerminatorInst(Type::getVoidTy(C), Instruction::TerminatePad,
+    : TerminatorInst(Type::getVoidTy(ParentPad->getContext()),
+                     Instruction::TerminatePad,
                      OperandTraits<TerminatePadInst>::op_end(this) - Values,
                      Values, InsertBefore) {
-  init(BB, Args);
+  init(ParentPad, BB, Args);
 }
 
-TerminatePadInst::TerminatePadInst(LLVMContext &C, BasicBlock *BB,
+TerminatePadInst::TerminatePadInst(Value *ParentPad, BasicBlock *BB,
                                    ArrayRef<Value *> Args, unsigned Values,
                                    BasicBlock *InsertAtEnd)
-    : TerminatorInst(Type::getVoidTy(C), Instruction::TerminatePad,
+    : TerminatorInst(Type::getVoidTy(ParentPad->getContext()),
+                     Instruction::TerminatePad,
                      OperandTraits<TerminatePadInst>::op_end(this) - Values,
                      Values, InsertAtEnd) {
-  init(BB, Args);
+  init(ParentPad, BB, Args);
 }
 
 BasicBlock *TerminatePadInst::getSuccessorV(unsigned Idx) const {
@@ -1067,39 +1035,6 @@ void TerminatePadInst::setSuccessorV(unsigned Idx, BasicBlock *B) {
 }
 
 //===----------------------------------------------------------------------===//
-//                        CleanupPadInst Implementation
-//===----------------------------------------------------------------------===//
-void CleanupPadInst::init(ArrayRef<Value *> Args, const Twine &NameStr) {
-  assert(getNumOperands() == Args.size() && "NumOperands not set up?");
-  std::copy(Args.begin(), Args.end(), op_begin());
-  setName(NameStr);
-}
-
-CleanupPadInst::CleanupPadInst(const CleanupPadInst &CPI)
-    : Instruction(CPI.getType(), Instruction::CleanupPad,
-                  OperandTraits<CleanupPadInst>::op_end(this) -
-                      CPI.getNumOperands(),
-                  CPI.getNumOperands()) {
-  std::copy(CPI.op_begin(), CPI.op_end(), op_begin());
-}
-
-CleanupPadInst::CleanupPadInst(LLVMContext &C, ArrayRef<Value *> Args,
-                               const Twine &NameStr, Instruction *InsertBefore)
-    : Instruction(Type::getTokenTy(C), Instruction::CleanupPad,
-                  OperandTraits<CleanupPadInst>::op_end(this) - Args.size(),
-                  Args.size(), InsertBefore) {
-  init(Args, NameStr);
-}
-
-CleanupPadInst::CleanupPadInst(LLVMContext &C, ArrayRef<Value *> Args,
-                               const Twine &NameStr, BasicBlock *InsertAtEnd)
-    : Instruction(Type::getTokenTy(C), Instruction::CleanupPad,
-                  OperandTraits<CleanupPadInst>::op_end(this) - Args.size(),
-                  Args.size(), InsertAtEnd) {
-  init(Args, NameStr);
-}
-
-//===----------------------------------------------------------------------===//
 //                      UnreachableInst Implementation
 //===----------------------------------------------------------------------===//
 
@@ -4074,32 +4009,24 @@ InvokeInst *InvokeInst::cloneImpl() const {
 
 ResumeInst *ResumeInst::cloneImpl() const { return new (1) ResumeInst(*this); }
 
-CleanupEndPadInst *CleanupEndPadInst::cloneImpl() const {
-  return new (getNumOperands()) CleanupEndPadInst(*this);
-}
-
 CleanupReturnInst *CleanupReturnInst::cloneImpl() const {
   return new (getNumOperands()) CleanupReturnInst(*this);
 }
 
-CatchEndPadInst *CatchEndPadInst::cloneImpl() const {
-  return new (getNumOperands()) CatchEndPadInst(*this);
-}
-
 CatchReturnInst *CatchReturnInst::cloneImpl() const {
   return new (getNumOperands()) CatchReturnInst(*this);
 }
 
-CatchPadInst *CatchPadInst::cloneImpl() const {
-  return new (getNumOperands()) CatchPadInst(*this);
+CatchSwitchInst *CatchSwitchInst::cloneImpl() const {
+  return new CatchSwitchInst(*this);
 }
 
-TerminatePadInst *TerminatePadInst::cloneImpl() const {
-  return new (getNumOperands()) TerminatePadInst(*this);
+FuncletPadInst *FuncletPadInst::cloneImpl() const {
+  return new (getNumOperands()) FuncletPadInst(*this);
 }
 
-CleanupPadInst *CleanupPadInst::cloneImpl() const {
-  return new (getNumOperands()) CleanupPadInst(*this);
+TerminatePadInst *TerminatePadInst::cloneImpl() const {
+  return new (getNumOperands()) TerminatePadInst(*this);
 }
 
 UnreachableInst *UnreachableInst::cloneImpl() const {
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 58f9c5388bf..9862bfcc4fa 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -399,9 +399,9 @@ private:
   void visitEHPadPredecessors(Instruction &I);
   void visitLandingPadInst(LandingPadInst &LPI);
   void visitCatchPadInst(CatchPadInst &CPI);
-  void visitCatchEndPadInst(CatchEndPadInst &CEPI);
+  void visitCatchReturnInst(CatchReturnInst &CatchReturn);
   void visitCleanupPadInst(CleanupPadInst &CPI);
-  void visitCleanupEndPadInst(CleanupEndPadInst &CEPI);
+  void visitCatchSwitchInst(CatchSwitchInst &CatchSwitch);
   void visitCleanupReturnInst(CleanupReturnInst &CRI);
   void visitTerminatePadInst(TerminatePadInst &TPI);
 
@@ -2885,25 +2885,24 @@ void Verifier::visitEHPadPredecessors(Instruction &I) {
     }
     return;
   }
+  if (auto *CPI = dyn_cast<CatchPadInst>(&I)) {
+    if (!pred_empty(BB))
+      Assert(BB->getUniquePredecessor() == CPI->getCatchSwitch()->getParent(),
+             "Block containg CatchPadInst must be jumped to "
+             "only by its catchswitch.",
+             CPI);
+    return;
+  }
 
   for (BasicBlock *PredBB : predecessors(BB)) {
     TerminatorInst *TI = PredBB->getTerminator();
-    if (auto *II = dyn_cast<InvokeInst>(TI))
+    if (auto *II = dyn_cast<InvokeInst>(TI)) {
       Assert(II->getUnwindDest() == BB && II->getNormalDest() != BB,
              "EH pad must be jumped to via an unwind edge", &I, II);
-    else if (auto *CPI = dyn_cast<CatchPadInst>(TI))
-      Assert(CPI->getUnwindDest() == BB && CPI->getNormalDest() != BB,
-             "EH pad must be jumped to via an unwind edge", &I, CPI);
-    else if (isa<CatchEndPadInst>(TI))
-      ;
-    else if (isa<CleanupReturnInst>(TI))
-      ;
-    else if (isa<CleanupEndPadInst>(TI))
-      ;
-    else if (isa<TerminatePadInst>(TI))
-      ;
-    else
+    } else if (!isa<CleanupReturnInst>(TI) && !isa<TerminatePadInst>(TI) &&
+               !isa<CatchSwitchInst>(TI)) {
       Assert(false, "EH pad must be jumped to via an unwind edge", &I, TI);
+    }
   }
 }
 
@@ -2952,67 +2951,29 @@ void Verifier::visitCatchPadInst(CatchPadInst &CPI) {
   visitEHPadPredecessors(CPI);
 
   BasicBlock *BB = CPI.getParent();
+
   Function *F = BB->getParent();
   Assert(F->hasPersonalityFn(),
          "CatchPadInst needs to be in a function with a personality.", &CPI);
 
+  Assert(isa<CatchSwitchInst>(CPI.getParentPad()),
+         "CatchPadInst needs to be directly nested in a CatchSwitchInst.",
+         CPI.getParentPad());
+
   // The catchpad instruction must be the first non-PHI instruction in the
   // block.
   Assert(BB->getFirstNonPHI() == &CPI,
-         "CatchPadInst not the first non-PHI instruction in the block.",
-         &CPI);
+         "CatchPadInst not the first non-PHI instruction in the block.", &CPI);
 
-  if (!BB->getSinglePredecessor())
-    for (BasicBlock *PredBB : predecessors(BB)) {
-      Assert(!isa<CatchPadInst>(PredBB->getTerminator()),
-             "CatchPadInst with CatchPadInst predecessor cannot have any other "
-             "predecessors.",
-             &CPI);
-    }
-
-  BasicBlock *UnwindDest = CPI.getUnwindDest();
-  Instruction *I = UnwindDest->getFirstNonPHI();
-  Assert(
-      isa<CatchPadInst>(I) || isa<CatchEndPadInst>(I),
-      "CatchPadInst must unwind to a CatchPadInst or a CatchEndPadInst.",
-      &CPI);
-
-  visitTerminatorInst(CPI);
+  visitInstruction(CPI);
 }
 
-void Verifier::visitCatchEndPadInst(CatchEndPadInst &CEPI) {
-  visitEHPadPredecessors(CEPI);
-
-  BasicBlock *BB = CEPI.getParent();
-  Function *F = BB->getParent();
-  Assert(F->hasPersonalityFn(),
-         "CatchEndPadInst needs to be in a function with a personality.",
-         &CEPI);
-
-  // The catchendpad instruction must be the first non-PHI instruction in the
-  // block.
-  Assert(BB->getFirstNonPHI() == &CEPI,
-         "CatchEndPadInst not the first non-PHI instruction in the block.",
-         &CEPI);
-
-  unsigned CatchPadsSeen = 0;
-  for (BasicBlock *PredBB : predecessors(BB))
-    if (isa<CatchPadInst>(PredBB->getTerminator()))
-      ++CatchPadsSeen;
-
-  Assert(CatchPadsSeen <= 1, "CatchEndPadInst must have no more than one "
-                               "CatchPadInst predecessor.",
-         &CEPI);
+void Verifier::visitCatchReturnInst(CatchReturnInst &CatchReturn) {
+  Assert(isa<CatchPadInst>(CatchReturn.getOperand(0)),
+         "CatchReturnInst needs to be provided a CatchPad", &CatchReturn,
+         CatchReturn.getOperand(0));
 
-  if (BasicBlock *UnwindDest = CEPI.getUnwindDest()) {
-    Instruction *I = UnwindDest->getFirstNonPHI();
-    Assert(
-        I->isEHPad() && !isa<LandingPadInst>(I),
-        "CatchEndPad must unwind to an EH block which is not a landingpad.",
-        &CEPI);
-  }
-
-  visitTerminatorInst(CEPI);
+  visitTerminatorInst(CatchReturn);
 }
 
 void Verifier::visitCleanupPadInst(CleanupPadInst &CPI) {
@@ -3030,57 +2991,76 @@ void Verifier::visitCleanupPadInst(CleanupPadInst &CPI) {
          "CleanupPadInst not the first non-PHI instruction in the block.",
          &CPI);
 
+  auto *ParentPad = CPI.getParentPad();
+  Assert(isa<CatchSwitchInst>(ParentPad) || isa<ConstantTokenNone>(ParentPad) ||
+             isa<CleanupPadInst>(ParentPad) || isa<CatchPadInst>(ParentPad),
+         "CleanupPadInst has an invalid parent.", &CPI);
+
   User *FirstUser = nullptr;
   BasicBlock *FirstUnwindDest = nullptr;
   for (User *U : CPI.users()) {
     BasicBlock *UnwindDest;
     if (CleanupReturnInst *CRI = dyn_cast<CleanupReturnInst>(U)) {
       UnwindDest = CRI->getUnwindDest();
+    } else if (isa<CleanupPadInst>(U) || isa<CatchSwitchInst>(U) ||
+               isa<TerminatePadInst>(U)) {
+      continue;
     } else {
-      UnwindDest = cast<CleanupEndPadInst>(U)->getUnwindDest();
+      Assert(false, "bogus cleanuppad use", &CPI);
     }
 
     if (!FirstUser) {
       FirstUser = U;
       FirstUnwindDest = UnwindDest;
     } else {
-      Assert(UnwindDest == FirstUnwindDest,
-             "Cleanuprets/cleanupendpads from the same cleanuppad must "
-             "have the same unwind destination",
-             FirstUser, U);
+      Assert(
+          UnwindDest == FirstUnwindDest,
+          "cleanupret instructions from the same cleanuppad must have the same "
+          "unwind destination",
+          FirstUser, U);
     }
   }
 
   visitInstruction(CPI);
 }
 
-void Verifier::visitCleanupEndPadInst(CleanupEndPadInst &CEPI) {
-  visitEHPadPredecessors(CEPI);
+void Verifier::visitCatchSwitchInst(CatchSwitchInst &CatchSwitch) {
+  visitEHPadPredecessors(CatchSwitch);
+
+  BasicBlock *BB = CatchSwitch.getParent();
 
-  BasicBlock *BB = CEPI.getParent();
   Function *F = BB->getParent();
   Assert(F->hasPersonalityFn(),
-         "CleanupEndPadInst needs to be in a function with a personality.",
-         &CEPI);
+         "CatchSwitchInst needs to be in a function with a personality.",
+         &CatchSwitch);
 
-  // The cleanupendpad instruction must be the first non-PHI instruction in the
+  // The catchswitch instruction must be the first non-PHI instruction in the
   // block.
-  Assert(BB->getFirstNonPHI() == &CEPI,
-         "CleanupEndPadInst not the first non-PHI instruction in the block.",
-         &CEPI);
+  Assert(BB->getFirstNonPHI() == &CatchSwitch,
+         "CatchSwitchInst not the first non-PHI instruction in the block.",
+         &CatchSwitch);
 
-  if (BasicBlock *UnwindDest = CEPI.getUnwindDest()) {
+  if (BasicBlock *UnwindDest = CatchSwitch.getUnwindDest()) {
     Instruction *I = UnwindDest->getFirstNonPHI();
-    Assert(
-        I->isEHPad() && !isa<LandingPadInst>(I),
-        "CleanupEndPad must unwind to an EH block which is not a landingpad.",
-        &CEPI);
+    Assert(I->isEHPad() && !isa<LandingPadInst>(I),
+           "CatchSwitchInst must unwind to an EH block which is not a "
+           "landingpad.",
+           &CatchSwitch);
   }
 
-  visitTerminatorInst(CEPI);
+  auto *ParentPad = CatchSwitch.getParentPad();
+  Assert(isa<CatchSwitchInst>(ParentPad) || isa<ConstantTokenNone>(ParentPad) ||
+             isa<CleanupPadInst>(ParentPad) || isa<CatchPadInst>(ParentPad),
+         "CatchSwitchInst has an invalid parent.", ParentPad);
+
+  visitTerminatorInst(CatchSwitch);
 }
 
 void Verifier::visitCleanupReturnInst(CleanupReturnInst &CRI) {
+  Assert(isa<CleanupPadInst>(CRI.getOperand(0)),
+         "CleanupReturnInst needs to be provided a CleanupPad", &CRI,
+         CRI.getOperand(0));
+
   if (BasicBlock *UnwindDest = CRI.getUnwindDest()) {
     Instruction *I = UnwindDest->getFirstNonPHI();
     Assert(I->isEHPad() && !isa<LandingPadInst>(I),
@@ -3115,6 +3095,11 @@ void Verifier::visitTerminatePadInst(TerminatePadInst &TPI) {
            &TPI);
   }
 
+  auto *ParentPad = TPI.getParentPad();
+  Assert(isa<CatchSwitchInst>(ParentPad) || isa<ConstantTokenNone>(ParentPad) ||
+             isa<CleanupPadInst>(ParentPad) || isa<CatchPadInst>(ParentPad),
+         "TerminatePadInst has an invalid parent.", ParentPad);
+
   visitTerminatorInst(TPI);
 }