[WinEH] Emit int3 after noreturn calls on Win64

The Win64 unwinder disassembles forwards from each PC to try to
determine if this PC is in an epilogue. If so, it skips calling the EH
personality function for that frame. Typically, this means you cannot
catch an exception in the same frame that you threw it, because 'throw'
calls a noreturn runtime function.

Previously we avoided this problem with the TrapUnreachable
TargetOption, but that's a much bigger hammer than we need. All we need
is a 1 byte non-epilogue instruction right after the call.  Instead,
what we got was an unconditional branch to a shared block containing the
ud2, potentially 7 bytes instead of 1. So, this reverts r206684, which
added TrapUnreachable, and replaces it with something better.

The new code pattern matches for invoke/call followed by unreachable and
inserts an int3 into the DAG. To be 100% watertight, we would need to
insert SEH_Epilogue instructions into all basic blocks ending in a call
with no terminators or successors, but in practice this is unlikely to
come up.

llvm-svn: 248959

4 files changed, 26 insertions, 15 deletions

diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index 13b097cfc60..f97edb4ef09 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -1569,10 +1569,8 @@ bool FastISel::selectOperator(const User *I, unsigned Opcode) {
   }
 
   case Instruction::Unreachable:
-    if (TM.Options.TrapUnreachable)
-      return fastEmit_(MVT::Other, MVT::Other, ISD::TRAP) != 0;
-    else
-      return true;
+    // Nothing to emit.
+    return true;
 
   case Instruction::Alloca:
     // FunctionLowering has the static-sized case covered.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 72af2760bb7..a48de572113 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -2205,10 +2205,7 @@ void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
                           getValue(I.getAddress())));
 }
 
-void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
-  if (DAG.getTarget().Options.TrapUnreachable)
-    DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
-}
+void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {}
 
 void SelectionDAGBuilder::visitFSub(const User &I) {
   // -0.0 - X --> fneg
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 6fd04733e01..0ed196a4acb 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2945,6 +2945,20 @@ static SDValue getMOVL(SelectionDAG &DAG, SDLoc dl, EVT VT, SDValue V1,
   return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
 }
 
+/// Check if the fall through instruction after a call site is unreachable.
+/// FIXME: This will fail if there are interesting non-code generating IR
+/// instructions between the call and the unreachable (lifetime.end). In
+/// practice, this should be rare because optimizations like to delete non-call
+/// code before unreachable.
+static bool isCallFollowedByUnreachable(ImmutableCallSite CS) {
+  const Instruction *NextInst;
+  if (auto *II = dyn_cast<InvokeInst>(CS.getInstruction()))
+    NextInst = II->getNormalDest()->getFirstNonPHIOrDbg();
+  else
+    NextInst = CS.getInstruction()->getNextNode();
+  return isa<UnreachableInst>(NextInst);
+}
+
 SDValue
 X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                              SmallVectorImpl<SDValue> &InVals) const {
@@ -3450,6 +3464,15 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     InFlag = Chain.getValue(1);
   }
 
+  if (Subtarget->isTargetWin64() && CLI.CS) {
+    // Look for a call followed by unreachable. On Win64, we need to ensure that
+    // the call does not accidentally fall through to something that looks like
+    // an epilogue. We do this by inserting a DEBUGTRAP, which lowers to int3,
+    // which is what MSVC emits after noreturn calls.
+    if (isCallFollowedByUnreachable(*CLI.CS))
+      Chain = DAG.getNode(ISD::DEBUGTRAP, dl, MVT::Other, Chain);
+  }
+
   // Handle result values, copying them out of physregs into vregs that we
   // return.
   return LowerCallResult(Chain, InFlag, CallConv, isVarArg,
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 2e869eb7c3c..d318244e0f7 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -110,13 +110,6 @@ X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT,
                         OL),
       TLOF(createTLOF(getTargetTriple())),
       Subtarget(TT, CPU, FS, *this, Options.StackAlignmentOverride) {
-  // Windows stack unwinder gets confused when execution flow "falls through"
-  // after a call to 'noreturn' function.
-  // To prevent that, we emit a trap for 'unreachable' IR instructions.
-  // (which on X86, happens to be the 'ud2' instruction)
-  if (Subtarget.isTargetWin64())
-    this->Options.TrapUnreachable = true;
-
   // By default (and when -ffast-math is on), enable estimate codegen for
   // everything except scalar division. By default, use 1 refinement step for
   // all operations. Defaults may be overridden by using command-line options.